|| || || || || ||

---

ParameterSearcher.h

Go to the documentation of this file.

#ifndef Impala_Core_Training_ParameterSearcher_h
#define Impala_Core_Training_ParameterSearcher_h

#include "Util/PropertySet.h"
#include "Util/RangeIterator.h"

#include "Link/Mpi/MpiFuncs.h"
#include "Util/Mpi/Reduce.h"
#include "Util/Mpi/JobManager.h"
#include "Core/Training/ParameterEvaluator.h"

#include <algorithm>
#include <sstream>

namespace Impala
{
namespace Core
{
namespace Training
{

class ParameterSearcher
{
public:
    ParameterSearcher(Util::PropertySet* properties, ParameterEvaluator* evaluator)
    {
#ifdef MPI_USED
        mParallelMode = true;
#else
        mParallelMode = false;
#endif
        mEvaluator = evaluator;
        mProperties = *properties;
        mBestScore = 0;

        int i;
        for(i=0 ; i<mProperties.Size() ; ++i)
        {
            std::string val = mProperties.GetValue(i);
            ILOG_DEBUG("prop found: "<< val);
            if(Util::IsRangeDefinition(val))
            {
                mIterator.AddRange(mProperties.GetName(i), val);
            }
        }
        int r = mProperties.GetInt("repetitions");
        int f = mProperties.GetInt("folds");
        mScores.assign(mIterator.GetTotalIterations() * f * r, 0);
    }

    virtual
    ~ParameterSearcher()
    {
        delete mEvaluator;
    }

    void
    OverrideParallelMode(bool mode)
    {
        mParallelMode = mode;
    }

    Util::PropertySet
    Search()
    {
        int r = mProperties.GetInt("repetitions");
        int f = mProperties.GetInt("folds");
        mScores.assign(mIterator.GetTotalIterations() * f * r, 0);
        mBestScore = 0;
        // first try all parameter settings evaluation is stored in mScores
        if(mParallelMode)
            SearchParallel();
        else
            SearchSequential();

        // average the scores
        std::vector<double> scores;
        int iters = mIterator.GetTotalIterations();
        scores.assign(iters, 0);
        int i;
        for(i=0 ; i<mScores.size() ; ++i)
        {
            scores[i%iters] += mScores[i];
        }
        double d = f*r;
        for(i=0 ; i<scores.size() ; ++i)
        {
            scores[i] /= d;
        }

        // select the wining setting
        std::vector<double>::iterator best =
            std::max_element(scores.begin(), scores.end());
        int bestIteration = best - scores.begin();
        mIterator.SetIteration(bestIteration);
        mIterator.GetParameters(&mProperties);
        mBestScore = *best;
        ILOG_INFO_HEADNODE("best score = " << mBestScore << 
                " @ params: " << mProperties.GetDescription());
        return mProperties;
    }

    double
    GetBestScore()
    {
        return mBestScore;
    }

    std::vector<Util::PropertySet*>
    GetAllScores()
    {
        std::vector<Util::PropertySet*> res;
        int index = 0;
        for (int r=0 ; r<mProperties.GetInt("repetitions") ; r++)
        {
            for (int f=0 ; f<mProperties.GetInt("folds") ; f++)
            {
                for (int i=0 ; i<mIterator.GetTotalIterations() ; i++)
                {
                    Util::PropertySet* props = new Util::PropertySet;
                    mIterator.SetIteration(i);
                    mIterator.GetParameters(props);
                    props->Add("score", mScores[index]);
                    index++;
                    props->Add("repetition", r);
                    props->Add("fold", f);
                    res.push_back(props);
                }
            }
        }
        return res;
    }

private:

    double
    CallEvaluator(int iteration, int index)
    {
        mIterator.SetIteration(iteration);
        mIterator.GetParameters(&mProperties);
        ILOG_DEBUG("calling evaluator::Evaluate");
        double score = mEvaluator->Evaluate(&mProperties);
        ILOG_DEBUG("evaluator::Evaluate returned");
        mScores[index] += score;
        return score;
    }

    void
    SearchSequential()
    {
        if(mEvaluator == 0)
        {
            ILOG_ERROR("no evaluator set");
            return;
        }

        int index = 0;
        int repetition, repetitionCount;
        repetitionCount = mProperties.GetInt("repetitions");
        ILOG_INFO("#repetitions " << repetitionCount)
        for(repetition=0 ; repetition<repetitionCount ; ++repetition)
        {
            ILOG_INFO("repetition #" << repetition)
            mEvaluator->SetRepetition(repetition, repetitionCount);
            int fold, foldCount;
            foldCount = mProperties.GetInt("folds");
            for(fold=0 ; fold<foldCount ; ++fold)
            {
                ILOG_INFO("fold #" << fold);
                mEvaluator->SetFold(fold, foldCount);
                int i;
                for(i=0 ; i<mIterator.GetTotalIterations() ; ++i)
                {
                    double score = CallEvaluator(i, index);
                    ++index;
                    ILOG_INFO("it " << i << ", " << mProperties.GetDescription() <<
                              " : " << score);
                }
            }
        }
    }


    void
    SearchParallel()
    {
        if(Link::Mpi::MyId() == 0)
            SearchServer();
        else
            SearchClient();
        Util::Mpi::Reduce(mScores);
    }

    void
    SearchServer()
    {
        ILOG_DEBUG_NODE("I am the server");
        // create job groups
        Util::Mpi::JobManager jobManager;
        int f;
        for(f=0 ; f<mProperties.GetInt("folds") ; ++f)
        {
            int r;
            for(r=0 ; r<mProperties.GetInt("repetitions") ; ++r)
            {
                std::string id("r0f0");
                id[1] = '0'+r;
                id[3] = '0'+f;
                Util::PropertySet ps;
                ps.Add("repetition", r);
                ps.Add("fold", f);
                // maybe we shouldn't create a group with a description, but
                // rather with the properties themselves
                jobManager.CreateGroup(ps.GetDescription(), id,
                                       mIterator.GetTotalIterations());
            }
        }

        // handle request from clients until all work is done
        bool done=false;
        int runningJobs=0;
        while(true)
        {
            //receive request for work
            int source;
            std::string message = Link::Mpi::ReceiveString(source);
            ILOG_DEBUG_NODE("server got mssg: " << message);
            Util::PropertySet job(message);
            if(job.GetString("JobManager::job-id") != "-1")
                --runningJobs;
            //pick a job from a group
            jobManager.GetJob(&job);
            //send job
            if(job.GetString("JobManager::job-id") != "-1")
                ++runningJobs;
            std::ostringstream oss;
            job.Print(oss);
            ILOG_DEBUG_NODE("SERVER: sending job assignment " << 
                job.GetString("JobManager::job-id") << "," << 
                job.GetString("JobManager::group-id") << " to " << source);
            Link::Mpi::SendString(oss.str(), source);
            if(runningJobs == 0)
                break;
        }
    }

    void
    SearchClient()
    {
        ILOG_DEBUG_NODE("I am a client");
        int id = Link::Mpi::MyId(); 
        int lastRepetition = -1;
        int lastFold = -1;
        Util::PropertySet job;
        job.Add("JobManager::job-id", "-1");
        while(true)
        {
            ILOG_DEBUG_NODE("client sends request for work");
            //send request for work
            Link::Mpi::SendString(job.GetDescription(),0);
            //receive assignment
            int source;
            std::string message = Link::Mpi::ReceiveString(source);
            job.Parse(message);
            int repetition = job.GetInt("repetition", -1);
            int fold = job.GetInt("fold", -1);
            int iteration = job.GetInt("JobManager::job-id", -1);

            ILOG_DEBUG_NODE("got assignment " << iteration << " from group " <<
                            job.GetString("JobManager::group-id"));
            if(iteration == -1)
                break;
            if(repetition != lastRepetition)
            {
                lastFold = -1; // need to reinitialise the folds
                mEvaluator->SetRepetition(repetition,
                                          mProperties.GetInt("repetitions"));
                lastRepetition = repetition;
            }
            if(fold != lastFold)
            {
                mEvaluator->SetFold(fold, mProperties.GetInt("folds"));
                lastFold = fold;
            }
            int iters = mIterator.GetTotalIterations();
            int folds = mProperties.GetInt("folds");
            int index = (((repetition * folds) + fold) * iters) + iteration;
            double score = CallEvaluator(iteration, index);
            ILOG_INFO("r" << repetition << " f" << fold << " i" << iteration <<
                      ", " <<  mProperties.GetDescription() << " -> " << score);
        }
    }

    // copy not implemented
    ParameterSearcher* operator=(ParameterSearcher&);
    ParameterSearcher(ParameterSearcher&);

    ParameterEvaluator *mEvaluator;
    Util::PropertySet mProperties;
    Util::RangeIterator mIterator;
    std::vector<double> mScores;
    double mBestScore;
    int mScoreIndex;
    bool mParallelMode;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(ParameterSearcher, Impala.Core.Training);



}//namespace
}//namespace
}//namespace

#endif

---