|| || || || || ||

---

Factory.h

Go to the documentation of this file.

#ifndef Impala_Core_Training_Factory_h
#define Impala_Core_Training_Factory_h

#include "Core/Training/ParameterSearcher.h"
#include "Core/Training/MakeEvaluation.h"
#include "Core/Training/Svm.h"
#include "Core/Training/BestFile.h"
#include "Core/Training/ClassifierEvaluator.h"
#include "Core/Training/TrainDataSrcFeature.h"
#include "Core/Training/TrainDataSrcKernelDistributed.h"
#include "Core/Database/MakeRawDataSet.h"
#include "Persistency/KeywordListRepository.h"
#include "Persistency/AnnotationTableRepository.h"
#include "Core/Training/LoadDistributedAccess.h"
#include "Util/StlHelpers.h"

namespace Impala
{
namespace Core
{
namespace Training
{

class Factory
{
public:

    typedef Persistency::FeatureLocator FeatureLocator;
    typedef Persistency::ModelLocator ModelLocator;

    Factory(CmdOptions* options, bool doClassifier)
    {
        mOptions = options;
        String setName = mOptions->GetArg(0);
        if (Persistency::ImageSetsRepository::GetInstance().Contains(setName))
        {
            ImageSet::ImageSet* is = ImageSet::MakeImageSet(setName);
            is->SetImageSrc(true, true, true);
            mDataSet = is;
        }
        else
        {
            mDataSet = VideoSet::MakeVideoSet(setName);
        }

        if (doClassifier)
            mProperties = MakeClassifierProperties();
        else
            mProperties = 0;

        mConceptSet = mOptions->GetArg(1);
        mModelType = mOptions->GetArg(2);
        mFeature = mOptions->GetArg(3);
        mModelLoc = ModelLocator(mDataSet->GetLocator(), mConceptSet,
                                 mModelType, mFeature, "dummy");
        mCodebookLoc = FeatureLocator(mDataSet->GetLocator(), true, true, "",
                                      mFeature, "");
        mConceptStart = mOptions->GetInt("start", 0);
        mConceptNumber = mOptions->GetInt("number", -1);
        mDistributedAccess = 0;
    }

    virtual
    ~Factory()
    {
        if (mProperties)
        {
            delete mProperties;
            mProperties = 0;
        }
        if (mDistributedAccess)
        {
            mDistributedAccess->Unsubscribe();
            delete mDistributedAccess;
        }
    }

    Util::PropertySet*
    MakeClassifierProperties()
    {
        ILOG_VAR(Samples.mainCrossValidate.GetProperties);
        Util::PropertySet* properties = new Util::PropertySet;
        // todo: maybe we can change these calls into a loop.
        properties->Add("w1", mOptions->GetString("w1"));
        properties->Add("w2", mOptions->GetString("w2"));
        properties->Add("autoweight", mOptions->GetString("autoweight"));
        if(properties->GetBool("autoweight"))
        {
            properties->Add("w1", "1");
            properties->Add("w2", "1");
        }
        properties->Add("C", mOptions->GetString("C"));
        properties->Add("gamma", mOptions->GetString("gamma"));
        properties->Add("cache", mOptions->GetString("cache"));
        properties->Add("probability", mOptions->GetInt("probability"));
        properties->Add("kernel", mOptions->GetString("kernel"));
        properties->Add("precompute-kernel", 
                        mOptions->GetString("precompute-kernel"));
        properties->Add("folds", mOptions->GetInt("folds"));
        properties->Add("repetitions", mOptions->GetInt("repetitions"));
        properties->Add("episode-constrained",
                        mOptions->GetInt("episode-constrained"));
        properties->Add("evaluator", mOptions->GetString("evaluator"));
        properties->Add("maxVideoId", mOptions->GetInt("maxVideoId"));
        properties->Add("maxPosPerVideo", mOptions->GetInt("maxPosPerVideo"));
        properties->Add("maxNegPerVideo", mOptions->GetInt("maxNegPerVideo"));
        properties->Add("restrictTestFoldSet",
                        mOptions->GetInt("restrictTestFoldSet"));
        if (properties->GetInt("episode-constrained") && mDataSet->IsImageSet())
        {
            ILOG_WARN("ImageSet learning doesn't support episode-constrained!");
            properties->Add("episode-constrained", 0);
        }
        if (!properties->GetBool("episode-constrained"))
            ILOG_INFO_HEADNODE("not using episode constrained.")
        else
            ILOG_INFO_HEADNODE("using episode constrained.")
        ILOG_INFO_HEADNODE("using " << properties->GetString("evaluator") <<
                           " for evaluation.");

        return properties;
    }

    Util::PropertySet*
    GetProperties()
    {
        return mProperties;
    }

    static Classifier*
    MakeClassifier(String type)
    {
        if (type == "svm")
            return new Svm();
        ILOG_ERROR("unable to make classifier of type " << type);
        return 0;
    }

    ParameterSearcher*
    MakeSearcher(Table::AnnotationTable* annotation)
    {
        String evalDesc = mProperties->GetString("evaluator");
        Evaluation* eval = MakeEvaluation(evalDesc, annotation);
        Classifier* svm = MakeClassifier("svm");
        TrainDataSrc* src = MakeDataSrc(annotation);
        ParameterEvaluator* evaluator = new ClassifierEvaluator(svm, src, eval);
        Training::ParameterSearcher* searcher;
        searcher = new Training::ParameterSearcher(mProperties, evaluator);
        if (mProperties->GetString("kernel","rbf") == "dist-precomputed")
            searcher->OverrideParallelMode(false);
        return searcher;
    }

    TrainDataSrc*
    MakeDataSrc(Table::AnnotationTable* anno)
    {
        String kernelName = mProperties->GetString("kernel", "rbf");
        ILOG_INFO_HEADNODE("MakeDataSrc with kernel " << kernelName);
        if (kernelName == "dist-precomputed")
        {
            Matrix::DistributedAccess* da = GetDistributedAccess();
            return new TrainDataSrcKernelDistributed(da, anno);
        }
        return new TrainDataSrcFeature(anno, mDataSet, mFeature);
    }

    ModelLocator
    GetModelLocator()
    {
        return mModelLoc;
    }

    Array::Array2dVec3UInt8*
    MakeImage(Quid q)
    {
        QuidObj qo(q);
        if (qo.Class() != mDataSet->GetQuidClass())
        {
            ILOG_ERROR("[MakeImage] DataSet doesn't contain "<< qo.Class());
            return 0;
        }
        if (mDataSet->IsImageSet())
        {
            return static_cast<ImageSet::ImageSet*>(mDataSet)->GetImage(qo.Id());
        }
        else
        {
            ILOG_WARNING("MakeImage not tested for videosets");
            Stream::RgbDataSrc* src =
                static_cast<VideoSet::VideoSet*>(mDataSet)->GetVideo(qo.Object());
            src->GotoFrame(qo.Id());
            Array::Array2dVec3UInt8* im = Array::ArrayCreate<Array::Array2dVec3UInt8>
                (src->FrameWidth(), src->FrameHeight(), 0, 0, src->DataPtr(), true);
            delete src;
            return im;
        }
    }

    std::vector<String>
    MakeConceptList()
    {
        typedef Persistency::KeywordListLocator KeywordListLocator;
        typedef Persistency::KeywordListRepository KeywordListRepository;
        KeywordListLocator loc(mDataSet->GetLocator(), mConceptSet);
        std::vector<String> conceptList = *(KeywordListRepository().Get(loc));

        // truncate the concept list according to conceptStart and conceptNumber
        Util::SubSelectInPlace(&conceptList, mConceptStart, mConceptNumber);
        return conceptList;
    }

    Table::AnnotationTable*
    MakeAnnotation(String concept)
    {
        typedef Persistency::AnnotationTableLocator AnnotationTableLocator;
        typedef Persistency::AnnotationTableRepository AnnotationTableRepository;
        AnnotationTableLocator loc(mDataSet->GetLocator(),
                                   mDataSet->GetQuidClass(),
                                   mConceptSet, concept);
        Table::AnnotationTable* annotation = AnnotationTableRepository().Get(loc);
        /*
        int maxV = mProperties->GetInt("maxVideoId");
        mAnnotation->SelectQuidObjectMaxId(maxV);
        int maxP = mProperties->GetInt("maxPosPerVideo");
        mAnnotation->SelectQuidObjectMaxPositive(maxP);
        int maxN = mProperties->GetInt("maxNegPerVideo");
        mAnnotation->SelectQuidObjectMaxNegative(maxN);
        double pos = mAnnotation->GetNrPositive();
        double neg = mAnnotation->GetNrNegative();
        */
        ILOG_INFO_HEADNODE("Annotations: size = " << annotation->Size() );
        ILOG_INFO_HEADNODE("nr pos = "<< annotation->GetNrPositive() <<
                           ", nr neg = "<< annotation->GetNrNegative());
        return annotation;
    }

    Database::RawDataSet*
    GetDataSet()
    {
        return mDataSet;
    }

    Matrix::DistributedAccess*
    GetDistributedAccess()
    {
        if(mDistributedAccess == 0)
        {
            if(Link::Mpi::NrProcs() == 1)
            {
                mDistributedAccess = Training::LoadDistributedAccess
                    (mModelType, mFeature, mDataSet, 0, 0, 1);
            }
            else
            {
                mDistributedAccess = Training::LoadDistributedAccess
                    (mModelType, mFeature, mDataSet, 0, 1,
                     Link::Mpi::NrProcs() - 1);
                if(Link::Mpi::MyId() == 0)
                    mDistributedAccess->Subscribe();
            }
        }
        return mDistributedAccess;
    }

    void
    ServeDistributedAccess()
    {
        Matrix::DistributedAccess* da = GetDistributedAccess();
        da->StartEventLoop();
    }

    bool
    CodebookExists()
    {
        return Persistency::FeatureTableRepository().Exists(mCodebookLoc);
    }

    void
    WriteCodebook(Feature::FeatureTable* forest)
    {
        Persistency::FeatureTableRepository().Add(mCodebookLoc, forest);
    }

    Feature::FeatureDefinition
    GetFeatureDefinition()
    {
        return Feature::FeatureDefinition(mFeature);
    }

private:

    CmdOptions* mOptions;
    Database::RawDataSet* mDataSet;
    Util::PropertySet* mProperties;
    int mConceptStart;
    int mConceptNumber;
    String mConceptSet;
    String mModelType;
    String mFeature;
    ModelLocator mModelLoc;
    FeatureLocator mCodebookLoc;
    Matrix::DistributedAccess* mDistributedAccess;

    ILOG_VAR_DECL;
};

ILOG_VAR_INIT(Factory, Impala.Core.Training);

} //namespace Training
} //namespace Core
} //namespace Impala

#endif

---