|| || || || || ||

---

ApplyConcepts.h

Go to the documentation of this file.

#ifndef Impala_Core_Training_ApplyConcepts_h
#define Impala_Core_Training_ApplyConcepts_h

#include "Core/Database/MakeRawDataSet.h"
#include "Core/Feature/FeatureTable.h"
#include "Core/Training/Classifier.h"
#include "Process/MemoryInfo.h"
#include "Persistency/KeywordListRepository.h"
#include "Persistency/SvmRepository.h"
#include "Persistency/FeatureTableRepository.h"
#include "Persistency/SimilarityTableSetRepository.h"
#include "Core/Training/ApplyConceptsHelperFeatures.h"
#include "Core/Training/ApplyConceptsHelperKernels.h"
#include "Core/Training/ApplyConceptsHelperFik.h"

namespace Impala
{
namespace Core
{
namespace Training
{


class ApplyConcepts
{
public:
    typedef Table::ScoreTable ScoreTable;
    typedef Feature::FeatureDefinition FeatureDefinition;
    typedef Feature::FeatureTable FeatureTable;
    typedef Table::KeywordList KeywordList;
    typedef Persistency::ModelLocator ModelLocator;
    typedef Persistency::FeatureLocator FeatureLocator;
    typedef Persistency::SimilarityTableSetLocator SimilarityTableSetLocator;

public:

    ApplyConcepts(CmdOptions& options, Database::RawDataSet* dataSet,
                  bool onIndex)
    {
        if (options.GetNrArg() < 6)
        {
            ILOG_ERROR("Missing argument");
            return;
        }
        mDataSet = dataSet;
        mAnnoSet = Database::MakeRawDataSet(options.GetArg(2));
        String conceptSet = options.GetArg(3);
        String modelType = options.GetArg(4);
        String featureName = options.GetArg(5);
        Persistency::KeywordListLocator keyLoc(mAnnoSet->GetLocator(),
                                               conceptSet);
        mConcepts = *(Persistency::KeywordListRepository().Get(keyLoc));

        String walkType =
            (onIndex) ? options.GetString("classifyIndexCat") : "dummy";
        String container = (onIndex) ? "" : "dummy";
        mFeatLoc = FeatureLocator(mDataSet->GetLocator(), false, onIndex,
                                  walkType, featureName, container);
        mSimSetLoc = SimilarityTableSetLocator(mDataSet->GetLocator(), onIndex,
                                               walkType, conceptSet, modelType,
                                               featureName, container);
        mModelLoc = ModelLocator(mAnnoSet->GetLocator(), conceptSet, modelType,
                                 featureName, mConcepts[0]);
        mHelper = MakeHelper(options.GetBool("precomputed"),
                             options.GetBool("fik"));
        mHelper->Initialise(mAnnoSet, mModelLoc);
        mModelLoc.SetConcept("dummy");
        mKernelDataOnly = options.GetBool("kernelDataOnly");
    }

    virtual
    ~ApplyConcepts()
    {
        delete mHelper;
        delete mAnnoSet;
    }

    virtual void
    SetWalkType(CString walkType)
    {
        mFeatLoc.SetWalkType(walkType);
        mSimSetLoc.SetWalkType(walkType);
    }

    virtual void
    NextContainer(CString container)
    {
        mFeatLoc.SetContainer(container);
        mSimSetLoc.SetContainer(container);
        if ((!CmdOptions::GetInstance().GetBool("override")) &&
            Persistency::SimilarityTableSetRepository().Exists(mSimSetLoc))
        {
            ILOG_INFO("SimilarityTableSet already exists, skipping");
            return;
        }
        ILOG_DEBUG("loading features: "<<Process::MemoryInfo::GetUsageString());
        FeatureTable* features = mHelper->LoadFeatures(mFeatLoc);
        if (mKernelDataOnly)
        {
            ILOG_INFO("KernelDataOnly so skipping real apply");
            return;
        }

        ILOG_DEBUG("done features: "<< Process::MemoryInfo::GetUsageString());
        Table::SimilarityTableSet simSet(mConcepts, features->Size());
        Core::Table::Copy(simSet.GetQuidTable(), features); // copy Quids only

        Timer timerTotal;
        for (int i=0 ; i<mConcepts.size() ; i++)
        {
            ILOG_INFO("concept " << i << " of " << mConcepts.size());
            Timer timerConcept;
            ModelLocator modelLoc = mModelLoc;
            modelLoc.SetConcept(mConcepts[i]);
            ILOG_INFO("model " << modelLoc);
            ILOG_DEBUG("pre-predict: "<< Process::MemoryInfo::GetUsageString());
            ScoreTable* scores = mHelper->Predict(modelLoc);
            simSet.SetSimTable(i, scores);
            delete scores;
            ILOG_DEBUG("post-predict: "<< Process::MemoryInfo::GetUsageString());
            simSet.ComputeRank(i, true);
            ILOG_INFO("time this concept = " << timerConcept.SplitTimeStr()
                      << ", total = " << timerTotal.SplitTimeStr());
        }
        ILOG_DEBUG("pre-save: "<< Process::MemoryInfo::GetUsageString());
        Persistency::SimilarityTableSetRepository().Add(mSimSetLoc, &simSet);
        ILOG_DEBUG("pre-unload: "<< Process::MemoryInfo::GetUsageString());
        mHelper->UnloadFeatures();
        ILOG_DEBUG("post-unload: "<< Process::MemoryInfo::GetUsageString());
    }

private:

    ApplyConceptsHelper*
    MakeHelper(bool isPrecomputed, bool fik)
    {
        if (fik)
            return new ApplyConceptsHelperFik;
        if (isPrecomputed)
            return new ApplyConceptsHelperKernels;
        return new ApplyConceptsHelperFeatures;
    }

    ApplyConceptsHelper*      mHelper;
    KeywordList               mConcepts;
    ModelLocator              mModelLoc;
    FeatureLocator            mFeatLoc;
    SimilarityTableSetLocator mSimSetLoc;
    Database::RawDataSet*     mDataSet;
    Database::RawDataSet*     mAnnoSet;
    bool                      mKernelDataOnly;

    ILOG_VAR_DEC;

};

ILOG_VAR_INIT(ApplyConcepts, Impala.Core.Training);

} // namespace Training
} // namespace Core
} // namespace Impala

#endif

---