|| || || || || ||

---

ApplyConceptsHelperKernels.h

Go to the documentation of this file.

#ifndef Impala_Core_Training_ApplyConceptsHelperKernels_h
#define Impala_Core_Training_ApplyConceptsHelperKernels_h

#include "Core/Training/ApplyConceptsHelper.h"
#include "Util/StringParser.h"
#include "Core/Array/Set.h"
#include "Core/Array/Add.h"
#include "Core/Array/DivVal.h"
#include "Core/Array/MulVal.h"
#include "Core/Array/Exp.h"
#include "Core/Vector/Apply.h"
#include "Core/Vector/Chi2Distance.h"
#include "Core/Training/TrainDataSrcKernelMatrix.h"
#include "Persistency/FeatureTableRepository.h"
#include "Persistency/DistributedAccessRepository.h"
#include "Persistency/KernelMatrixRepository.h"
#include "Persistency/SvmRepository.h"

namespace Impala
{
namespace Core
{
namespace Training
{


class ApplyConceptsHelperKernels : public ApplyConceptsHelper
{
public:

    ApplyConceptsHelperKernels() 
    {
        mKernelMatrix = 0;
        mFeatureExternal = 0;
    }

    virtual 
    ~ApplyConceptsHelperKernels()
    {
    }
        
    void
    Initialise(Database::RawDataSet* annoSet, const ModelLocator& modelLoc)
    {
        CmdOptions& options = CmdOptions::GetInstance();
        mFeatureIndexCat = options.GetString("featureIndexCat");
        mAnnoSet = annoSet;
        mModel = modelLoc.GetModel();
        mKernelMatrixName = modelLoc.GetFeature();
        typedef Persistency::KernelMatrixLocator KernelMatrixLocator;
        KernelMatrixLocator loc(mAnnoSet->GetLocator(), true, "", "",
                                mModel, mKernelMatrixName, "");
        loc.SetFeatureIndexCat(mFeatureIndexCat);
        loc.SetStartNode(0);
        loc.SetNodeCount(Link::Mpi::NrProcs());
        loc.SetDoParts(0);
        mDA = Persistency::DistributedAccessRepository().Get(loc);
    }

    FeatureTable*
    LoadFeatures(const FeatureLocator& featLoc)
    {
        typedef Persistency::KernelMatrixLocator KernelMatrixLocator;
        typedef Persistency::KernelMatrixRepository KernelMatrixRepository;
        typedef Persistency::FeatureTableRepository FeatureTableRepository;
        CmdOptions& options = CmdOptions::GetInstance();
        if (mFeatureExternal)
            ILOG_ERROR("features not empty");

        KernelMatrixLocator kernelDataLoc
            (featLoc, false, featLoc.GetWalkType(), mAnnoSet->GetSetName(),
             mModel, featLoc.GetFeatureString(), featLoc.GetContainer());
        kernelDataLoc.SetFeatureIndexCat(mFeatureIndexCat);
        if (KernelMatrixRepository().Exists(kernelDataLoc))
        {
            ILOG_INFO("Using cache from KernelData: " << kernelDataLoc);

            // we have an on-disk cache in the KernelData folder
            // mFeatureExternal is used elsewhere for the quids and the size
            // so this featuretable can perform that role as well
            KernelMatrix* kerData = KernelMatrixRepository().Get(kernelDataLoc);
            mFeatureExternal = kerData->GetFeatureTab();
            // don't delete kerData, awaiting further refactoring

            // wrap the storage of this feature table in a matrix/array
            size_t w = mFeatureExternal->GetColumn2()->GetStorage()->CW();
            size_t h = mFeatureExternal->GetColumn2()->GetStorage()->CH();
            ILOG_DEBUG("size of kernel matrix is ("<< w <<", "<< h <<")");
            mKernelMatrix = new Array::Array2dScalarReal64(w, h, 0, 0, 
                mFeatureExternal->GetColumn2()->GetStorage()->mData, true);
            return mFeatureExternal;
        }

        int sizeCheck = -1;
        Matrix::Mat* accumulator = 0;
        bool isHik = StringStartsWith(mModel, "hik");
        for (int i=0 ; i<mDA->GetNrFeatures() ; i++)
        {
            // load apply to feature
            FeatureLocator fLoc = featLoc;
            fLoc.SetFeatureString(mDA->GetFeature(i));
            FeatureTable* feature = FeatureTableRepository().Get(fLoc);
            if (sizeCheck == -1)
                sizeCheck = feature->Size();
            if (feature->Size() != sizeCheck)
            {
                ILOG_ERROR("Size mismatch in the FeatureData for the dataset to"
                           << " apply to: have " << sizeCheck << " and " <<
                           feature->Size());
                throw "bye";
            }
            ILOG_INFO("size of apply set " << mDA->GetFeature(i) <<
                      " is " << feature->Size());

            // load train feature
            FeatureLocator loc(mAnnoSet->GetLocator(), false, true,
                               mFeatureIndexCat, mDA->GetFeature(i), "");
            FeatureTable* trainFeature =
                Persistency::FeatureTableRepository().Get(loc);
            ILOG_INFO("size of train " << mDA->GetFeature(i) << " is "
                      << trainFeature->Size());

            // compute kernel distances between features
            typedef Feature::FeatureTable::ColumnVectorSet VectorSet;
            VectorSet* vectorsV = feature->GetColumn2();
            VectorSet* vectorsH = trainFeature->GetColumn2();
            vectorsV->SetSize(feature->Size());
            vectorsH->SetSize(trainFeature->Size());
            ILOG_INFO("Compute distance matrix for feature " << i << "...");
            Matrix::Mat* distanceMatrix = 0;
            if (isHik)
            {
                distanceMatrix = Apply(&Core::Vector::HistogramIntersectionDouble,
                                       vectorsH, vectorsV);
            }
            else
            {
                distanceMatrix = Apply(&Core::Vector::Chi2Distance,
                                       vectorsH, vectorsV);
            }
            delete trainFeature;

            // accumulate; this is the part inside the 'exp' in the kernel function
            MulVal(distanceMatrix, distanceMatrix, mDA->GetWeight(i));
            if (!isHik)
                DivVal(distanceMatrix, distanceMatrix, -mDA->GetAverage(i));
            if (accumulator == 0)
                Set(accumulator, distanceMatrix);
            else
                Add(accumulator, accumulator, distanceMatrix);
            delete distanceMatrix;
            if (i == 0)
            {
                mFeatureExternal = feature;
            }
            else
            {
                delete feature;
            }
        }
        // only thing to do is weigh and exp
        DivVal(accumulator, accumulator, mDA->GetTotalWeight());
        if (!isHik)
            Exp(accumulator, accumulator);
        mKernelMatrix = accumulator;
        if (options.GetBool("storeKernelData"))
        {
            FeatureTable* storage =
                new FeatureTable(mKernelMatrixName, sizeCheck,
                                 Matrix::MatNrCol(mKernelMatrix));
            Column::Copy(storage->GetColumn1(), mFeatureExternal->GetColumn1(),
                         sizeCheck);
            // the VectorSet likes owning the storage after setting it,
            // so we must create a wrapper to prevent it from deleting the
            // kernel matrix when the storage is destroyed
            Array::Array2dScalarReal64* wrapper = 
                new Array::Array2dScalarReal64(mKernelMatrix->CW(),
                                               mKernelMatrix->CH(), 0, 0,
                                               mKernelMatrix->mData, true);
            storage->GetColumn2()->SetStorage(wrapper);
            storage->GetColumn2()->SetSize(sizeCheck);
            storage->SetSize(sizeCheck);
            KernelMatrix kerData(storage);
            Persistency::KernelMatrixRepository().Add(kernelDataLoc, &kerData);
        }
        return mFeatureExternal;
    }

    Table::ScoreTable*
    Predict(const ModelLocator& loc)
    {
        Training::Classifier* classifier = Persistency::SvmRepository().Get(loc);
        Util::PropertySet ps;
        ps.Add("kernel","precomputed");
        classifier->OverrideModelOptions(&ps);

        if (!mKernelMatrix)
        {
            ILOG_ERROR("Error: kernel matrix is NULL in Predict");
        }

        Training::TrainDataSrcKernelMatrix dataSrc(mKernelMatrix);
        Table::ScoreTable* scores = classifier->Predict(&dataSrc);
        delete classifier;
        return scores;
    }

    void
    UnloadFeatures()
    {
        if (mFeatureExternal)
        {
            delete mFeatureExternal;
            mFeatureExternal = 0;
        }
        if (mKernelMatrix)
        {
            delete mKernelMatrix;
            mKernelMatrix = 0;
        }
    }

private:

    Matrix::DistributedAccess* mDA;
    Database::RawDataSet*      mAnnoSet;
    FeatureTable*              mFeatureExternal;
    String                     mFeatureIndexCat;
    String                     mModel;
    String                     mKernelMatrixName;
    Matrix::Mat*               mKernelMatrix;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(ApplyConceptsHelperKernels, Impala.Core.Training);

} //namespace
} //namespace
} //namespace

#endif

---