|| || || || || ||

---

PrecomputeTask.h

Go to the documentation of this file.

#ifndef Impala_Core_Training_PrecomputeTask_h
#define Impala_Core_Training_PrecomputeTask_h

#include "Persistency/FeatureTableRepository.h"
#include "Persistency/KernelMatrixRepository.h"
#include "Link/Cuda/Cuda.h"
#include "Core/Feature/WeightedFeatureList.h"
#include "Core/Training/ComputeKernelMatrix.h"

namespace Impala
{
namespace Core
{
namespace Training
{


class PrecomputeTask
{
public:

    typedef Matrix::DistributedAccess DistributedAccess;
    typedef Matrix::VirtualMatrix VirtualMatrix;
    typedef Feature::VirtualFeatureTable VirtualFeatureTable;
    typedef Feature::VirtualFeatureTableFactory VFTFactory;
    typedef Persistency::File File;
    typedef Persistency::FeatureLocator FeatureLocator;
    typedef Persistency::DistributedAccessRepository DistributedAccessRepository;
    typedef Persistency::FeatureTableRepository FeatureTableRepository;
    typedef Persistency::KernelMatrixLocator KernelMatrixLocator;
    typedef Persistency::KernelMatrixRepository KernelMatrixRepository;

    static void
    ComputeDevelKernel(Database::RawDataSet* dataSet,
                       const KernelMatrixLocator& kerLoc,
                       CString inputFeatures)
    {
        bool override = CmdOptions::GetInstance().GetBool("override");
        if (DistributedAccessRepository().Exists(kerLoc) && !override)
        {
            ILOG_INFO("Skipping, Kernel already exists: " << kerLoc);
            return;
        }

        Feature::WeightedFeatureList fList(inputFeatures);
        if (fList.Size() == 0)
        {
            ILOG_ERROR("Empty feature list");
            return;
        }

        FeatureLocator featLoc(dataSet->GetLocator(), false, true,
                               kerLoc.GetFeatureIndexCat(),
                               fList.GetFeature(0), "");
        VirtualFeatureTable* features =
            VFTFactory::GetInstance().ConstructIOBufferReader(featLoc, true);
        int mSize = features->Size();
        DistributedAccess* kernel = new DistributedAccess(mSize, mSize, 1, 1,
                                                          0, 1);
        features->GetQuids(kernel->GetRowQuids());
        features->GetQuids(kernel->GetColumnQuids());
        delete features;

        bool isChi2 = (kerLoc.GetModel() == "chi2");
        for (int i=0 ; i<fList.Size() ; i++)
        {
            Real64 average = -1;
            if (isChi2)
            {
                FeatureLocator loc = featLoc;
                loc.SetSuffix(".average.raw");
                loc.SetFeatureString(fList.GetFeature(i));
                File file = FeatureTableRepository().ExposeFile(loc, false);
                if (file.Valid())
                {
                    std::vector<double> v;
                    file.ReadNative(std::back_inserter(v), true);
                    average = v[0];
                    kernel->SetHasOwnAverages(true);
                }
                else
                {
                    ILOG_ERROR("Could not load average for " << loc);
                    delete kernel;
                    return;
                }
            }
            kernel->AddFeature(fList.GetFeature(i), fList.GetWeight(i), average);
        }

        PrecomputeTask* pt = new PrecomputeTask(false, kerLoc, kernel, featLoc,
                                                featLoc);
        Execute(pt);
    }

    static void
    ComputeTestKernel(const Persistency::Locator& dstLoc, CString walkType,
                      CString container, Database::RawDataSet* develSet,
                      DistributedAccess* develKernel, CString featureIndexCat,
                      CString model, CString kernelMatrixName)
    {
        ILOG_VAR(Impala.Core.Training.PrecomputeTask.ComputeTestKernel);
        KernelMatrixLocator thisKerLoc(dstLoc, false, walkType,
                                       develSet->GetSetName(), model,
                                       kernelMatrixName, container);
        thisKerLoc.SetFeatureIndexCat(featureIndexCat);

        bool override = CmdOptions::GetInstance().GetBool("override");
        if (KernelMatrixRepository().Exists(thisKerLoc) && !override)
        {
            ILOG_INFO("Skipping, KernelData already exists: " << thisKerLoc);
            return;
        }

        if (develKernel->GetNrFeatures() == 0)
        {
            ILOG_ERROR("No features");
            return;
        }

        FeatureLocator thisFeatLoc(dstLoc, false, false, walkType,
                                   "empty", container);
        FeatureLocator develFeatLoc(develSet->GetLocator(), false, true, "",
                                    "empty", "");

        PrecomputeTask* pt = new Training::PrecomputeTask
            (true, thisKerLoc, develKernel, thisFeatLoc, develFeatLoc);
        Execute(pt);
    }

    static void
    Execute(PrecomputeTask* pt)
    {
//#ifdef KOENFIX
//#define CPUFLOAT double
//#else
        typedef float CPUFLOAT;
//#endif

        bool GPU = Link::Cuda::CudaUsed();
        ILOG_INFO("GPU mode: " << GPU);
        int slabWidth = 1024;
        using Training::ComputeKernelMatrix;
        if (GPU)
        {
            ComputeKernelMatrix<float>(pt, slabWidth, GPU);
        }
        else
        {
#ifdef CUDA
            ComputeKernelMatrix<CPUFLOAT>(pt, slabWidth, GPU);
#else
            ComputeKernelMatrix<double>(pt, slabWidth, GPU);
#endif
        }
        delete pt;
    }

    // Takes ownership of develKernel only when not computing a test kernel
    PrecomputeTask(bool computingTestKernel,
                   const KernelMatrixLocator& thisKerLoc,
                   DistributedAccess* develKernel,
                   const FeatureLocator& thisFeatLoc,
                   const FeatureLocator& develFeatLoc)
    {
        mComputingTestKernel = computingTestKernel;
        mDevelKernel = develKernel;
        mThisKerLoc = thisKerLoc;
        mThisFeatLoc = thisFeatLoc;
        mDevelFeatLoc = develFeatLoc;
        mMatrix = 0;
        mWriter = 0;
        for (int i=0 ; i<NrFeatures() ; i++)
        {
            ILOG_INFO(i << " " << GetFeatureAverage(i) << " (" <<
                      GetFeatureWeight(i) << "*" << GetFeatureName(i) << ")");
        }
    }

    virtual
    ~PrecomputeTask()
    {
        if (!mComputingTestKernel)
            delete mDevelKernel;
        if (mWriter)
            delete mWriter;
    }

    size_t
    NrFeatures() const
    {
        return mDevelKernel->GetNrFeatures();
    }

    String
    GetFeatureName(int feature) const
    {
        return mDevelKernel->GetFeature(feature);
    }

    Real64
    GetFeatureWeight(int feature) const
    {
        return mDevelKernel->GetWeight(feature);
    }

    Real64
    GetTotalFeatureWeight() const
    {
        return mDevelKernel->GetTotalWeight();
    }

    Real64
    GetFeatureAverage(int feature) const
    {
        return mDevelKernel->GetAverage(feature);
    }

    bool
    IsSymmetric() const
    {
        return (!mComputingTestKernel);
    }

    bool
    IsChi2() const
    {
        return (mThisKerLoc.GetModel() == "chi2");
    }

    FeatureLocator
    GetFeatureLocatorA(int index)
    {
        FeatureLocator loc = mDevelFeatLoc;
        loc.SetFeatureString(mDevelKernel->GetFeature(index));
        return loc;
    }

    FeatureLocator
    GetFeatureLocatorB(int index)
    {
        FeatureLocator loc = mThisFeatLoc;
        loc.SetFeatureString(mDevelKernel->GetFeature(index));
        return loc;
    }

    VirtualMatrix*
    GetWritableMatrix(int nrRow, int nrCol)
    {
        typedef Core::Matrix::VirtualMatrixFactory VMFactory;
        using Persistency::RepositoryGetFile;

        String path = KernelMatrixRepository().ExposeFilePath(mThisKerLoc, true);
        if (path.empty())
        {
            ILOG_ERROR("Couldn't get writable file for kernel " << mThisKerLoc);
            return 0;
        }
        String suffix = (mComputingTestKernel) ? ".tab"
                                               : ".precomputed.part-R0-C0.raw";
        String fName = FileNamePath(path) + FileNameBase(path) + suffix;
        Persistency::FileLocator fileLoc(mThisKerLoc, fName);
        File file = RepositoryGetFile(fileLoc, true, false);
        if (mComputingTestKernel)
        {
            mWriter = VFTFactory::GetInstance().ConstructIOBufferWriter
                                             (nrRow, nrCol, file, false);
            return mWriter->GetMatrix();
        }
        else
        {
            mMatrix = VMFactory::GetInstance().ConstructIOBufferWriter
                                             (nrRow, nrCol, file, true, true);
            return mMatrix;
        }
    }

    void
    Finalize()
    {
        if (mComputingTestKernel)
        {
            FeatureLocator loc = GetFeatureLocatorB(0);
            VirtualFeatureTable* features =
                VFTFactory::GetInstance().ConstructIOBufferReader(loc, false);
            Table::QuidTable quids(features->Size());
            features->GetQuids(&quids);
            delete features;
            mWriter->AddQuids(&quids);
            mWriter->SetCompleted();
        }
        else
        {
            delete mMatrix;
            KernelMatrixLocator loc = mThisKerLoc;
            loc.SetDoParts(0);
            Persistency::DistributedAccessRepository().Add(loc, mDevelKernel);
        }
    }

private:

    bool                mComputingTestKernel;
    KernelMatrixLocator mThisKerLoc;
    DistributedAccess*  mDevelKernel;
    FeatureLocator      mThisFeatLoc;
    FeatureLocator      mDevelFeatLoc;
    VirtualMatrix*      mMatrix; // used in computation of devel kernel
    Core::Feature::VirtualFeatureTableIOBufferWriter* mWriter;// for test kernel

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(PrecomputeTask, Impala.Core.Training);

} // namespace Training
} // namespace Core
} // namespace Impala

#endif

---