|| || || || || ||

---

mainConceptSet.cpp

Go to the documentation of this file.

#include "Persistency/FikSvmRepository.h"
#include "Persistency/KeywordListRepository.h"
#include "Persistency/DistributedAccessRepository.h"
#include "Persistency/SvmRepository.h"
#include "Persistency/FeatureTableRepository.h"
#include "Core/Database/MakeRawDataSet.h"
#include "Core/Matrix/SetColumn.h"

namespace Impala
{
namespace Application
{
namespace ConceptSet
{


using namespace Impala::Core::Database;
using namespace Impala::Core::Table;
using namespace Impala::Core::Matrix;
using namespace Impala::Core::Feature;
using namespace Impala::Core::Training;
using namespace Impala::Persistency;


void
DoApproxFikModel(RawDataSet* dataSet, CString conceptSet, CString concept,
                 CmdOptions& options, int curArg)
{
    ILOG_VAR(Impala.Application.ConceptSet.DoApproxFikModel);
    if (options.GetNrArg() < curArg + 3)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String model = options.GetArg(curArg++);
    String kernel = options.GetArg(curArg++);
    int nrBins = atol(options.GetArg(curArg++));
    String indexCat = options.GetString("featureIndexCat");

    String fikModel = model + "-approx-" + MakeString(nrBins);
    ModelLocator fikLoc(dataSet->GetLocator(), conceptSet, fikModel, kernel,
                        concept);
    if (FikSvmRepository().Exists(fikLoc))
    {
        ILOG_INFO("Model for " << concept << " already exists, skipping.");
        return;
    }

    KernelMatrixLocator daLoc(dataSet->GetLocator(), true, "", "", model,
                              kernel, "");
    daLoc.SetStartNode(0);
    daLoc.SetNodeCount(Link::Mpi::NrProcs());
    daLoc.SetDoParts(0);
    daLoc.SetFeatureIndexCat(indexCat);
    DistributedAccess* da = DistributedAccessRepository().Get(daLoc);
    ModelLocator mLoc(dataSet->GetLocator(), conceptSet, model, kernel, concept);
    Svm* svm = SvmRepository().Get(mLoc);
    const svm_model* svmModel = svm->GetModel();
    int nrSV = svmModel->l;
    double rho = svmModel->rho[0];
    ILOG_INFO("nr_sv = " << nrSV);
    double* coeffs = svmModel->sv_coef[0];
    QuidTable* kernelQuids = da->GetColumnQuids();
    QuidTable svQuids;
    for (int i=0 ; i<nrSV ; i++)
    {
        svm_node* node = svmModel->SV[i];
        int idx = node->value - 1;
        Quid q = kernelQuids->Get1(idx);
        svQuids.Add(q);
    }

    Mat* mat = 0;
    int nrFeatures = da->GetNrFeatures();
    WeightedFeatureList features;
    for (int i=0 ; i<nrFeatures ; i++)
        features.Add(da->GetFeature(i), da->GetWeight(i));
    Real64 probA = svmModel->probA[0];
    Real64 probB = svmModel->probB[0];
    int probIndex = (svmModel->label[1] == 1) ? 1 : 0;
    FikSvm fikSvm(features, nrBins, nrSV, rho, probA, probB, probIndex);
    for (int i=0 ; i<nrFeatures ; i++)
    {
        ILOG_INFO("feature = " << da->GetFeature(i));
        FeatureLocator fLoc(dataSet->GetLocator(), false, true, indexCat,
                            da->GetFeature(i), "");
        FeatureTable* tab = FeatureTableRepository().Get(fLoc);
        int featDim = tab->GetFeatureVectorLength();
        if ((mat == 0) || (mat->CH() != featDim))
        {
            ILOG_INFO("featDim = " << featDim);
            if (mat)
                delete mat;
            mat = MatCreate<Mat>(featDim, nrSV);
        }
        for (int sv=0 ; sv<nrSV ; sv++)
        {
            Quid q = svQuids.Get1(sv);
            SetColumn(mat, sv, tab->FindFeature(q));
        }
        fikSvm.CreateSvm(featDim, coeffs, mat->CPB());
        //fikSvm.Dump(i, 3);
        delete tab;
    }
    ILOG_INFO("Saving in " << fikLoc);
    FikSvmRepository().Add(fikLoc, &fikSvm);
    delete mat;
    delete svm;
    delete da;
}

int
mainConceptSet(int argc, char* argv[])
{
    Link::Mpi::Init(&argc, &argv);

    CmdOptions& options = CmdOptions::GetInstance();
    options.Initialise(false, false, true);
    options.AddOption(0, "start", "index of concept to start with", "0");
    options.AddOption(0, "number", "number of concepts", "-1");
    options.AddOption(0, "concept", "name", "");
    options.AddOption(0, "featureIndexCat", "name", "");

    if (! options.ParseArgs(argc, argv, "cmd dataSet conceptSet [cmdArgs]", 3))
    {
        Link::Mpi::Finalize();
        return 1;
    }

    ILOG_VAR(Impala.Application.ConceptSet.mainConceptSet);

    int curArg = 0;
    String cmd = options.GetArg(curArg++);
    String dataSetName = options.GetArg(curArg++);
    String conceptSet = options.GetArg(curArg++);

    RawDataSet* dataSet = MakeRawDataSet(dataSetName);
    KeywordListLocator loc(dataSet->GetLocator(), conceptSet);
    KeywordList concepts = *(KeywordListRepository().Get(loc));


    int start = options.GetInt("start");
    int number = options.GetInt("number");
    String concept = options.GetString("concept");
    if (!concept.empty())
    {
        for (int i=0 ; i<concepts.size() ; i++)
        {
            if (concepts[i] == concept)
            {
                start = i;
                number = 1;
            }
        }
    }
    if (number == -1)
        number = concepts.size();
    if (start + number >= concepts.size())
        number = concepts.size() - start;
    int last = start + number;
    int numProcs = Link::Mpi::NrProcs();
    int myId = Link::Mpi::MyId();
    int taskSize = number / numProcs;
    int restSize = number % numProcs;
    int cur = start;
    for (int i=0 ; i<=myId ; i++)
    {
        start = cur;
        number = taskSize;
        if (--restSize >= 0)
            number++;
        if (start + number > last)
            number = last - start;
        cur += number;
    }
    ILOG_INFO("Process " << myId << " of " << numProcs << " is doing concepts "
              << start << " till " << start + number);

    ILOG_INFO_HEADNODE("conceptset command : " << cmd);
    for (int i=start ; i<start+number ; i++)
    {
        ILOG_INFO("doing concept " << concepts[i]);
        ILOG_NDC_PUSH(concepts[i]);
        if (cmd == "approxfikmodel")
            DoApproxFikModel(dataSet, conceptSet, concepts[i], options, curArg);
        else
        {
            ILOG_ERROR("Unknown command : " << cmd);
            ILOG_NDC_POP;
            break;
        }
        ILOG_NDC_POP;
    }

    int nrOfErrors = ILOG_ERROR_COUNT;
    ILOG_DEBUG("nr of errors collected = " << nrOfErrors);

    nrOfErrors = Link::Mpi::ReduceSum(nrOfErrors);
    ILOG_INFO_HEADNODE("Root: total nr error = " << nrOfErrors);
    Link::Mpi::Finalize();

    return nrOfErrors;
}

} // namespace ConceptSet
} // namespace Application
} // namespace Impala

int
main(int argc, char* argv[])
{

#ifdef NO_DIALOGONCRASH
#ifdef WIN32
    DWORD oldMode = ::SetErrorMode(SEM_NOGPFAULTERRORBOX);
    ::SetErrorMode(oldMode | SEM_NOGPFAULTERRORBOX);
#endif
#endif

    return Impala::Application::ConceptSet::mainConceptSet(argc, argv);
}

---