|| || || || || ||

---

mainActiveLearner.cpp

Go to the documentation of this file.

#include "Basis/CmdOptions.h"
#include "Core/Table/Select.h"
#include "Core/Training/TrainDataSrcKernelDistributed.h"
#include "Core/Training/LoadDistributedAccess.h"
#include "Core/Database/MakeRawDataSet.h"
#include "Util/DatabaseReadString.h"
#include "Persistency/SvmRepository.h"
#include "Persistency/AnnotationTableRepository.h"
#include "Persistency/SimilarityTableSetRepository.h"

// since we are not using libraries:
#include "Link/Svm/LinkSvm.cpp"


namespace Impala
{
namespace Application
{


using namespace Core;
using namespace Core::Training;
using namespace Persistency;

Table::SimilarityTableSet* 
LearnConceptFromAnnotations(CmdOptions& options, Matrix::DistributedAccess* da,
                            String concept,
                            const ModelLocator& modelLoc,
                            Table::AnnotationTable *annotations)
{
    ILOG_VAR(Application.LearnConceptFromAnnotations);


    ILOG_INFO("concept " << concept);

    //load annotation
    Table::QuidTable* allQuids = da->GetColumnQuids();
    Table::CriterionElement1InSet<Table::AnnotationTable> c(allQuids);
    Table::AnnotationTable* annotation = Select(annotations, c);

    Table::QuidTable* positive = annotation->GetPositive();
    Table::QuidTable* negative = annotation->GetNegative();
    Table::QuidTable* learnQuids = positive;
    learnQuids->Append(negative);
    delete negative;

    if ((positive->Size() == 0) || (negative->Size() == 0))
    {
        delete learnQuids;
        delete annotation;
        return 0;
    }

    Util::PropertySet params;
    params.Add("probability", 1);
    params.Add("cache", options.GetInt("cache"));

    params.Add("gamma", -1);
    params.Add("autoweight", 1);
    params.Add("kernel", "dist-precomputed");
    params.Add("precompute-kernel", "chi2");
    params.Add("evaluator", "AP");
    params.Add("C", 1.0);
    params.Add("w1", 30.0); // estimate these
    params.Add("w2", 1.2); // estimate these
    for(int i = 0; i < params.Size(); i++)
        ILOG_INFO(params.GetName(i) + " " + params.GetValue(i));

    //compute model:
    Training::Svm svm;
    Training::TrainDataSrcKernelDistributed* dataSrc =
        new TrainDataSrcKernelDistributed(da, annotation);
    svm.Train(&params, dataSrc);
    ILOG_INFO("saving model");
    SvmRepository().Add(modelLoc, &svm);

    // compute resulting ranking:
    std::vector<String> names;
    names.push_back(concept);
    Table::QuidTable* rowQuids = da->GetRowQuids();
    Table::SimilarityTableSet *simSet =
        new Table::SimilarityTableSet(names, rowQuids->Size());
    Table::Copy(simSet->GetQuidTable(), rowQuids);

    ILOG_INFO("applying model");
    svm.PredictForActiveLearn(da, learnQuids, simSet->GetSimTable(0));
    
    ILOG_INFO("ranking");
    simSet->ComputeRank(0, true);
    
    delete annotation;
    delete learnQuids;
    delete dataSrc;
    return simSet;
}

int
RunDistributedLearningEngine(CmdOptions& options)
{
    ILOG_VAR(Application.RunDistributedLearningEngine);
    String setName = options.GetArg(0);
    String conceptsName = options.GetArg(1); //"conceptsActiveLearn.txt";
    String model = options.GetArg(2);
    String kernel = options.GetArg(3);

    Database::RawDataSet* dataSet = Database::MakeRawDataSet(setName);
    int quidClass = dataSet->GetQuidClass();

    int startnode = 1;
    int nodes = Link::Mpi::NrProcs() - startnode;
    Matrix::DistributedAccess* da =
        LoadDistributedAccess(model, kernel, dataSet, 0, startnode, nodes);
    if ((Link::Mpi::MyId() <= nodes + startnode) && (Link::Mpi::MyId() != 0))
    {
        da->StartEventLoop();
    }
    else
    {
        da->Subscribe();
        ILOG_INFO_NODE("engine loaded, waiting for annotations.");

        while (true)
        {
            // announce waiting for table
            // receive table
            String identifier;
            while (true)
            {
                String fName = "Annotations/" + QuidClassToString(quidClass)
                    + "/" + conceptsName + "/startlearner.txt";
                FileLocator fLoc(dataSet->GetLocator(), fName);
                Persistency::File file = RepositoryGetFile(fLoc, false, true);
                if (file.Valid())
                {
                    std::vector<String> ids;
                    file.ReadStrings(ids);
                    if (ids.size() > 0)
                    {
                        identifier = ids[0];
                        ILOG_INFO("Starting model train cycle for "<<identifier);
                        ids.clear();
                        file = RepositoryGetFile(fLoc, true, false);
                        file.WriteStrings(ids.begin(), ids.end());
                        break;
                    }
                }
            }

            AnnotationTableLocator annoLoc(dataSet->GetLocator(), quidClass,
                                           conceptsName, identifier);
            if (!AnnotationTableRepository().Exists(annoLoc))
                continue;
            Table::AnnotationTable* annotationtable =
                AnnotationTableRepository().Get(annoLoc);

            ModelLocator modelLoc(dataSet->GetLocator(), conceptsName,
                                  "activelearn", "activelearn", identifier);
            Timer timer;
            Table::SimilarityTableSet* simSet = 
                LearnConceptFromAnnotations(options, da, identifier, modelLoc,
                                            annotationtable);
            ILOG_INFO("Learned in " << timer.SplitTimeStr());

            if (!simSet)
            {
                ILOG_ERROR("Learning aborted, no positives or negatives?");
                continue;
            }

            // save concept
            SimilarityTableSetLocator simLoc(dataSet->GetLocator(), true, "",
                                             conceptsName, "activelearn",
                                             "activelearn", "");
            SimilarityTableSetRepository().Add(simLoc, simSet);

            delete annotationtable;
            delete simSet;

            ILOG_INFO("Loop done: " << timer.SplitTimeStr());
        }
        da->Unsubscribe();
    }
    delete da;
    Link::Mpi::Finalize();
    return 0;
}


int
mainActiveLearner(int argc, char** argv)
{
    Link::Mpi::Init(&argc, &argv);
    CmdOptions& options = CmdOptions::GetInstance();
    options.Initialise(false, false, true);
    options.AddOption(0, "assume-shotid", "bool", "0");
    options.AddOption('m', "cache", "megabytes", "10");
    //options.AddOption(0, "kernel", "string: [linear,poly,rbf,sigmoid,precomputed,hist,dist-precomputed]", "rbf");
    options.AddOption(0, "maxVideoId", "index", "-1");
    options.AddOption(0, "maxPosPerVideo", "number", "-1");
    options.AddOption(0, "maxNegPerVideo", "number", "-1");

    if (! options.ParseArgs(argc, argv, "<video set> <concept definitions> <model type> <features>", 4))
    {
        Link::Mpi::Finalize();
        return 1;
    }

    return RunDistributedLearningEngine(options);
}

} // namespace Application
} // namespace Impala

int
main(int argc, char* argv[])
{
    return Impala::Application::mainActiveLearner(argc, argv);
}

---