|| || || || || ||

---

mainTable.cpp

Go to the documentation of this file.

#include "Util/Channel.h"
#include "Core/Database/MakeRawDataSet.h"
#include "Core/VideoSet/TaskTable.h"
#include "Core/Table/AnnotationTableSet.h"
#include "Core/Vector/Norm1Dist.h"
#include "Core/Vector/Similarity.h"
#include "Core/Vector/VectorTem.h"
#include "Core/Vector/SubAssign.h"
#include "Core/Vector/DivAssign.h"
#include "Core/Vector/Max.h"
#include "Core/Vector/Min.h"
#include "Core/Column/MakeFilter.h"
#include "Core/Column/CopyUnique.h"
#include "Core/Table/Select.h"
#include "Core/Table/TableVxs.h"
#include "Core/Table/Write.h"
#include "Core/Table/Copy.h"
#include "Core/Feature/FeatureTableSet.h"
#include "Core/Feature/VisSem.h"
#include "Core/Table/SimilarityTableSet.h"
#include "Basis/CmdOptions.h"
#include "Basis/FileExists.h"
#include "Basis/FileName.h"
#include "Core/VideoSet/MakeVideoSet.h"
#include "Core/VideoSet/Keyframes.h"
#include "Core/ImageSet/MakeImageSet.h"
#include "Core/Array/Percentile.h"
#include "Core/Array/Dilation.h"
#include "Core/Array/Erosion.h"
#include "Core/Array/Equals.h"
#include "Core/Array/Mul.h"
#include "Core/Matrix/MatSet.h"


// since we are not using libraries:
#include "Link/ImpalaLib.cpp"

namespace Impala
{
namespace Application
{
namespace Table
{

template <class ColElemT>
void
DoDump(String fName)
{
    typedef Core::Column::ColumnTem<ColElemT> ColumnType;
    typedef Core::Table::TableTem<ColumnType> TableType;

    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    TableType table(0);
    Core::Table::Read(&table, fName, db);
    table.Dump(options.GetInt("start"), options.GetInt("end"));
}

template <class ColElemT>
void
DoDiff(String fName1, String fName2)
{
    typedef Core::Column::ColumnTem<ColElemT> ColumnType;
    typedef Core::Table::TableTem<ColumnType> TableType;

    ILOG_VAR(Impala.Application.Table.DoDiff);
    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    TableType table1(0);
    Core::Table::Read(&table1, fName1, db);
    TableType table2(0);
    Core::Table::Read(&table2, fName2, db);
    if (table1.Size() != table2.Size())
    {
        ILOG_INFO("Different table sizes: " << table1.Size() << " vs "
                  << table2.Size());
        return;
    }
    int verb = options.GetInt("verb");
    int nDiff = 0;
    for (int i=0 ; i<table1.Size() ; i++)
    {
        //if (table1.Get1(i) != table2.Get1(i))
        if ((table1.Get1(i) - table2.Get1(i)) > 0.00001)
        {
            nDiff++;
            if (verb)
                ILOG_INFO("diff elem " << i << ": " << table1.Get1(i)
                          << " vs " << table2.Get1(i));
        }
    }
    if (nDiff > 0)
        ILOG_INFO("Found " << nDiff << " differences");
}

void
DoVxsTable()
{
    ILOG_VAR(Impala.Application.Table.DoVxsTable);
    CmdOptions& options = CmdOptions::GetInstance();
    String fName = options.GetArg(1);
    Core::Table::TableVxs table;
    table.ReadVxsFile(fName);
    //table.Dump();

    Core::Column::ColumnTem<String> concept(table.Size());
    int nr = Core::Column::CopyUnique(&concept, table.GetColumn3(), 0, 0,
                                      table.Size());
    concept.Reserve(nr, true);

    for (int i=0 ; i<concept.Capacity() ; i++)
    {
        ILOG_INFO("concept: " << concept.Get(i));
        bool* filter = 0;
        Core::Column::MakeFilterEqual(filter, table.GetColumn3(), false,
                                      concept.Get(i));
        Core::Table::TableVxs t;
        Core::Table::Select(&t, &table, filter, true);
        //t.Dump();
        t.WriteVxsFile(concept.Get(i) + ".vxs");
        delete filter;
        ILOG_INFO("  size: " << t.Size());
    }
}

void
DoScaleFeatureTable()
{
    typedef Core::Feature::FeatureTable FeatureTable;
    typedef Core::Vector::VectorTem<Real64> VectorReal64;

    CmdOptions& options = CmdOptions::GetInstance();
    String tablename = options.GetArg(1);
    String scaledtablename = options.GetArg(2);
    String minmaxtablename = options.GetArg(3);

    Core::Database::RawDataSet* dataSet = 0;
    String setName = options.GetString("videoSet");
    if (! setName.empty())
        dataSet = Core::VideoSet::MakeVideoSet(setName);
    setName = options.GetString("imageSet");
    if (! setName.empty())
        dataSet = Core::ImageSet::MakeImageSet(setName);
    Util::Database* db = &Util::Database::GetInstance();
    if (dataSet)
        db = dataSet->GetDatabase();

    FeatureTable tableIn(tablename);
    Core::Table::Read(&tableIn, tablename, db);

//    FeatureTable tableMinMax(tableIn.GetFeatureDefinition());
    FeatureTable tableMinMax(tableIn.GetFeatureDefinition(), 2, tableIn.GetFeatureVectorLength());
    FeatureTable tableOut(tableIn.GetFeatureDefinition(), tableIn.Size(), tableIn.GetFeatureVectorLength());

    VectorReal64 vMax;
    VectorReal64 vMin;

    if(FileExists(minmaxtablename))
    {
        // read minmax from file
        std::cout << "read minmax from file: " << minmaxtablename << std::endl;
        Core::Table::Read(&tableMinMax, minmaxtablename, db);
        vMax = tableMinMax.Get2(0);
        vMin = tableMinMax.Get2(1);
    }
    else
    {
        // create minmax
        Core::Vector::Max(&vMax, tableIn.GetColumn2(), tableIn.Size(), 0);
    tableMinMax.Add(0, vMax);

        Core::Vector::Min(&vMin, tableIn.GetColumn2(), tableIn.Size(), 0);
    tableMinMax.Add(1, vMin);

        std::cout << "write minmax to file: " << minmaxtablename << std::endl;
    Core::Table::Write(&tableMinMax, minmaxtablename, db, true);
    }

    std::cout << "max: " << vMax << std::endl;
    std::cout << "min: " << vMin << std::endl;
    //FeatureTable tableOut(tableIn);

    VectorReal64 vMaxMinMin = vMax;
    Core::Vector::SubAssign(vMaxMinMin,vMin);
    //std::cout << "vMaxMinMin: " << vMaxMinMin << std::endl;

    VectorReal64 v;
    for(int i=0;i<tableIn.Size();i++)
    {
    v = tableIn.Get2(i);
    Core::Vector::SubAssign(v, vMin);
    Core::Vector::DivAssign(v, vMaxMinMin);
        tableOut.Add(tableIn.Get1(i), v);
    }

    // create dir if neccecary
    std::string path = FileNamePath(scaledtablename);
    db->MakeDir(path);
    //table.Dump(dataSet, options.GetInt("start"), options.GetInt("end"));
    Core::Table::Write(&tableOut, scaledtablename, db, true);
}


void
DoAppendFeatureTable()
{
    typedef Core::Feature::FeatureDefinition FeatureDef;
    typedef Core::Feature::FeatureTable FeatureTable;

    ILOG_VAR(Impala.Application.Table.DoAppendFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();
    String dstName = options.GetArg(1);
    String srcName = options.GetArg(2);
    int nr = atol(options.GetArg(3));

    Util::Database* db = &Util::Database::GetInstance();
    FeatureTable srcTable(srcName);
    Core::Table::Read(&srcTable, srcName, db);

    FeatureTable dstTable(dstName, nr*srcTable.Size(),
                          srcTable.GetFeatureVectorLength());
    for (int i=0 ; i<nr ; i++)
        dstTable.Append(&srcTable);

    Core::Table::Write(&dstTable, dstName, db, true);
}

void
DoConcatFeatureTable()
{
    typedef Core::Feature::FeatureDefinition FeatureDef;
    typedef Core::Feature::FeatureTable FeatureTable;
    typedef Core::Feature::FeatureTableSet FeatureTableSet;

    ILOG_VAR(Impala.Application.Table.DoConcatFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();
    String dir = options.GetArg(1);
    bool useName = StringToBool(options.GetArg(2));
    FeatureDef resultDef(options.GetArg(3));
    String srcDefs = options.GetArg(4);
    for (int i=5 ; i<options.GetNrArg() ; i++)
        srcDefs += ";" + options.GetArg(i);

    Util::Database* db = &Util::Database::GetInstance();
    FeatureTableSet* fSet = FeatureTableSet::MakeFromFile(dir, useName, srcDefs);
    FeatureTable* res = fSet->ConcatTables(resultDef);
    String fName = dir + "/";
    if (useName)
        fName += resultDef.GetName() + "/";
    fName += resultDef.AsString();
    Core::Table::Write(res, fName, db, true);
    fSet->Delete();
    delete fSet;
    delete res;
}

void
DoFeatureTable2Svm()
{
    typedef Core::Feature::FeatureTable FeatureTable;
    typedef Core::Table::AnnotationTable AnnotationTable;

    ILOG_VAR(Impala.Application.Table.DoFeatureTable2Svm);
    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    if (options.GetNrArg() < 3)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String fName = options.GetArg(1);
    String svmName = options.GetArg(2);
    AnnotationTable* truth = 0;
    if (options.GetNrArg() > 3)
    {
        truth = AnnotationTable::MakeFromFile("", options.GetArg(3), db);
    }
    FeatureTable table(fName);
    Core::Table::Read(&table, fName, db);
    table.WriteSvmFile(svmName, db, truth);
}

void
DoFeatureTableReadAscii()
{
    typedef Core::Feature::FeatureTable FeatureTable;

    ILOG_VAR(Impala.Application.Table.DoFeatureTableReadAscii);
    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    if (options.GetNrArg() < 3)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String tabName = options.GetArg(1);
    String asciiName = options.GetArg(2);
    FeatureTable table(tabName);
    table.ReadPlainAsciiFile(asciiName, db);
    Core::Table::Write(&table, tabName, db, true);
}

void
DoFeatureTableWriteAscii()
{
    typedef Core::Feature::FeatureTable FeatureTable;

    ILOG_VAR(Impala.Application.Table.DoFeatureTableWriteAscii);
    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    if (options.GetNrArg() < 3)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String tabName = options.GetArg(1);
    String asciiName = options.GetArg(2);
    FeatureTable table(tabName);
    Core::Table::Read(&table, tabName, db);
    table.WritePlainAsciiFile(asciiName, db);
}

void
DoSimFeatureTable()
{
    typedef Core::Feature::FeatureTable FeatureTable;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Application.Table.DoSimFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String outDir = options.GetArg(1);
    String fName1 = options.GetArg(2);
    FeatureTable table1(fName1);
    ILOG_INFO("Reading " << fName1);
    Core::Table::Read(&table1, fName1, db);
    String fName2 = options.GetArg(3);
    FeatureTable table2(fName2);
    ILOG_INFO("Reading " << fName2);
    Core::Table::Read(&table2, fName2, db);

    std::vector<String> names;
    for (int i=0 ; i<table1.Size() ; i++)
        names.push_back(QuidObj(table1.Get1(i)).ToString());
    ILOG_INFO("nrNames = " << names.size());
    SimilarityTableSet simSet(names, table2.Size());
    String outPath = outDir + "/";
    simSet.SaveNames(outPath, db);
    Core::Table::Copy(simSet.GetQuidTable(), &table2); // copy Quids only
    simSet.SaveQuids(outPath, db, true);
    for (int i=0 ; i<names.size() ; i++)
    {
        ILOG_INFO("i = " << i);
        SimTableType* simTable = simSet.GetSimTable(i);
        Real64* sim = simTable->GetColumn1()->GetData();
        Core::Vector::Similarity(sim, table1.Get2(i),
                                 table2.GetColumn2(), 0, 1,
                                 Core::Vector::Norm1Dist<Real64>);
        simTable->SetSize(table2.Size());
        ILOG_INFO("  compute rank");
        simSet.ComputeRank(i, false);
        simSet.Save(i, outPath, db, true);
    }
}

void
DoSelectFeatureTableSet()
{
    typedef Core::Feature::FeatureTableSet FeatureTableSet;
    typedef Core::Feature::FeatureTable FeatureTable;

    ILOG_VAR(Impala.Application.Table.DoSelectFeatureTableSet);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String setName = options.GetArg(1);
    String resName = options.GetArg(2);
    int optimalSize = atol(options.GetArg(3));
    String feature = options.GetArg(4);
    Core::Database::RawDataSet* ds = Core::Database::MakeRawDataSet(setName);

    Core::Feature::Computor* c = 0;
    if (feature == "weibull")
        c = new Core::Feature::VisSem("vissem", options);
    else if (feature == "gabor")
        c = new Core::Feature::VisSem("vissemgabor", options);
    else
        ILOG_ERROR("unknown feature : " << feature);
    c->ReadPrototypes(setName, "", -1);

    std::vector<String> res;
    std::vector<int> resVal;
    for (int p=0 ; p<c->GetNrPixelFeatureSets() ; p++)
    {
        for (int r=0 ; r<c->GetNrRegionFeatureSets() ; r++)
        {
            std::cout << "p=" << p << " r=" << r << std::endl;
            std::vector<FeatureTable*> tabs = c->GetProtoFeatureTables(p, r);
            int bestIdx = -1;
            int bestVal = 0;
            for (int t=0 ; t<tabs.size() ; t++)
            {
                FeatureTable* tab = tabs[t];
                std::cout << "  " << tab->GetFeatureDefinition().AsString()
                          << " = " << tab->Size() << std::endl;
                int s = tab->Size();
                if (bestIdx == -1)
                {
                    bestIdx = t;
                    bestVal = s;
                    continue;
                }
                if (s >= optimalSize)
                {
                    if ((bestVal < optimalSize) || (s < bestVal))
                    {
                        bestIdx = t;
                        bestVal = s;
                    }
                }
                else
                {
                    if ((bestVal < optimalSize) && (s > bestVal))
                    {
                        bestIdx = t;
                        bestVal = s;
                    }
                }
            }
            std::cout << "    bestVal = " << bestVal << std::endl;
            String def = tabs[bestIdx]->GetFeatureDefinition().AsString();
            res.push_back(def);
            resVal.push_back(bestVal);
        }
    }

    std::cout << std::endl << "Selection: " << std::endl;
    for (int i=0 ; i<res.size() ; i++)
    {
        std::cout << resVal[i] << " from " << res[i] << std::endl;
    }
    String fName = ds->GetFilePathPrototype(c->GetProtoDataFile(), c->GetName(),
                                            resName, true, false);
    if (!fName.empty())
    {
        std::cout << "Writing to " << fName << std::endl;
        Util::DatabaseWriteString(fName, ds->GetDatabase(), res.begin(),
                                  res.end());
    }
}

void
DoDumpSimilarityTableSetTrec()
{
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;

    ILOG_VAR(Impala.Application.Table.DoDumpSimilarityTableSetTrec);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 3)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String setName = options.GetArg(1);
    Core::VideoSet::VideoSet* vidSet = Core::VideoSet::MakeVideoSet(setName);
    Core::VideoSet::Keyframes keyframes(vidSet, "keyframes");
    String path = options.GetArg(2);
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(path,
                                                      vidSet->GetDatabase());

    Core::Table::QuidTable* qTable = simSet->GetQuidTable();
    int start = options.GetInt("start");
    int end = options.GetInt("end");
    if ((end == -1) || (end > qTable->Size()))
        end = qTable->Size();

    std::cout << "SimilarityTableSet " << simSet->GetDescription() << std::endl;
    for (int t=0 ; t<simSet->NrTables() ; t++)
    {
        SimilarityTableSet::SimTableType* simTable = simSet->GetSimTable(t);
        std::cout << "table " << t << ", name = " << simSet->GetName(t)
                  << std::endl;
        if (simTable->Size() != qTable->Size())
        {
            ILOG_ERROR("DumpRanking: simtable size doesn't match");
            continue;
        }
        String concept = simSet->GetName(t);
        for (int i=start ; i<end ; i++)
        {
            Quid quid = qTable->Get1(i);
            //std::cout << i << ", ";
            std::cout << concept << ", ";
            int keyId = keyframes.GetFrameId(quid);
            if (keyId != -1)
                std::cout << FileNameBase(keyframes.GetName(keyId)) << ", ";
            else
                std::cout << vidSet->QuidToString(quid, true) << ", ";
            std::cout << simTable->Get1(i) << std::endl;
        }
        std::cout << std::endl;
    }
    delete simSet;
}

void
DoImportSimilarityTableSet()
{
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Application.Table.DoImportSimilarityTableSet);
    ILOG_ERROR("Will fail because doesn't have Quids (yet)");
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String outDir = options.GetArg(1);
    String conceptFile = options.GetArg(2);
    String inDir = options.GetArg(3);
    int nrIds = atol(options.GetArg(4));
    String fileExt("");
    if (options.GetNrArg() > 5)
        fileExt = options.GetArg(5);
    Util::Database* db = &Util::Database::GetInstance();

    std::vector<String> concepts;
    Util::DatabaseReadString(std::back_inserter(concepts), conceptFile, db, true);
    ILOG_INFO("nrConcepts = " << concepts.size());
    SimilarityTableSet simSet(concepts, nrIds);
    String outPath = outDir + "/" + FileNameTail(conceptFile) + "/";
    simSet.SaveNames(outPath, db);
    for (int c=0 ; c<concepts.size() ; c++)
    {
        String fName = inDir + "/" + concepts[c] + fileExt;
        ILOG_INFO("importing " << concepts[c] << " from " << fName);
        std::vector<String> lines;
        Util::DatabaseReadString(std::back_inserter(lines), fName, db, true);
        if (lines.size() != nrIds)
        {
            ILOG_ERROR("wrong number of lines : " << lines.size() <<
                       " instead of " << nrIds);
            continue;
        }
        SimTableType* simTable = simSet.GetSimTable(c);
        simTable->SetSize(0);
        for (int i=0 ; i<lines.size() ; i++)
            simTable->Add(atof(lines[i]));
        simSet.ComputeRank(c, true);
        simSet.Save(c, outPath, db, true);
    }
}

void
DoProcSimilarityTableSet()
{
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::SimTableType SimTableType;
    typedef Core::Table::SimilarityTableSet::RankTableType RankTableType;
    typedef Core::Array::Array2dScalarReal64 ArrayType;

    ILOG_VAR(Impala.Application.Table.DoProcSimilarityTableSet);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 7)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String setName = options.GetArg(1);
    Core::VideoSet::VideoSet* vidSet = Core::VideoSet::MakeVideoSet(setName);
    String walkType = options.GetArg(2);
    String conceptSet = options.GetArg(3);
    String model = options.GetArg(4);
    String feature = options.GetArg(5);
    int fileId = Impala::atol(options.GetArg(6));

    SimilarityTableSet* srcSet = Core::Table::SimilarityTableSet::MakeFromFile
        (vidSet, walkType, conceptSet, model, feature, fileId);
    //String fName2 = options.GetArg(2);

    std::vector<String> srcNames = srcSet->GetNames();
    std::vector<String> dstNames;
    for (int i=0 ; i<srcNames.size() ; i++)
    {
        dstNames.push_back(srcNames[i]);
        dstNames.push_back(srcNames[i] + "_proc");
    }
    SimilarityTableSet* dstSet =
        new SimilarityTableSet(dstNames, srcSet->TableSize());


    Core::Table::Copy(dstSet->GetQuidTable(), srcSet->GetQuidTable());
    for (int t=0 ; t<srcSet->NrTables() ; t++)
    {
        SimTableType* srcTable = srcSet->GetSimTable(t);
        Core::Table::Copy(dstSet->GetSimTable(2*t), srcTable);

        SimTableType* dstTable = dstSet->GetSimTable(2*t+1);
        Core::Table::Copy(dstTable, srcTable);
        ArrayType srcWrap(srcTable->Size(), 1, 0, 0,
                          srcTable->GetColumn1()->GetData(), true);
        ArrayType* tmp = 0;

        //Core::Array::Percentile(tmp, &srcWrap, 15, 0.8);

        Core::Array::Percentile(tmp, &srcWrap, 5, 0.5);
        ArrayType* ker = Core::Array::ArrayCreate<ArrayType>(101, 1);
        Core::Matrix::MatSet(ker, 0);
        Core::Array::Erosion(tmp, tmp, ker);
        Core::Array::Dilation(tmp, tmp, ker);
        delete ker;

        Real64* tmpPtr = tmp->CPB(0, 0);
        for (int i=tmp->CW()-1 ; i>=1 ; i--)
        {
            if (tmpPtr[i] == tmpPtr[i-1])
                tmpPtr[i] = 0;
        }

        ker = Core::Array::ArrayCreate<ArrayType>(501, 1);
        Core::Matrix::MatSet(ker, 0);
        ArrayType* localMax = 0;
        Core::Array::Dilation(localMax, tmp, ker);
        Core::Array::Equals(localMax, localMax, tmp);
        Core::Array::Mul(tmp, localMax, tmp);
        delete ker;
        delete localMax;

        ArrayType* dstWrap = new ArrayType(dstTable->Size(), 1, 0, 0,
                          dstTable->GetColumn1()->GetData(), true);
        Core::Array::Set(dstWrap, tmp);
        delete dstWrap;
        delete tmp;

    }
    dstSet->ComputeRanks(true);
    String con = FileNameBase(conceptSet) + "_proc." + FileNameExt(conceptSet);
    dstSet->Save(vidSet, walkType, con, model, feature, fileId, true);
}

void
DoMakeRandomSimilarityTableSet()
{
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Application.Table.DoSimFeatureTable);
    //ILOG_WARNING("DoMakeRandomSimilarityTableSet broken by svn rev 4145");
    //ILOG_ERROR("DoMakeRandomSimilarityTableSet broken by svn rev 4145");
    
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 2)
    {
        ILOG_ERROR("Need more arguments");
        return;
    }
    String setName = options.GetArg(1);
    Core::VideoSet::VideoSet* vidSet = Core::VideoSet::MakeVideoSet(setName);
    Core::VideoSet::Keyframes keyframes(vidSet, "keyframes");

    std::vector<String> names;
    names.push_back("RandomKeyframes");


    for (int v=0 ; v<vidSet->NrFiles() ; v++)
    {
        int nrKeys = keyframes.GetNrKeyframesVideo(v);
        SimilarityTableSet simSet(names, nrKeys);
        int idx = 0;
        SimTableType* simTable = simSet.GetSimTable(idx);
        Core::Table::QuidTable* qTable = simSet.GetQuidTable();
        int firstKey = keyframes.GetFirstKeyframeVideo(v);
        for (int k=0 ; k<nrKeys ; k++)
        {
            Quid q = keyframes.GetQuidFrame(firstKey + k);
            qTable->Add(q);
            Real64 sim = (Real64) rand() / RAND_MAX;
            simTable->Add(sim);
        }
        simSet.ComputeRank(0, true);
        simSet.Save(vidSet, "Keyframes", "artificialConcepts.txt", "no_model",
                    "Random", v, true);
    }
    
}

void
DoDumpTaskTable()
{
    typedef Core::VideoSet::TaskTable TaskTable;

    CmdOptions& options = CmdOptions::GetInstance();
    Util::Database* db = &Util::Database::GetInstance();
    String fName = options.GetArg(1);
    TaskTable table(fName, db);
    //Core::Table::Read(&table, fName);
    table.Dump(options.GetInt("start"), options.GetInt("end"));
}

void
DoDumpTableReal()
{
    CmdOptions& options = CmdOptions::GetInstance();
    DoDump<Real64>(options.GetArg(1));
}

void
DoDumpTableInt()
{
    CmdOptions& options = CmdOptions::GetInstance();
    DoDump<Int32>(options.GetArg(1));
}

void
DoDiffTableReal()
{
    CmdOptions& options = CmdOptions::GetInstance();
    DoDiff<Real64>(options.GetArg(1), options.GetArg(2));
}

void
DoDiffTableInt()
{
    CmdOptions& options = CmdOptions::GetInstance();
    DoDiff<Int32>(options.GetArg(1), options.GetArg(2));
}

void
DoDiffTableUInt64()
{
    CmdOptions& options = CmdOptions::GetInstance();
    DoDiff<UInt64>(options.GetArg(1), options.GetArg(2));
}

int
mainTable(int argc, char* argv[])
{
    CmdOptions& options = CmdOptions::GetInstance();
    options.Initialise(false, false, true);
    options.AddOption(0, "start", "nr", "0");
    options.AddOption(0, "end", "nr", "-1");
    options.AddOption(0, "order", "", "false");
    String usageStr = "cmd = \n\n";
    usageStr += "  vxstable file.vxs\n";
    usageStr += "  scalefeaturetable tablename scaledtablename minmaxtablename\n";
    usageStr += "  concatfeaturetable dir useName resultDef tableDefs\n";
    usageStr += "  featuretable2svm tablename svmname [concept.truth]\n";
    usageStr += "  featuretablereadascii tablename asciiname\n";
    usageStr += "  featuretablewriteascii tablename asciiname\n";
    usageStr += "  simfeaturetable outDir tablename1 tablename2\n";
    usageStr += "  selectfeaturetableset dataset.txt feature output.txt\n";
    usageStr += "  dumpsimilaritytablesettrec videoset.txt names.txt\n";
    usageStr += "  importsimilaritytableset outdir concepts indir nrIds [fileext]\n";
    usageStr += "  procsimilaritytableset videoset.txt walkType concepts.txt model feature fileId\n";
    usageStr += "  makerandomsimilaritytableset videoset.txt\n";
    usageStr += "  dumptasktable tablename\n";
    usageStr += "  dumptablereal tablename\n";
    usageStr += "  dumptableint tablename\n";
    usageStr += "  difftablereal tablename1 tablename2\n";
    usageStr += "  difftableint tablename1 tablename2\n";
    usageStr += "  difftableuint64 tablename1 tablename2\n";
        if (! options.ParseArgs(argc, argv, usageStr, 1))
        return 1;

    ILOG_VAR(Impala.Application.mainTable);

    String cmd = options.GetArg(0);
    ILOG_DEBUG("table command : " << cmd);

    if (cmd == "vxstable")
        DoVxsTable();
    else if (cmd == "scalefeaturetable")
        DoScaleFeatureTable();
    else if (cmd == "append")
        DoAppendFeatureTable();
    else if (cmd == "concatfeaturetable")
        DoConcatFeatureTable();
    else if (cmd == "featuretable2svm")
        DoFeatureTable2Svm();
    else if (cmd == "featuretablereadascii")
        DoFeatureTableReadAscii();
    else if (cmd == "featuretablewriteascii")
        DoFeatureTableWriteAscii();
    else if (cmd == "simfeaturetable")
        DoSimFeatureTable();
    else if (cmd == "selectfeaturetableset")
        DoSelectFeatureTableSet();
    else if (cmd == "dumpsimilaritytablesettrec")
        DoDumpSimilarityTableSetTrec();
    else if (cmd == "importsimilaritytableset")
        DoImportSimilarityTableSet();
    else if (cmd == "procsimilaritytableset")
        DoProcSimilarityTableSet();
    else if (cmd == "makerandomsimilaritytableset")
        DoMakeRandomSimilarityTableSet();
    else if (cmd == "dumptasktable")
        DoDumpTaskTable();
    else if (cmd == "dumptablereal")
        DoDumpTableReal();
    else if (cmd == "dumptableint")
        DoDumpTableInt();
    else if (cmd == "difftablereal")
        DoDiffTableReal();
    else if (cmd == "difftableint")
        DoDiffTableInt();
    else if (cmd == "difftableuint64")
        DoDiffTableUInt64();

    else ILOG_ERROR("Unknown cmd : " << cmd);

    return 0;
}

} // namespace Table
} // namespace Application
} // namespace Impala

int
main(int argc, char* argv[])
{
    return Impala::Application::Table::mainTable(argc, argv);
}

---