|| || || || || || || ||

---

mainMonetTest.cpp

Go to the documentation of this file.

#include "Util/BinHex.h"
#include "Core/Array/ImageArchiveMapi.h"
#include "Core/Array/ImageArchiveDiff.h"
#include "Core/Table/KeywordListDiff.h"
#include "Core/VideoSet/MakeVideoSet.h"
#include "Core/VideoSet/Segmentation.h"
#include "Core/VideoSet/Keyframes.h"
#include "Core/Table/SimilarityTableSet.h"
#include "Core/Table/AnnotationTableSet.h"
#include "Core/Table/SimTableTypeDiff.h"
#include "Core/Feature/FeatureTable.h"
#include "Core/Table/Read.h"


// since we are not using libraries:
#include "Link/ImpalaLib.cpp"

namespace Impala
{
namespace Samples
{
namespace MonetTest
{

using Link::Monet::Connection;
using namespace Core::Array;
using namespace Core::Table;
using namespace Core::Feature;
using namespace Core::VideoSet;
using namespace Persistency;



VideoSet*
MakeVideoSetMapi(String vidSetName, Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeVideoSetMapi);

    String vidSetBase = FileNameBase(vidSetName);
    String query =
        "select f.filename, f.quid \
         from   file f, video_set vs, video_sets vss \
         where  vss.set_name = '" + vidSetBase + "' and \
                vs.video_sets_id = vss.video_sets_id and \
                f.file_id = vs.file_id \
         order by f.quid;";

    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return 0;

    std::vector<String> fName;
    if (!conn->QueryPartFetchString(hdl, 0, fName))
        return 0;

    Quid* quid = 0;
    int quidSize = 0;
    if (!conn->QueryPartFetchULL(hdl, 1, quid, quidSize))
        return 0;

    conn->QueryPartEnd(hdl);

    VideoSet* vidSet = new VideoSet(0, vidSetName, false);
    for (int i=0 ; i<fName.size() ; i++)
    {
        int fileId = QuidId(quid[i]);
        vidSet->AddFile(fName[i], fileId);
        if (vidSet->GetQuidVideo(i, true) != quid[i])
            ILOG_ERROR("Quid " << i << " doesn't match");
    }
    delete quid;
    return vidSet;
}

Segmentation*
MakeSegmentationMapi(VideoSet* vidSet, Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeSegmentationMapi);

    String vidSetBase = vidSet->GetSetNameBase();
    String query =
        "select f.quid, fr.fragment_start, fr.fragment_length, fr.fragment_name \
         from   fragment fr, file f, video_set vs, video_sets vss \
         where  vss.set_name = '" + vidSetBase + "' and \
                vs.video_sets_id = vss.video_sets_id and \
                f.file_id = vs.file_id and \
                fr.media_id = f.media_id and \
                fr.fragment_length > 1 and \
                fr.keyframe = false \
         order by f.quid, fr.fragment_start;";

    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return 0;

    Quid* quid = 0;
    int quidSize = 0;
    if (!conn->QueryPartFetchULL(hdl, 0, quid, quidSize))
        return 0;

    int* start = 0;
    int startSize = 0;
    if (!conn->QueryPartFetchInt(hdl, 1, start, startSize))
        return 0;

    int* length = 0;
    int lengthSize = 0;
    if (!conn->QueryPartFetchInt(hdl, 2, length, lengthSize))
        return 0;

    std::vector<String> name;
    if (!conn->QueryPartFetchString(hdl, 3, name))
        return 0;

    conn->QueryPartEnd(hdl);

    Segmentation* seg = new Segmentation(vidSet, "");
    for (int i=0 ; i<quidSize ; i++)
    {
        int id = QuidId(quid[i]);
        int end = start[i] + length[i] - 1;
        seg->Add(id, start[i], end, name[i]);
    }
    seg->UpdateGroups();

    delete quid;
    delete start;
    delete length;
    return seg;
}

Keyframes*
MakeKeyframesMapi(VideoSet* vidSet, Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeKeyframesMapi);

    String vidSetBase = vidSet->GetSetNameBase();
    String query =
        "select f.quid, fr.fragment_start, fr.fragment_name \
         from   fragment fr, file f, video_set vs, video_sets vss \
         where  vss.set_name = '" + vidSetBase + "' and \
                vs.video_sets_id = vss.video_sets_id and \
                f.file_id = vs.file_id and \
                fr.media_id = f.media_id and \
                fr.keyframe = true \
         order by f.quid, fr.fragment_start;";

    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return 0;

    Quid* quid = 0;
    int quidSize = 0;
    if (!conn->QueryPartFetchULL(hdl, 0, quid, quidSize))
        return 0;

    int* start = 0;
    int startSize = 0;
    if (!conn->QueryPartFetchInt(hdl, 1, start, startSize))
        return 0;

    std::vector<String> name;
    if (!conn->QueryPartFetchString(hdl, 2, name))
        return 0;

    conn->QueryPartEnd(hdl);

    // Need segmentation for shotId
    Segmentation* seg = MakeSegmentationMapi(vidSet, conn);
    Keyframes* key = new Keyframes(vidSet, "");
    for (int i=0 ; i<quidSize ; i++)
    {
        int vidId = QuidId(quid[i]);
        int shotId = seg->GetShotId(vidId, start[i]);
        key->Add(vidId, shotId, start[i], name[i]);
    }
    key->UpdateGroups();
    delete seg;

    delete quid;
    delete start;
    return key;
}

KeywordList
MakeKeywordListMapi(String vidSetName, String conceptSet, Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeKeywordMapi);

    String vidSetBase = FileNameBase(vidSetName);
    String conceptSetBase = FileNameBase(conceptSet);
    String query =
        "select k.keyword_name \
         from   keyword k, keyword_set ks, keyword_sets kss, video_sets vss \
         where  vss.set_name = '" + vidSetBase + "' and \
                kss.video_sets_id = vss.video_sets_id and \
                kss.set_name = '" + conceptSetBase + "' and \
                ks.keyword_sets_id = kss.keyword_sets_id and \
                k.keyword_id = ks.keyword_id \
         order by k.keyword_id;";

    KeywordList concepts;
    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return concepts;

    if (!conn->QueryPartFetchString(hdl, 0, concepts))
        return concepts;

    conn->QueryPartEnd(hdl);

    return concepts;
}

AnnotationTable*
MakeAnnotationTableMapi(VideoSet* vidSet, String conceptSet, String keyword, 
                        int quidClass, Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeAnnotationTableMapi);

    if (quidClass != QUID_CLASS_FRAME)
    {
        ILOG_ERROR("Can do Frame only");
        return 0;
    }

    String vidSetBase = vidSet->GetSetNameBase();
    String query =
        "select f.quid, fr.fragment_start, a.relevance \
         from   fragment fr, file f, video_set vs, video_sets vss, \
                keyword_sets kss, keyword_set ks, keyword k, annotator a, \
                annotation a \
         where  vss.set_name = '" + vidSetBase + "' and \
                vs.video_sets_id = vss.video_sets_id and \
                f.file_id = vs.file_id and \
                fr.media_id = f.media_id and \
                fr.fragment_length = 1 and \
                fr.keyframe = false and \
                kss.set_name = '" + FileNameBase(conceptSet) + "' and \
                kss.video_sets_id = vss.video_sets_id and \
                ks.keyword_sets_id = kss.keyword_sets_id and \
                ks.keyword_id = k.keyword_id and \
                k.keyword_name = '" + keyword + "' and \
                a.annotator_name = 'fabchannel' and \
                a.fragment_id = fr.fragment_id and \
                a.keyword_id = k.keyword_id and \
                a.approved = true \
         order by f.quid, fr.fragment_start;";

    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return 0;

    Quid* quid = 0;
    int quidSize = 0;
    if (!conn->QueryPartFetchULL(hdl, 0, quid, quidSize))
        return 0;

    int* start = 0;
    int startSize = 0;
    if (!conn->QueryPartFetchInt(hdl, 1, start, startSize))
        return 0;

    double* rel = 0;
    int relSize;
    if (!conn->QueryPartFetchDouble(hdl, 2, rel, relSize))
        return 0;

    conn->QueryPartEnd(hdl);

    AnnotationTable* anno = new AnnotationTable(keyword, quidSize);
    for (int i=0 ; i<quidSize ; i++)
    {
        int vidId = QuidId(quid[i]);
        Quid q = vidSet->GetQuidFrame(vidId, start[i], false);
        if (rel[i] == 1.0)
            anno->AddPositive(q);
        else if (rel[i] == 0.0)
            anno->AddNegative(q);
        else
            anno->AddSkip(q);
    }

    delete quid;
    delete start;
    delete rel;
    return anno;
}

AnnotationTableSet*
MakeAnnotationTableSetMapi(VideoSet* vidSet, String conceptSet, bool readTables,
                           int quidClass, Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeAnnotationTableSetMapi);

    KeywordList keywords = MakeKeywordListMapi(vidSet->GetSetName(), conceptSet,
                                               conn);
    ILOG_INFO("Nr keywords = " << keywords.size());

    AnnotationTableSet* res = new AnnotationTableSet();
    int nrRead = 0;
    for (int i=0 ; i<keywords.size() ; i++)
    {
        ILOG_INFO("Doing table " << keywords[i]);
        AnnotationTable* table = 0;
        if (readTables)
        {
            table = MakeAnnotationTableMapi(vidSet, conceptSet, keywords[i],
                                            quidClass, conn);
            if (table != 0)
                nrRead++;
        }
        if (table == 0)
            table = new AnnotationTable(keywords[i], 0);
        res->Add(table);
    }
    if (readTables && (nrRead == 0))
        ILOG_ERROR("No tables found");
    return res;
}

FeatureTable*
MakeFeatureTableMapi(VideoSet* vidSet, int fileIdx, FeatureDefinition def,
                     Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.MakeFeatureTableMapi);

    Quid vidQuid = vidSet->GetQuidVideo(fileIdx, true);
    String query =
        "select fr.fragment_start, fv.vector \
         from feature_vector fv, fragment fr, file f, feature fe \
         where  f.quid = " + MakeString(vidQuid) + " and \
                fr.media_id = f.media_id and \
                fr.fragment_length = 1 and \
                fr.keyframe = false and \
                fe.feature_name = '" + def.AsString() + "' and \
                fe.feature_id = fv.feature_id and \
                fv.fragment_id = fr.fragment_id and \
                fv.file_id = f.file_id \
         order by fr.fragment_start;";

    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return 0;

    int* start = 0;
    int startSize = 0;
    if (!conn->QueryPartFetchInt(hdl, 0, start, startSize))
        return 0;

    std::vector<String> res;
    if (!conn->QueryPartFetchString(hdl, 1, res))
        return 0;

    conn->QueryPartEnd(hdl);

    FeatureTable* feat = 0;
    for (int i=0 ; i<startSize ; i++)
    {
        Quid q = vidSet->GetQuidFrame(fileIdx, start[i], true);

        size_t dataSize;
        UInt8* data = Util::Hex2Bin(res[i], dataSize);        
        int n = dataSize / sizeof(Real64);
        Real64* realData = (Real64*) data;
        FeatureTable::VectorReal64 v(n, realData, true);

        if (feat == 0)
            feat = new FeatureTable(def, res.size(), n);
        feat->Add(q, v);
        delete data;
    }

    if (feat == 0)
        ILOG_ERROR("No features found");
    delete start;
    return feat;
}

void
MakeSimTableMapi(VideoSet* vidSet, int fileIdx, String conceptSet, String model,
                 String feature, String concept, Connection* conn,
                 SimilarityTableSet::SimTableType*& simTab,
                 QuidTable*& quidTab)
{
    typedef SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Samples.MonetTest.MakeSimTableMapi);

    String vidSetBase = vidSet->GetSetNameBase();
    String conceptSetBase = FileNameBase(conceptSet);
    Quid vidQuid = vidSet->GetQuidVideo(fileIdx, true);
    String query =
        "select fr.fragment_start, s.confidence \
         from video_sets vss, video_set vs, keyword_sets kss, keyword_set ks, \
              keyword k, model_sets_name msn, model_sets mss, model_ref mr, \
              file f, fragment fr, score s \
         where vss.set_name = '" + vidSetBase + "' and \
               kss.set_name = '" + conceptSetBase + "' and \
               kss.video_sets_id = vss.video_sets_id and \
               msn.set_name = '" + feature + "' and \
               mss.model_sets_name_id = msn.model_sets_name_id and \
               k.keyword_name = '" + concept + "' and \
               ks.keyword_sets_id = kss.keyword_sets_id and \
               ks.keyword_id = k.keyword_id and \
               mr.model_name = '" + model + "' and \
               mr.keyword_set_id = ks.keyword_set_id and \
               mr.model_ref_id = mss.model_ref_id and \
               vs.video_sets_id = vss.video_sets_id and \
               f.file_id = vs.file_id and \
               f.quid = " + MakeString(vidQuid) + " and \
               fr.media_id = f.media_id and \
               fr.fragment_length = 1 and \
               fr.keyframe = false and \
               s.fragment_id = fr.fragment_id and \
               s.model_sets_id = mss.model_sets_id and \
               s.file_id = f.file_id \
         order by fr.fragment_start;";

    MapiHdl hdl = conn->QueryPartStart(query);
    if (hdl == 0)
        return;

    int* start = 0;
    int startSize = 0;
    if (!conn->QueryPartFetchInt(hdl, 0, start, startSize))
        return;

    double* sims = 0;
    int simSize = 0;
    if (!conn->QueryPartFetchDouble(hdl, 1, sims, simSize))
        return;

    conn->QueryPartEnd(hdl);

    simTab = new SimTableType(startSize);
    quidTab = new QuidTable(startSize);
    for (int i=0 ; i<startSize ; i++)
    {
        simTab->Add(sims[i]);
        Quid q = vidSet->GetQuidFrame(fileIdx, start[i], true);
        quidTab->Add(q);
    }

    delete start;
    delete sims;
}


// Assumes we are handling the "default" start + number specification
int
GetIntInRange(String optName, int minVal, int maxVal, bool isInterval)
{
    CmdOptions& options = CmdOptions::GetInstance();
    int v = options.GetInt(optName);
    if (isInterval)
    {
        if (v == -1)
            return maxVal - minVal;
        if (minVal + v > maxVal)
            return maxVal - minVal;
    }
    else
    {
        if (v < minVal)
            return minVal;
        if (v > maxVal)
            return maxVal;
    }
    return v;
}


void
DoSample(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoSample);

    //conn.Query("select * from emp", true, true);
    //conn.Query("select * from video", true, true);
    //conn.Query("select * from model", true, true);
    //conn->Query("insert into model values (6, 'model nr 6');", true, true);
    conn->Query("copy 2 records into i_bulk_frame_image from stdin using delimiters ' ';\n488640559569698818 0 frame0 FFD8FFE000104A46494600010100000100010000 96 72 image/jpg\n488640559569698818 1 frame1 FFD8FFE000104A46494600010100000100010000 96 72 image/jpg\n", true, true);

}

void
DoInsertVideoSets(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertVideoSets);

    VideoSetsRepository& rep = VideoSetsRepository::GetInstance();
    std::vector<int> ids = rep.GetAllId();
    for (int i=0 ; i<ids.size() ; i++)
    {
        String name = rep.GetSetName(ids[i]);
        String q = "select i_add_video_set(" + MakeString(ids[i])
            + ", '" + name + "');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);
    }
}


void
DoInsertVideoFiles(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertVideoSet);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String vidSetBase = vidSet->GetSetNameBase();

    int startFile = options.GetInt("startFile");
    for (int i=startFile ; i<vidSet->NrFiles() ; i++)
    {
        String file = vidSet->GetAsPath(i);
        Quid quid = vidSet->GetQuidVideo(i, true);
        // todo : get info from video
        //String q = "select i_add_video_file('" + vidSetBase + "', "
        //    + MakeString(quid) + ", '" + file
        //    + "', 352, 288, 25, -1, 'mpeg1', 'bitrate');";
        bool uniqueEvent = false;
        String evName = (uniqueEvent) ? vidSetBase + "_" + MakeString(quid)
                                      : FileNameBase(file);
        String q = "select i_add_video_file_event('" + vidSetBase + "', '"
            + evName + "', 'no description', " + MakeString(quid) + ", '"
            + file + "', 352, 288, true, 25, -1, 'mpeg1', 'bitrate');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);
    }
    delete vidSet;
}


void
DoCheckVideoFiles(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckVideoSet);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    VideoSet* vidSet2 = MakeVideoSetMapi(vidSetName, conn);
    vidSet->Diff(vidSet2);
    delete vidSet;
    delete vidSet2;
}


void
DoInsertSegmentation(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertSegmentation);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);

    Segmentation segmentation(vidSet, "segmentation");
    ILOG_INFO("nrVideos = " << segmentation.GetNrVideos());
    ILOG_INFO("nrShots = " << segmentation.GetNrShots());
    ILOG_INFO("nrFrames = " << segmentation.GetTotalNrFrames());
    conn->Query("delete from i_bulk_fragment;", false, false);
    int startFile = options.GetInt("startFile");
    Timer timer;
    for (int v=startFile ; v<segmentation.GetNrVideos() ; v++)
    {
        ILOG_INFO("v = " << v);
        Quid vidQuid = vidSet->GetQuidVideo(v, true);
        int first = segmentation.GetFirstShotVideo(v);
        int nr = segmentation.GetNrShotsVideo(v);
        String q = "copy " + MakeString(nr) +
            " records into i_bulk_fragment from stdin using delimiters ' ';\n";
        for (int s=first ; s<first+nr ; s++)
        {
            int start = segmentation.GetStart(s);
            int len = segmentation.GetEnd(s) - start + 1;
            q += MakeString(vidQuid) + " " + MakeString(start) + " "
                + MakeString(len) + " " + segmentation.GetName(s)
                + " false false\n";
        }
        conn->Query(q, false, false);

        nr = segmentation.GetNrFramesVideo(v);
        q = "copy " + MakeString(nr) +
            " records into i_bulk_fragment from stdin using delimiters ' ';\n";
        for (int f=0 ; f<nr ; f++)
        {
            q += MakeString(vidQuid) + " " + MakeString(f) + " 1 frame"
                + MakeString(f) + " false false\n";
        }
        conn->Query(q, false, false);
    }
    ILOG_INFO("Did bulk in " << timer.SplitTime());
    conn->Query("insert into fragment (media_id, fragment_start, \
                                       fragment_length, fragment_name, \
                                       keyframe, representative) \
                 select m.media_id, bf.fragment_start, bf.fragment_length, \
                        bf.fragment_name, bf.keyframe, bf.representative \
                 from i_bulk_fragment bf, media m \
                 where bf.quid = m.quid \
                 order by bf.quid, bf.fragment_start;", false, false);
    ILOG_INFO("Did insert select at " << timer.SplitTime());
    conn->Query("delete from i_bulk_fragment;", false, false);
}


void
DoCheckSegmentation(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckSegmentation);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    Segmentation segmentation(vidSet, "segmentation");
    Segmentation* segmentation2 = MakeSegmentationMapi(vidSet, conn);
    segmentation.Diff(segmentation2);
    delete segmentation2;
    delete vidSet;
}


void
DoInsertKeyframes(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertKeyframes);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);

    Keyframes keyframes(vidSet, "keyframes");
    ILOG_INFO("nrVideos = " << keyframes.GetNrVideos());
    ILOG_INFO("nrKeyframes = " << keyframes.GetNrKeyframes());
    conn->Query("delete from i_bulk_fragment;", false, false);
    int startFile = options.GetInt("startFile");
    Timer timer;
    for (int v=startFile ; v<keyframes.GetNrVideos() ; v++)
    {
        ILOG_INFO("v = " << v);
        Quid vidQuid = vidSet->GetQuidVideo(v, true);
        int first = keyframes.GetFirstKeyframeVideo(v);
        int nr = keyframes.GetNrKeyframesVideo(v);
        String q = "copy " + MakeString(nr) +
            " records into i_bulk_fragment from stdin using delimiters ' ';\n";
        for (int k=first ; k<first+nr ; k++)
        {
            //int shot = keyframes.GetShotId(k); // cannot store in schema
            int frame = keyframes.GetFrameNr(k);
            String rkf = (keyframes.IsRKF(k)) ? "true" : "false";
            q += MakeString(vidQuid) + " " + MakeString(frame) + " 1 "
                + keyframes.GetName(k) + " true " + rkf + "\n";
        }
        conn->Query(q, false, false);
    }
    ILOG_INFO("Did bulk in " << timer.SplitTime());
    conn->Query("insert into fragment (media_id, fragment_start, \
                                       fragment_length, fragment_name, \
                                       keyframe, representative) \
                 select m.media_id, bf.fragment_start, bf.fragment_length, \
                        bf.fragment_name, bf.keyframe, bf.representative \
                 from i_bulk_fragment bf, media m \
                 where bf.quid = m.quid \
                 order by bf.quid, bf.fragment_start;", false, false);
    ILOG_INFO("Did insert select at " << timer.SplitTime());
    conn->Query("delete from i_bulk_fragment;", false, false);
}


void
DoCheckKeyframes(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckKeyframes);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    Keyframes keyframes(vidSet, "keyframes");
    Keyframes* keyframes2 = MakeKeyframesMapi(vidSet, conn);
    keyframes.Diff(keyframes2);
    delete keyframes2;
    delete vidSet;
}


void
DoInsertFrameArchive(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertFrameArchive);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);

    conn->Query("delete from i_bulk_frame_image;", false, false);
    int startFile = options.GetInt("startFile");
    Timer timer;
    for (int v=startFile ; v<vidSet->NrFiles() ; v++)
    {
        String path = vidSet->GetFilePathFrames(v, "images_jpg.raw", false,
                                                false);
        if (path.empty())
            continue;
        ImageArchiveFile ar(path, false, vidSet->GetDatabase());
        ILOG_INFO("vid " << v << " has " << ar.NrImages() << " images");
        ILOG_NDC_PUSH("v=" << v);
        Quid vidQuid = vidSet->GetQuidVideo(v, true);

        conn->Query("delete from i_bulk_frame_image;", false, false);
        int totalSize = ar.NrImages();
        int bulkSize = 4000;
        Timer timer;

        for (int i=0 ; i<totalSize ; i+=bulkSize)
        {
            int left = totalSize - i;
            int nr = (left > bulkSize) ? bulkSize : left;
            String q = "copy " + MakeString(nr) +
                " records into i_bulk_frame_image from stdin" +
                " using delimiters ' ';\n";
            for (int j=i ; j<i+nr ; j++)
            {
                Array2dVec3UInt8* im = ar.ReadImage(j);
                size_t dataSize = 0;
                UInt8* data = ar.GetImageData(j, dataSize);
                String hexData = Util::Bin2Hex(data, dataSize);
                delete data;
                String name = "frame" + MakeString(j);
                String width = MakeString(im->CW());
                String height = MakeString(im->CH());
                q += MakeString(vidQuid) + " " + MakeString(j) + " " + name + " "
                    + hexData + " " + width + " " + height + " image/jpg \n";
                delete im;
            }
            conn->Query(q, false, false);
            ILOG_INFO("Did bulk " << i << " at " << timer.SplitTime());

            conn->Query("insert into frame_image (fragment_id, image, width, \
                                                  height, mime_type) \
                         select fr.fragment_id, bf.image, bf.width, bf.height, \
                                bf.mime_type \
                         from i_bulk_frame_image bf, file f, fragment fr \
                         where bf.quid = f.quid and \
                               f.media_id = fr.media_id and \
                               fr.fragment_start = bf.fragment_start and \
                               fr.fragment_length = 1 and \
                               fr.keyframe = false \
                         order by bf.quid, bf.fragment_start;", false, false);
            ILOG_INFO("Did insert select at " << timer.SplitTime());
            conn->Query("delete from i_bulk_frame_image;", false, false);
        }
        ILOG_NDC_POP;
    }
    delete vidSet;
}


void
DoCheckFrameArchive(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckFrameArchive);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);

    int startFile = options.GetInt("startFile");
    Timer timer;
    for (int v=startFile ; v<vidSet->NrFiles() ; v++)
    {
        String path = vidSet->GetFilePathFrames(v, "images_jpg.raw", false,
                                                false);
        if (path.empty())
            continue;
        ImageArchiveFile ar(path, false, vidSet->GetDatabase());
        ILOG_INFO("video " << v << " has " << ar.NrImages() << " images");

        Quid vidQuid = vidSet->GetQuidVideo(v, true);

        ImageArchiveMapi ar2(vidQuid, conn);
        Core::Array::ImageArchiveDiff(&ar, &ar2);
        ILOG_INFO("Did diff at " << timer.SplitTime());
    }
    delete vidSet;
}


void
DoInsertKeywords(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertKeywords);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 6)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String conceptSet = options.GetArg(5);
    String path = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (path.empty())
        return;
    KeywordList concepts;
    Util::Database* db = vidSet->GetDatabase();
    Util::DatabaseReadString(std::back_inserter(concepts), path, db, true);

    String vidSetBase = vidSet->GetSetNameBase();
    String conceptSetBase = FileNameBase(conceptSet);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        String q = "select i_add_keyword('" + vidSetBase + "', '" +
            conceptSetBase + "', '" + concepts[i] + "');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);
    }
}


void
DoCheckKeywords(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckKeywords);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 6)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String conceptSet = options.GetArg(5);
    String path = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (path.empty())
        return;
    KeywordList concepts;
    Util::Database* db = vidSet->GetDatabase();
    Util::DatabaseReadString(std::back_inserter(concepts), path, db, true);


    KeywordList concepts2 = MakeKeywordListMapi(vidSetName, conceptSet, conn);
    Core::Table::KeywordListDiff(concepts, concepts2);
    delete vidSet;
}


void
DoInsertAnnotations(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertAnnotations);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 7)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String conceptSet = options.GetArg(5);
    int quidClass = StringToQuidClass(options.GetArg(6));
    if (quidClass != QUID_CLASS_FRAME)
    {
        ILOG_ERROR("Can do frame annotations only.");
        return;
    }

    AnnotationTableSet* tSet =
        Core::Table::AnnotationTableSet::MakeFromFile(vidSet, conceptSet, true,
                                                      quidClass);
    if (tSet == 0)
        return;

    conn->Query("select i_add_annotator('fabchannel', 'the boss', true);",
                false, false);

    conn->Query("delete from i_bulk_annotation;", false, false);
    Timer timer;
    for (int t=0 ; t<tSet->Size() ; t++)
    {
        AnnotationTable* tab = tSet->GetTable(t);
        String keyword = tSet->GetLabel(t);
        ILOG_INFO("keyword " << keyword << " has " << tab->Size() << " annos");
        String date("2007/08/22-11:00:00");
        int nr = tab->Size();
        String q = "copy " + MakeString(nr) +
            " records into i_bulk_annotation from stdin using delimiters ' ';\n";
        for (int i=0 ; i<nr ; i++)
        {
            Quid frameQuid = tab->Get1(i);
            int vidId = QuidObject(frameQuid);
            Quid vidQuid = vidSet->GetQuidVideo(vidId, false);
            int frameNr = QuidId(frameQuid);
            Real64 relevance = 0.5; // skip, using Mpeg7 convention
            if (tab->IsPositive(i))
                relevance = 1.0;
            else if (tab->IsNegative(i))
                relevance = 0.0;
            Real64 confidence = 1.0;
            q += keyword + " " + MakeString(vidQuid) + " " + MakeString(frameNr)
                + " " + MakeString(relevance) + " " + MakeString(confidence)
                + " " + date + "\n";
        }
        conn->Query(q, false, false);
    }
    ILOG_INFO("Did bulk in " << timer.SplitTime());
    String vidSetBase = vidSet->GetSetNameBase();
    conn->Query("insert into annotation (fragment_id, keyword_id, annotator_id, \
                                         relevance, confidence, \
                                         annotation_date, approved) \
                 select fr.fragment_id, k.keyword_id, a.annotator_id, \
                        ba.relevance, ba.confidence, ba.annotation_date, \
                        a.approved \
                 from i_bulk_annotation ba, media m, fragment fr, \
                      video_sets vss, \
                      keyword_sets kss, keyword_set ks, keyword k, annotator a \
                 where ba.quid = m.quid and \
                       m.media_id = fr.media_id and \
                       fr.fragment_start = ba.fragment_start and \
                       fr.fragment_length = 1 and \
                       fr.keyframe = false and \
                       vss.set_name = '" + vidSetBase + "' and \
                       kss.set_name = '" + FileNameBase(conceptSet) + "' and \
                       kss.video_sets_id = vss.video_sets_id and \
                       ks.keyword_sets_id = kss.keyword_sets_id and \
                       ks.keyword_id = k.keyword_id and \
                       k.keyword_name = ba.keyword_name and \
                       a.annotator_name = 'fabchannel' \
                 order by ba.quid, ba.fragment_start;", false, false);
    conn->Query("delete from i_bulk_annotation;", false, false);
    delete tSet;
    delete vidSet;
}


void
DoCheckAnnotations(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckAnnotations);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 7)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String conceptSet = options.GetArg(5);
    int quidClass = StringToQuidClass(options.GetArg(6));
    if (quidClass != QUID_CLASS_FRAME)
    {
        ILOG_ERROR("Can do frame annotations only.");
        return;
    }

    AnnotationTableSet* tSet =
        Core::Table::AnnotationTableSet::MakeFromFile(vidSet, conceptSet, true,
                                                      quidClass);
    if (tSet == 0)
        return;

    AnnotationTableSet* tSet2 = MakeAnnotationTableSetMapi(vidSet, conceptSet,
                                                           true, quidClass,
                                                           conn);
    tSet->Diff(tSet2);
    delete tSet;
    delete tSet2;
    delete vidSet;
}


void
DoInsertFeatures(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertFeatures);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 6)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    FeatureDefinition def(options.GetArg(5));

    String q = "select i_add_feature('" + def.AsString() + "');";
    conn->Query(q, false, false);

    int startFile = options.GetInt("startFile");
    for (int v=startFile ; v<vidSet->NrFiles() ; v++)
    {
        String fName = vidSet->GetFilePathFeatureData("Frames", def, v, false,
                                                      -1, false, false);
        if (fName.empty())
        {
            ILOG_ERROR("Unable to find " << def.AsString());
            continue;
        }
        FeatureTable* tab =
            Core::Feature::FeatureTable::MakeFromFile(def, fName,
                                                      vidSet->GetDatabase());
        ILOG_INFO("vid " << v << " has " << tab->Size() << " vectors");
        ILOG_NDC_PUSH("v=" << v);
        Quid vidQuid = vidSet->GetQuidVideo(v, true);

        conn->Query("delete from i_bulk_feature_vector;", false, false);
        int totalSize = tab->Size();
        int bulkSize = 2000;
        Timer timer;
        for (int i=0 ; i<totalSize ; i+=bulkSize)
        {
            int left = totalSize - i;
            int nr = (left > bulkSize) ? bulkSize : left;
            String q = "copy " + MakeString(nr) +
                " records into i_bulk_feature_vector from stdin" +
                " using delimiters ' ';\n";
            for (int j=i ; j<i+nr ; j++)
            {
                Quid frameQuid = tab->Get1(j);
                if (QuidClass(frameQuid) != QUID_CLASS_FRAME)
                    ILOG_ERROR("Quid is not a frame");
                int frameNr = QuidId(frameQuid);
                FeatureTable::VectorReal64 v = tab->Get2(j);
                size_t dataSize = v.Size() * sizeof(Real64);
                UInt8* data = (UInt8*) v.GetData();
                String hexData = Util::Bin2Hex(data, dataSize);

                q += MakeString(vidQuid) + " " + MakeString(frameNr) + " "
                    + hexData + "\n";
            }
            conn->Query(q, false, false);
            ILOG_INFO("Did bulk " << i << " at " << timer.SplitTime());

            String vidSetBase = vidSet->GetSetNameBase();
            conn->Query("insert into feature_vector (fragment_id, file_id, \
                                                     feature_id, vector) \
                         select fr.fragment_id, f.file_id, fe.feature_id, \
                                bfv.vector \
                         from i_bulk_feature_vector bfv, file f, \
                              fragment fr, feature fe \
                         where bfv.quid = f.quid and \
                               f.media_id = fr.media_id and \
                               fr.fragment_start = bfv.fragment_start and \
                               fr.fragment_length = 1 and \
                               fr.keyframe = false and \
                               fe.feature_name = '" + def.AsString() + "' \
                         order by bfv.fragment_start;", false, false);
            ILOG_INFO("Did insert select at " << timer.SplitTime());
            conn->Query("delete from i_bulk_feature_vector;", false, false);
        }
        delete tab;
        ILOG_NDC_POP;
    }
    delete vidSet;
}


void
DoCheckFeatures(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoCheckFeatures);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 6)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    FeatureDefinition def(options.GetArg(5));

    for (int v=0 ; v<vidSet->NrFiles() ; v++)
    {
        ILOG_NDC_PUSH("v=" << v);
        Timer timer;
        // reversed order to reduce memory usage
        FeatureTable* tab2 = MakeFeatureTableMapi(vidSet, v, def, conn);
        ILOG_INFO("Did mapi at " << timer.SplitTime());

        String fName = vidSet->GetFilePathFeatureData("Frames", def, v, false,
                                                      -1, false, false);
        if (fName.empty())
        {
            ILOG_ERROR("Unable to find " << def.AsString());
            delete tab2;
            ILOG_NDC_POP;
            continue;
        }
        FeatureTable* tab =
            Core::Feature::FeatureTable::MakeFromFile(def, fName,
                                                      vidSet->GetDatabase());
        ILOG_INFO("Did file at " << timer.SplitTime());

        if (tab->Diff(tab2) == 0)
            ILOG_INFO("No differences");
        delete tab;
        delete tab2;
        ILOG_NDC_POP;
    }
    delete vidSet;
}


void
DoInsertModels(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertModels);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 8)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    String conceptSet = options.GetArg(5);
    String modelName = options.GetArg(6);
    String featureDef(options.GetArg(7));

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String path = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (path.empty())
        return;
    KeywordList concepts;
    Util::Database* db = vidSet->GetDatabase();
    Util::DatabaseReadString(std::back_inserter(concepts), path, db, true);

    String q = "select i_add_feature('" + featureDef + "');";
    ILOG_INFO("q = [" + q + "]");
    conn->Query(q, false, false);

    String vidSetBase = vidSet->GetSetNameBase();
    String conceptSetBase = FileNameBase(conceptSet);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        String q = "select i_add_model('" + vidSetBase + "', '" +
            conceptSetBase + "', '" + concepts[i] + "', '" + modelName +
            "', '" + featureDef + "');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);

        // a model is also a set with itself as element
        q = "select i_add_model_sets('" + vidSetBase + "', '" +
            conceptSetBase + "', '" + concepts[i] + "', '" + modelName +
            "', '" + featureDef + "');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);
        q = "select i_add_model_to_set('" + vidSetBase + "', '" +
            conceptSetBase + "', '" + concepts[i] + "', '" + modelName +
            "', '" + featureDef + "', '" + featureDef + "');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);
    }
}


void
DoInsertModelSets(Connection* conn)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoInsertModelSets);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 9)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    String conceptSet = options.GetArg(5);
    String modelName = options.GetArg(6);
    String modelSetName = options.GetArg(7);
    std::vector<String> featureDefs;
    for (int i=8 ; i<options.GetNrArg() ; i++)
        featureDefs.push_back(options.GetArg(i));

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String path = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (path.empty())
        return;
    KeywordList concepts;
    Util::Database* db = vidSet->GetDatabase();
    Util::DatabaseReadString(std::back_inserter(concepts), path, db, true);

    String vidSetBase = vidSet->GetSetNameBase();
    String conceptSetBase = FileNameBase(conceptSet);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        String q = "select i_add_model_sets('" + vidSetBase + "', '" +
            conceptSetBase + "', '" + concepts[i] + "', '" + modelName +
            "', '" + modelSetName + "');";
        ILOG_INFO("q = [" + q + "]");
        conn->Query(q, false, false);

        String baseQ = "select i_add_model_to_set('" + vidSetBase + "', '" +
            conceptSetBase + "', '" + concepts[i] + "', '" + modelName +
            "', '" + modelSetName + "', '";
        for (int f=0 ; f<featureDefs.size() ; f++)
        {
            String q = baseQ + featureDefs[f] + "');";
            ILOG_INFO("q = [" + q + "]");
            conn->Query(q, false, false);
        }
    }
}


void
InsertOneSimTable(Connection* conn, VideoSet* vidSet, String conceptSet,
                  String model, String feature, int fileIdx, String concept,
                  QuidTable* quidTab)
{
    typedef SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Samples.MonetTest.InsertOneSimTable);
    CmdOptions& options = CmdOptions::GetInstance();

    SimTableType simTable(0);
    String path =
        vidSet->GetFilePathSimilarityData("Frames", conceptSet, model,
                                          feature, fileIdx,
                                          concept + "_sim.tab", false, false);
    if (path.empty())
        return;
    Core::Table::Read(&simTable, path, vidSet->GetDatabase());
    ILOG_INFO("simTable size = " << simTable.Size());
    Quid vidQuid = vidSet->GetQuidVideo(fileIdx, true);

    conn->Query("delete from i_bulk_score;", false, false);
    int startScore = GetIntInRange("startScore", 0, simTable.Size(), false);
    int numberScores = GetIntInRange("numberScores", startScore,
                                     simTable.Size(), true);
    int bulkSize = 100000;
    Timer timer;
    for (int i=startScore ; i<startScore+numberScores ; i+=bulkSize)
    {
        int left = numberScores - i;
        int nr = (left > bulkSize) ? bulkSize : left;
        String q = "copy " + MakeString(nr) +
                 " records into i_bulk_score from stdin using delimiters ' ';\n";
        for (int j=i ; j<i+nr ; j++)
        {
            Quid frameQuid = quidTab->Get1(j);
            if (QuidClass(frameQuid) != QUID_CLASS_FRAME)
                ILOG_ERROR("Quid is not a frame");
            int frameNr = QuidId(frameQuid);

            q += MakeString(vidQuid) + " " + MakeString(frameNr) + " "
                + MakeString(simTable.Get1(j)) + "\n";
        }
        conn->Query(q, false, false);
        ILOG_INFO("Did bulk at " << timer.SplitTime());
    }

    String vidSetBase = vidSet->GetSetNameBase();
    String conceptSetBase = FileNameBase(conceptSet);
    conn->Query("insert into score (fragment_id, model_sets_id, file_id, \
                                    confidence) \
                 select fr.fragment_id, mss.model_sets_id, f.file_id, \
                        bs.confidence \
                 from video_sets vss, video_set vs, keyword_sets kss, \
                      keyword_set ks, keyword k, model_sets_name msn, \
                      model_sets mss, model_ref mr, \
                      file f, fragment fr, i_bulk_score bs \
                 where vss.set_name = '" + vidSetBase + "' and \
                       kss.set_name = '" + conceptSetBase + "' and \
                       kss.video_sets_id = vss.video_sets_id and \
                       msn.set_name = '" + feature + "' and \
                       mss.model_sets_name_id = msn.model_sets_name_id and \
                       k.keyword_name = '" + concept + "' and \
                       ks.keyword_sets_id = kss.keyword_sets_id and \
                       ks.keyword_id = k.keyword_id and \
                       mr.model_name = '" + model + "' and \
                       mr.keyword_set_id = ks.keyword_set_id and \
                       mr.model_ref_id = mss.model_ref_id and \
                       vs.video_sets_id = vss.video_sets_id and \
                       f.file_id = vs.file_id and \
                       bs.quid = f.quid and \
                       fr.media_id = f.media_id and \
                       fr.fragment_start = bs.fragment_start and \
                       fr.fragment_length = 1 and \
                       fr.keyframe = false \
                 order by bs.fragment_start;", false, false);
    ILOG_INFO("Did insert select at " << timer.SplitTime());
    conn->Query("delete from i_bulk_score;", false, false);
    ILOG_INFO("Done in " << timer.SplitTime());
}

void
CheckOneSimTable(Connection* conn, VideoSet* vidSet, String conceptSet,
                 String model, String feature, int fileId, String concept,
                 QuidTable* quidTab)
{
    typedef SimilarityTableSet::SimTableType SimTableType;
    CmdOptions& options = CmdOptions::GetInstance();

    ILOG_VAR(Impala.Samples.MonetTest.CheckOneSimTable);
    Timer timer;

    SimTableType table(0);
    String path =
        vidSet->GetFilePathSimilarityData("Frames", conceptSet, model,
                                          feature, fileId,
                                          concept + "_sim.tab", false, false);
    if (path.empty())
        return;
    Core::Table::Read(&table, path, vidSet->GetDatabase());
    ILOG_INFO("Did file in " << timer.SplitTime());
    ILOG_INFO("table size = " << table.Size());

    // ignore startScore for now
    //int startScore = options.GetInt("startScore");
    int numberScores = GetIntInRange("numberScores", 0, table.Size(), true);
    table.SetSize(numberScores);
    quidTab->SetSize(numberScores);

    SimTableType* simTab2 = 0;
    QuidTable* quidTab2 = 0;
    MakeSimTableMapi(vidSet, fileId, conceptSet, model, feature, concept, conn,
                     simTab2, quidTab2);
    ILOG_INFO("Did mapi at " << timer.SplitTime());
if (quidTab->Size() == quidTab2->Size() + 1)
{
ILOG_INFO("Adjusting size to " << quidTab2->Size());
quidTab->SetSize(quidTab2->Size());
}
    if (quidTab->Diff(quidTab2) == 0)
    {
        ILOG_INFO("No differences in quids");
    }
    else
    {
        //quidTab->Dump(vidSet, 0, -1);
        //quidTab2->Dump(vidSet, 0, -1);
    }
if (table.Size() == simTab2->Size() + 1)
{
ILOG_INFO("Adjusting size to " << simTab2->Size());
table.SetSize(simTab2->Size());
}
    if (SimTableTypeDiff(&table, simTab2) == 0)
        ILOG_INFO("No differences in sims");
    delete simTab2;
    delete quidTab2;
}

void
DoSimilarities(Connection* conn, bool doInsert)
{
    ILOG_VAR(Impala.Samples.MonetTest.DoSimilarities);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 8)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }

    String vidSetName = options.GetArg(4);
    String conceptSet = options.GetArg(5);
    String modelName = options.GetArg(6);
    String feature = options.GetArg(7);

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    String path = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (path.empty())
        return;
    KeywordList concepts;
    Util::Database* db = vidSet->GetDatabase();
    Util::DatabaseReadString(std::back_inserter(concepts), path, db, true);

    int startF = GetIntInRange("startFile", 0, vidSet->NrFiles(), false);
    int numberF = GetIntInRange("numberFiles", startF, vidSet->NrFiles(), true);
    for (int v=startF ; v<startF+numberF ; v++)
    {
        QuidTable quidTab(0);
        String path =
            vidSet->GetFilePathSimilarityData("Frames", conceptSet, modelName,
                                              feature, v, "all_quids.tab",
                                              false, false);
        if (path.empty())
            continue;
        Core::Table::Read(&quidTab, path, vidSet->GetDatabase());
        ILOG_INFO("quidTab size = " << quidTab.Size());
        int startConcept = GetIntInRange("startConcept", 0, concepts.size(),
                                         false);
        int nrConcepts = GetIntInRange("numberConcepts", startConcept,
                                       concepts.size(), true);
        for (int c=startConcept ; c<startConcept+nrConcepts ; c++)
        {
            ILOG_NDC_PUSH("video " << v << " - " << concepts[c]);
            if (doInsert)
                InsertOneSimTable(conn, vidSet, conceptSet, modelName,
                                  feature, v, concepts[c], &quidTab);
            else
                CheckOneSimTable(conn, vidSet, conceptSet, modelName,
                                 feature, v, concepts[c], &quidTab);
            ILOG_NDC_POP;
        }
    }

    delete vidSet;
}

int
mainMonetTest(int argc, char* argv[])
{
    CmdOptions& options = CmdOptions::GetInstance();
    options.Initialise(false, false, true);
    options.AddOption(0, "saveQueries", "log_dir", "");
    options.AddOption(0, "startFile", "idx", "0");
    options.AddOption(0, "numberFiles", "nr", "-1");
    options.AddOption(0, "startConcept", "idx", "0");
    options.AddOption(0, "numberConcepts", "nr", "-1");
    options.AddOption(0, "startScore", "idx", "0");
    options.AddOption(0, "numberScores", "nr", "-1");
    options.AddOption(0, "", "nr", "-1");
    String usageStr = "cmd = \n\n";
    usageStr += "  sample host port db\n";
    usageStr += "  insertvideosets host port db\n";
    usageStr += "  insertvideofiles host port db videoSet.txt\n";
    usageStr += "  checkvideofiles host port db videoSet.txt\n";
    usageStr += "  insertsegmentation host port db videoSet.txt\n";
    usageStr += "  checksegmentation host port db videoSet.txt\n";
    usageStr += "  insertkeyframes host port db videoSet.txt\n";
    usageStr += "  checkkeyframes host port db videoSet.txt\n";
    usageStr += "  insertframearchive host port db videoSet.txt\n";
    usageStr += "  checkframearchive host port db videoSet.txt\n";
    usageStr += "  insertkeywords host port db videoSet.txt concepts.txt\n";
    usageStr += "  checkkeywords host port db videoSet.txt concepts.txt\n";
    usageStr += "  insertannotations host port db videoSet.txt concepts.txt quidClass\n";
    usageStr += "  checkannotations host port db videoSet.txt concepts.txt quidClass\n";
    usageStr += "  insertfeatures host port db videoSet.txt featureDef\n";
    usageStr += "  checkfeatures host port db videoSet.txt featureDef\n";
    usageStr += "  insertmodels host port db videoSet.txt concepts.txt model featureDef\n";
    usageStr += "  insertmodelsets host port db videoSet.txt concepts.txt model modelSet featureDef[s]\n";
    usageStr += "  insertsimilarities host port db videoSet.txt concepts.txt model feature\n";
    usageStr += "  checksimilarities host port db videoSet.txt concepts.txt model feature\n";
    if (! options.ParseArgs(argc, argv, usageStr, 4))
        return 1;

    ILOG_VAR(Impala.Samples.mainMonetTest);

    String cmd = options.GetArg(0);
    String host = options.GetArg(1);
    int port = atol(options.GetArg(2));
    String dbName = options.GetArg(3);
    ILOG_INFO("Connecting to " << dbName << " on " << host << ":" << port);
    Link::Monet::Connection conn(host, port, dbName);
    if (!conn.Valid())
        return 1;
    conn.SetSaveQueries(options.GetString("saveQueries"));

    Timer timer;
    if (cmd == "sample")
        DoSample(&conn);
    else if (cmd == "insertvideosets")
        DoInsertVideoSets(&conn);
    else if (cmd == "insertvideofiles")
        DoInsertVideoFiles(&conn);
    else if (cmd == "checkvideofiles")
        DoCheckVideoFiles(&conn);
    else if (cmd == "insertsegmentation")
        DoInsertSegmentation(&conn);
    else if (cmd == "checksegmentation")
        DoCheckSegmentation(&conn);
    else if (cmd == "insertkeyframes")
        DoInsertKeyframes(&conn);
    else if (cmd == "checkkeyframes")
        DoCheckKeyframes(&conn);
    else if (cmd == "insertframearchive")
        DoInsertFrameArchive(&conn);
    else if (cmd == "checkframearchive")
        DoCheckFrameArchive(&conn);
    else if (cmd == "insertkeywords")
        DoInsertKeywords(&conn);
    else if (cmd == "checkkeywords")
        DoCheckKeywords(&conn);
    else if (cmd == "insertannotations")
        DoInsertAnnotations(&conn);
    else if (cmd == "checkannotations")
        DoCheckAnnotations(&conn);
    else if (cmd == "insertfeatures")
        DoInsertFeatures(&conn);
    else if (cmd == "checkfeatures")
        DoCheckFeatures(&conn);
    else if (cmd == "insertmodels")
        DoInsertModels(&conn);
    else if (cmd == "insertmodelsets")
        DoInsertModelSets(&conn);
    else if (cmd == "insertsimilarities")
        DoSimilarities(&conn, true);
    else if (cmd == "checksimilarities")
        DoSimilarities(&conn, false);
    else ILOG_ERROR("Unknown cmd : " << cmd);

    ILOG_INFO("Did command in " << timer.SplitTimeStr());
    int nrError = ILogErrors::GetInstance().GetTotalNrErrors();
    ILOG_INFO("NrError = " << nrError);
    return nrError;
}

} // namespace MonetTest
} // namespace Samples
} // namespace Impala

int
main(int argc, char* argv[])
{
    return Impala::Samples::MonetTest::mainMonetTest(argc, argv);
}

---