|| || || || || || || ||

---

mainUtil.cpp

Go to the documentation of this file.

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
#else // REPOSITORY_USED
#define REPOSITORY_TODO
#endif // REPOSITORY_USED

#include "Persistency/SegmentationRepository.h"
#include "Persistency/FrameRepository.h"
#include "Persistency/KeywordListRepository.h"
#include "Util/Channel.h"
#include "Basis/CmdOptions.h"
#include "Util/SimpleMap.h"
#include "Util/StringParser.h"
#include "Core/Table/AveragePrecision.h"
#include "Core/Table/AveragePrecisionConfusion.h"
#include "Core/Table/AnnotationTableSet.h"
#include "Core/Table/SimilarityTableSet.h"
#include "Core/Table/SelectQuids.h"
#include "Core/Trec/TrecTopic.h"
#include "Core/Trec/Collection.h"
#include "Core/Trec/SearchJudge.h"
#include "Core/Array/Element/CheckNativeTypeSize.h"
#include "Core/Array/WritePng.h"
#include "Core/Column/AddAssign.h"
#include "Core/Column/MulAssign.h"
#include "Core/Column/DivAssign.h"
#include "Core/Database/MakeRawDataSet.h"
#include "Core/VideoSet/Stills.h"
#include "Core/VideoSet/Keyframes.h"
#include "Core/VideoSet/Mpeg7DocWrite.h"


// since we are not using libraries:
#include "Link/ImpalaLib.cpp"

namespace Impala
{
namespace Application
{
namespace Util
{


using namespace Persistency;

void
DoTrecTopicData()
{
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::Array::Array2dVec3UInt8 Array2dVec3UInt8;

    ILOG_VAR(Impala.Application.Util.DoTrecTopicData);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    VideoSet* sVidSet = Core::VideoSet::MakeVideoSet(options.GetArg(1));
    Core::Trec::TrecTopic topic(options.GetArg(2));
    ILOG_DEBUG("Generating data sets");
    String dSetName = topic.GenerateDataSets(options.GetArg(3));
    bool doVideo = true;
    if (options.GetNrArg() > 4)
        if (options.GetArg(4) == "novideo")
            doVideo = false;
    VideoSet* dVidSet = Core::VideoSet::MakeVideoSet(dSetName);
    KeyframesLocator kLoc(dSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, dVidSet);
    int curKey = -1;
    for (int i=0 ; i<topic.NrTopic() ; i++)
    //for (int i=0 ; i<1 ; i++)
    {
        for (int j=0 ; j<topic.GetNrExamples(i) ; j++)
        {
            curKey++;
            ILOG_INFO("nr " << curKey << " of " << keyframes->GetNrKeyframes());
            ILOG_DEBUG("topic = " << i << ", example = " << j);
            ILOG_DEBUG("topic = " << topic.GetTopicNum(i));
            String vidName = topic.GetVideoExampleSrcFile(i, j);
            ILOG_DEBUG("vidName = " << vidName);
            int vidId = sVidSet->GetFileId(vidName);
            if (vidId == -1)
            {
                ILOG_ERROR("Unable to locate video " << vidName);
                continue;
            }
            Core::Stream::RgbDataSrc* src = sVidSet->GetVideo(vidId);
            if (src == 0)
            {
                ILOG_ERROR("Unable to locate video " << vidName);
                continue;
            }
            int startFr = topic.GetVideoExampleStartFrame(i, j);
            int endFr = topic.GetVideoExampleEndFrame(i, j);
            src->GotoFrame((startFr + endFr) / 2);
            Array2dVec3UInt8* wrapper = Core::Array::ArrayCreate<Array2dVec3UInt8>
                (src->FrameWidth(), src->FrameHeight(), 0, 0, src->DataPtr(),
                 true);
            int frameNr = keyframes->GetFrameNr(curKey);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            String fName = dVidSet->GetFilePathFrames(curKey, frameNr, ".png",
                                                      true, false);
            if (fName.empty())
            {
                ILOG_ERROR("Cannot write keyframe");
            }
            else
            {
                Core::Array::WritePng(wrapper, fName, dVidSet->GetDatabase());
            }
#else // REPOSITORY_USED
            String container = dVidSet->GetContainerFile(curKey);
            Persistency::FrameLocator loc(dVidSet->GetLocator(), container,
                                          frameNr, ".png");
            Persistency::FrameRepository().Add(loc, wrapper);
#endif // REPOSITORY_USED
            delete wrapper;
            delete src;

            if (!doVideo)
                continue;

            // get video clip
            String exeName("mencoder -oac copy -ovc copy -of mpeg");
            Impala::Util::StringParser pStart(topic.GetVideoExampleStart(i, j));
            int startM = pStart.GetInt('m');
            pStart.AdvanceP();
            int startS = pStart.GetInt('.');
            pStart.AdvanceP();
            int startF = pStart.GetInt('s');
            String duration = topic.GetVideoExampleDuration(i, j);
            String vidFileName = sVidSet->GetResolvedVideoPath(vidId);
            char sysBuf[1024];
            sprintf(sysBuf, "%s %s -o %s -ss %d:%d.%d -endpos %s",
                    exeName.c_str(), vidFileName.c_str(),
                    topic.GetVideoExampleName(i, j, true).c_str(),
                    startM, startS, startF, duration.c_str());

            for (int c=0 ; c<1024 ; c++)
                if (sysBuf[c] == '/')
                    sysBuf[c] = '\\';
                
            ILOG_DEBUG("sysbuf = [" << sysBuf << "]");
            int sysRes = system(sysBuf);
            if (sysRes != 0)
            {
                ILOG_ERROR("system ["  << sysBuf << "]");
                return;
            }
        }
    }
    delete sVidSet;
    delete dVidSet;
}

void
DoTrecSet()
{
    ILOG_VAR(Impala.Application.Util.DoTrecSet);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need at least 4 arguments");
        return;
    }
    String setName = options.GetArg(1);
    String use = options.GetArg(2);
    String collName = options.GetArg(3);
    String clipsKeys("");
    if (options.GetNrArg() > 4)
        clipsKeys = options.GetArg(4);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    Core::Trec::Collection coll(collName, &Impala::Util::Database::GetInstance());
#else // REPOSITORY_USED
    Persistency::FileLocator loc(collName);
    Core::Trec::Collection coll(loc);
#endif // REPOSITORY_USED
    //coll.WriteTo(std::cout);
    coll.WriteVideoSet(setName, use, clipsKeys);
}

void
DoTrecJudge()
{
#ifndef REPOSITORY_TODO
    ILOG_VAR(Impala.Application.Util.DoTrecJudge);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need at least 4 arguments");
        return;
    }
    String judgeFile = options.GetArg(1);
    String shotFile = options.GetArg(2);
    String topic = options.GetArg(3);
    Core::Trec::SearchJudge judge(judgeFile);
    judge.Stats();
    std::vector<String> shots;
    Impala::Util::Database* db = &Impala::Util::Database::GetInstance();
    Impala::Util::DatabaseReadStrings(shots, shotFile, db);

    Real64 infAP = judge.Judge(topic, shots);
    std::cout << "infAP = " << infAP << std::endl;
#endif
}

void
DoDumpStills()
{
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Stills Stills;

    ILOG_VAR(Impala.Application.Util.DoDumpStills);
    CmdOptions& options = CmdOptions::GetInstance();
    String fName = options.GetArg(1);
    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(fName);
    Stills stills(vidSet, "stills");
    ILOG_INFO("nrVideos = " << stills.GetNrVideos());
    ILOG_INFO("nrShots = " << stills.GetNrShots());
    ILOG_INFO("nrStills = " << stills.GetNrStills());
    //stills.DumpGroups();
    stills.Dump(options.GetInt("start"), options.GetInt("end"));
}

void
DoExportShotMpeg7()
{
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::VideoSet::Mpeg7Doc Mpeg7Doc;

    ILOG_VAR(Impala.Application.Util.DoExportShotMpeg7);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String videoSetName = options.GetArg(1);
    int fractions = Impala::atol(options.GetArg(2));
    if ((fractions != 25) && (fractions != 30000))
        ILOG_WARN("Never tried fractions = " << fractions);
    bool generateShotNames = Impala::atol(options.GetArg(3));
    bool withKeyframes = Impala::atol(options.GetArg(4));

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(videoSetName);
    SegmentationLocator sLoc(videoSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(videoSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
    for (int v=0 ; v<vidSet->NrFiles() ; v++)
    {
        String fileName = vidSet->GetFile(v);
        ILOG_INFO("doing file " << fileName);
        String id = fileName;
        int nrFrames = segmentation->GetNrFramesVideo(v);
        Mpeg7Doc mp7(id, fileName, nrFrames, fractions, withKeyframes, "", "");
        for (int s=0 ; s<segmentation->GetNrShotsVideo(v) ; s++)
        {
            int shot = segmentation->GetFirstShotVideo(v) + s;
            String shotName = (generateShotNames)
                ? "shot" + MakeString(v+1) + "_" + MakeString(s+1)
                : segmentation->GetName(shot);
            if (withKeyframes)
            {
                std::vector<String> names;
                std::vector<int> frames;
                for (int k=0 ; k<keyframes->GetNrKeyframesShot(shot) ; k++)
                {
                    int key = keyframes->GetFirstKeyframeShot(shot) + k;
                    String keyName = (generateShotNames)
                        ? "shot"+MakeString(v+1)+"_"+MakeString(s+1)+"_RKF"
                        : Impala::FileNameBase(keyframes->GetName(key));
                    names.push_back(keyName);
                    frames.push_back(keyframes->GetFrameNr(key));
                }
                mp7.AddShot(shotName, segmentation->GetStart(shot),
                            segmentation->GetEnd(shot), names, frames);
            }
            else
            {
                mp7.AddShot(shotName, segmentation->GetStart(shot),
                            segmentation->GetEnd(shot));
            }
        }
        Core::VideoSet::Mpeg7DocWrite(&mp7, vidSet, v);
    }
}

/* Export .truth files as Mpeg7 files. */
void
DoExportAnnoMpeg7()
{
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::VideoSet::Mpeg7Doc Mpeg7Doc;

    ILOG_VAR(Impala.Application.Util.DoExportAnnoMpeg7);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String videoSetName = options.GetArg(1);
    int fractions = Impala::atol(options.GetArg(2));
    if ((fractions != 25) && (fractions != 30000))
        ILOG_WARN("Never tried fractions = " << fractions);
    String conceptSet = options.GetArg(3);
    String anno = options.GetArg(4);
    bool hasKeyframes = false;
    if (anno == "hasKeyframes")
    {
        hasKeyframes = true;
    }
    else if (anno != "hasShots")
    {
        ILOG_ERROR("Argument should be hasShots or hasKeyframes");
        return;
    }

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(videoSetName);
    SegmentationLocator sLoc(videoSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(videoSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    String fName = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (fName.empty())
    {
        ILOG_ERROR("unable to read conceptSet");
        return;
    }
    std::vector<String> concepts;
    Impala::Util::Database* db = vidSet->GetDatabase();
    Impala::Util::DatabaseReadStrings(concepts, fName, db);
#else // REPOSITORY_USED
    Persistency::KeywordListLocator loc(vidSet->GetLocator(), conceptSet);
    Core::Table::KeywordList concepts =
        *(Persistency::KeywordListRepository().Get(loc));
#endif // REPOSITORY_USED
    for (int c=0 ; c<concepts.size() ; c++)
    {
        ILOG_INFO("doing concept " << concepts[c]);
        ILOG_NDC_PUSH(concepts[c]);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        fName = vidSet->GetFilePathGroundTruth(concepts[c] + ".truth", false,
                                               false);
        if (fName.empty())
        {
            ILOG_INFO("no annotations found for " << concepts[c]);
            ILOG_NDC_POP;
            continue;
        }
        std::vector<String> annos;
        Impala::Util::DatabaseReadStrings(annos, fName, db);
#else // REPOSITORY_USED
        String fName = "Annotations/groundtruth/" + concepts[c] + ".truth";
        Persistency::FileLocator fLoc(vidSet->GetLocator(), fName);
        typedef Persistency::RepositoryInFileSystem FS;
        Persistency::File file = FS::GetInstance().GetFile(fLoc, false, false);
        std::vector<String> annos;
        file.ReadStrings(annos);
#endif // REPOSITORY_USED
        std::sort(annos.begin(), annos.end());
        for (int v=0 ; v<vidSet->NrFiles() ; v++)
        {
            String vidFile = vidSet->GetFile(v);
            int nrFrames = segmentation->GetNrFramesVideo(v);
            Mpeg7Doc mp7(vidFile, vidFile, nrFrames, fractions, false,
                         conceptSet, concepts[c]);
            if (hasKeyframes)
            {
                for (int k=0 ; k<keyframes->GetNrKeyframesVideo(v) ; k++)
                {
                    int keyId = keyframes->GetFirstKeyframeVideo(v) + k;
                    String keyName = FileNameBase(keyframes->GetName(keyId));
                    int frameNr = keyframes->GetFrameNr(keyId);
                    double relevance = 0;
                    if (find(annos.begin(),annos.end(),keyName) != annos.end())
                        relevance = 1;
                    mp7.AddAnnotation(concepts[c], keyName, frameNr, frameNr,
                                      relevance, 1);
                }
            }
            else
            {
                for (int s=0 ; s<segmentation->GetNrShotsVideo(v) ; s++)
                {
                    int shotId = segmentation->GetFirstShotVideo(v) + s;
                    String shotName = segmentation->GetName(shotId);
                    double relevance = 0;
                    if (find(annos.begin(),annos.end(),shotName) != annos.end())
                        relevance = 1;
                    int firstKey = keyframes->GetFirstKeyframeShot(shotId);
                    int nrKey = keyframes->GetNrKeyframesShot(shotId);
                    for (int k=firstKey ; k<firstKey+nrKey ; k++)
                    {
                        String keyName = FileNameBase(keyframes->GetName(k));
                        int frameNr = keyframes->GetFrameNr(k);
                        mp7.AddAnnotation(concepts[c], keyName, frameNr,
                                          frameNr, relevance, 1);
                    }
                }
            }
            Core::VideoSet::Mpeg7DocWrite(&mp7, vidSet, v);
        }
        ILOG_NDC_POP;
    }
}

/* import .ann files (from IMAG).
One line looks like this:

RYY MCG-ICT-CAS Airplane BG_12460 TRECVID2007_1/shot1_100_RKF N [S]
==
Annotation_Person Organizer Concept Videoname Keyframename Judgment(Positive/Negative) Sign(S)

The optional S indicates that the annotator was not sure whether it is Pos/Neg
*/
void
DoImportAnnotations()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::VideoSet::Mpeg7Doc Mpeg7Doc;

    ILOG_VAR(Impala.Application.Util.DoImportAnnotations);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String videoSetName = options.GetArg(1);
    int fractions = Impala::atol(options.GetArg(2));
    if ((fractions != 25) && (fractions != 30000))
        ILOG_WARN("Never tried fractions = " << fractions);
    String conceptSet = options.GetArg(3);

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(videoSetName);
    SegmentationLocator sLoc(videoSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(videoSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
    std::vector<String> concepts;
    Impala::Util::Database* db = vidSet->GetDatabase();
    Impala::Util::DatabaseReadStrings(concepts, conceptSet, db);
    String path = FileNamePath(conceptSet) + "/";
    conceptSet = FileNameTail(conceptSet);
    for (int c=0 ; c<concepts.size() ; c++)
    {
        ILOG_INFO("doing concept " << concepts[c]);
        ILOG_NDC_PUSH(concepts[c]);
        String fName = path + concepts[c] + ".ann";
        std::vector<String> lines;
        Impala::Util::DatabaseReadStrings(lines, fName, db);
        int curVidId = -1;
        Mpeg7Doc* curDoc = 0;
        for (int l=0 ; l<lines.size() ; l++)
        {
            Impala::Util::StringParser p(lines[l]);
            String s = p.GetString(' '); // Person
            s = p.GetString(' '); // Organizer
            s = p.GetString(' '); // Concept
            s = p.GetString(' '); // Videoname
            s = Impala::FileNameTail(p.GetString(' ')); // Keyframename
            int key = keyframes->GetFrameId(s + ".jpg");
            if (key == -1)
            {
                ILOG_ERROR("Unable to find keyframe " << s);
                continue;
            }
            double relevance = (p.GetString(' ', false) == "P") ? 1.0 : 0.0;
            double confidence = 1.0;
            if (!p.TheEnd() && (p.GetString(' ', false) == "S"))
                relevance = 0.5; // open for debate :-)
            
            if (keyframes->GetVideoId(key) != curVidId)
            {   // new video
                if (curDoc)
                {
                    Core::VideoSet::Mpeg7DocWrite(curDoc, vidSet, curVidId);
                    delete curDoc;
                    curDoc = 0;
                }
                curVidId = keyframes->GetVideoId(key);
                String vidFile = vidSet->GetFile(curVidId);
                int nrFrames = segmentation->GetNrFramesVideo(curVidId);
                curDoc = new Mpeg7Doc(vidFile, vidFile, nrFrames, fractions,
                                      false, conceptSet, concepts[c]);
            }
            String name = FileNameBase(keyframes->GetName(key));
            int frameNr = keyframes->GetFrameNr(key);
            curDoc->AddAnnotation(concepts[c], name, frameNr, frameNr,
                                  relevance, confidence);
        }
        if (curDoc)
        {
            Core::VideoSet::Mpeg7DocWrite(curDoc, vidSet, curVidId);
            delete curDoc;
        }
        ILOG_NDC_POP;
    }
#endif
}

/* import .1 (positive shot1_1) and .0 (negative shot1_1) files from Ork
   Files are named after rels.
*/
void
DoImportAnnotationsOrk()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::VideoSet::Mpeg7Doc Mpeg7Doc;

    ILOG_VAR(Impala.Application.Util.DoImportAnnotations);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 6)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String videoSetName = options.GetArg(1);
    int fractions = Impala::atol(options.GetArg(2));
    if ((fractions != 25) && (fractions != 30000))
        ILOG_WARN("Never tried fractions = " << fractions);
    String conceptSet = options.GetArg(3);
    String mappingName = options.GetArg(4);
    String inputPath = options.GetArg(5);

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(videoSetName);
    SegmentationLocator sLoc(videoSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(videoSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
    Impala::Util::Database* db = vidSet->GetDatabase();

    String fName = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (fName.empty())
    {
        ILOG_ERROR("unable to read conceptSet");
        return;
    }
    std::vector<String> concepts;
    Impala::Util::DatabaseReadStrings(concepts, fName, db);

    std::vector<String> lines;
    Impala::Util::SimpleMap<String, String> conceptMapping;
    Impala::Util::DatabaseReadStrings(lines, mappingName, db);
    for (int i=0 ; i<lines.size() ; i++)
    {
        Impala::Util::StringParser p(lines[i]);
        String topic = p.GetString(' ');
        String concept = p.GetString(' ', false);
        std::cout << topic << " = " << concept << std::endl;
        conceptMapping.Add(concept, topic);
    }
    for (int c=0 ; c<concepts.size() ; c++)
    {
        ILOG_INFO("doing concept " << concepts[c]);
        ILOG_NDC_PUSH(concepts[c]);

        String topic;
        if (!conceptMapping.Get(concepts[c], topic))
        {
            ILOG_ERROR("no topic for " << concepts[c]);
            ILOG_NDC_POP;
            continue;
        }

        std::vector<String> posAnnos;
        fName = inputPath + "." + topic + ".1";
        fName = db->GetFilePath(fName, false, true);
        if (!fName.empty())
            Impala::Util::DatabaseReadStrings(posAnnos, fName, db);

        std::vector<String> negAnnos;
        fName = inputPath + "." + topic + ".0";
        fName = db->GetFilePath(fName, false, true);
        if (!fName.empty())
            Impala::Util::DatabaseReadStrings(negAnnos, fName, db);
        if ((posAnnos.size() == 0) || (negAnnos.size() == 0))
        {
            ILOG_ERROR("missing input for " << concepts[c]);
            ILOG_NDC_POP;
            continue;
        }

        std::sort(posAnnos.begin(), posAnnos.end());
        std::sort(negAnnos.begin(), negAnnos.end());
        for (int v=0 ; v<vidSet->NrFiles() ; v++)
        {
            String vidFile = vidSet->GetFile(v);
            int nrFrames = segmentation->GetNrFramesVideo(v);
            Mpeg7Doc mp7(vidFile, vidFile, nrFrames, fractions, false,
                         conceptSet, concepts[c]);

            for (int s=0 ; s<segmentation->GetNrShotsVideo(v) ; s++)
            {
                int shotId = segmentation->GetFirstShotVideo(v) + s;
                String shotName = segmentation->GetName(shotId);
                double relevance = -1;
                if (find(posAnnos.begin(), posAnnos.end(), shotName) !=
                    posAnnos.end())
                {
                    relevance = 1;
                }
                else if (find(negAnnos.begin(), negAnnos.end(), shotName) !=
                         negAnnos.end())
                {
                    relevance = 0;
                }
                if (relevance < 0)
                    continue;
                int key = keyframes->GetShotRKF(shotId);

                String keyName = FileNameBase(keyframes->GetName(key));
                int frameNr = keyframes->GetFrameNr(key);
                mp7.AddAnnotation(concepts[c], keyName, frameNr, frameNr,
                                  relevance, 1);
            }
            if (mp7.HasAnnotations())
                Core::VideoSet::Mpeg7DocWrite(&mp7, vidSet, v);
        }
        ILOG_NDC_POP;
    }
#endif
}

/* Import annotations from active learning (in Cees' format?).
 * Annotations are on RKF's and stored in Annotations/from_active.
 * Positives are stored as shot1_1 in file concept/shots_bookmarked.txt and
 * negatives are stored in concept/shots_bookmarked_negative.txt.
 */
void
DoImportAnnotationsActive()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::VideoSet::Mpeg7Doc Mpeg7Doc;

    ILOG_VAR(Impala.Application.Util.DoImportAnnotationsActive);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String videoSetName = options.GetArg(1);
    int fractions = Impala::atol(options.GetArg(2));
    if ((fractions != 25) && (fractions != 30000))
        ILOG_WARN("Never tried fractions = " << fractions);
    String conceptSet = options.GetArg(3);

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(videoSetName);
    SegmentationLocator sLoc(videoSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(videoSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
    Impala::Util::Database* db = vidSet->GetDatabase();

    String fName = vidSet->GetFilePathAnnotation(conceptSet, false, false);
    if (fName.empty())
    {
        ILOG_ERROR("unable to read conceptSet");
        return;
    }
    std::vector<String> concepts;
    Impala::Util::DatabaseReadStrings(concepts, fName, db);
    for (int c=0 ; c<concepts.size() ; c++)
    {
        ILOG_INFO("doing concept " << concepts[c]);
        ILOG_NDC_PUSH(concepts[c]);

        std::vector<String> posAnnos;
        fName = "from_active/" + concepts[c] + "/shots_bookmarked.txt";
        fName = vidSet->GetFilePathAnnotation(fName, false, true);
        if (!fName.empty())
            Impala::Util::DatabaseReadStrings(posAnnos, fName, db);

        std::vector<String> negAnnos;
        fName = "from_active/" + concepts[c] + "/shots_bookmarked_negative.txt";
        fName = vidSet->GetFilePathAnnotation(fName, false, true);
        if (!fName.empty())
            Impala::Util::DatabaseReadStrings(negAnnos, fName, db);
        if ((posAnnos.size() == 0) || (negAnnos.size() == 0))
        {
            ILOG_ERROR("missing input for " << concepts[c]);
            ILOG_NDC_POP;
            continue;
        }

        std::sort(posAnnos.begin(), posAnnos.end());
        std::sort(negAnnos.begin(), negAnnos.end());
        for (int v=0 ; v<vidSet->NrFiles() ; v++)
        {
            String vidFile = vidSet->GetFile(v);
            int nrFrames = segmentation->GetNrFramesVideo(v);
            Mpeg7Doc mp7(vidFile, vidFile, nrFrames, fractions, false,
                         conceptSet, concepts[c]);

            for (int s=0 ; s<segmentation->GetNrShotsVideo(v) ; s++)
            {
                int shotId = segmentation->GetFirstShotVideo(v) + s;
                String shotName = segmentation->GetName(shotId);
                double relevance = -1;
                if (find(posAnnos.begin(), posAnnos.end(), shotName) !=
                    posAnnos.end())
                {
                    relevance = 1;
                }
                else if (find(negAnnos.begin(), negAnnos.end(), shotName) !=
                         negAnnos.end())
                {
                    relevance = 0;
                }
                if (relevance < 0)
                    continue;
                int key = keyframes->GetShotRKF(shotId);

                String keyName = FileNameBase(keyframes->GetName(key));
                int frameNr = keyframes->GetFrameNr(key);
                mp7.AddAnnotation(concepts[c], keyName, frameNr, frameNr,
                                  relevance, 1);
            }
            if (mp7.HasAnnotations())
                Core::VideoSet::Mpeg7DocWrite(&mp7, vidSet, v);
        }
        ILOG_NDC_POP;
    }
#endif
}

void
DoExportConceptMpeg7()
{
    ILOG_VAR(Impala.Application.Util.DoExportConceptMpeg7);
    ILOG_ERROR("Deprecated : use vidset exportconcepts");
}

void
DoDumpParameter()
{
#ifndef REPOSITORY_TODO
    typedef Core::Database::RawDataSet RawDataSet;
    typedef Core::Feature::FeatureDefinition FeatureDef;

    ILOG_VAR(Impala.Application.Util.DoDumpParameter);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String dataSetName = options.GetArg(1);
    String conceptSet = options.GetArg(2);
    String model = options.GetArg(3);
    FeatureDef featureDef(options.GetArg(4));

    RawDataSet* dataSet = Core::Database::MakeRawDataSet(dataSetName);

    String fName = dataSet->GetFilePathAnnotation(conceptSet, false, false);
    if (fName.empty())
    {
        ILOG_ERROR("unable to read conceptSet " << conceptSet);
        return;
    }
    std::vector<String> concepts;
    Impala::Util::Database* db = dataSet->GetDatabase();
    Impala::Util::DatabaseReadStrings(concepts, fName, db);
    Real64 map = 0;
    int nrMap = 0;
    std::cout << conceptSet << "  " << model << "  " << featureDef.AsString()
              << std::endl;
    for (int c=0 ; c<concepts.size() ; c++)
    {
        String con = concepts[c];
        fName = dataSet->GetFilePathConceptModel(conceptSet, model, featureDef,
                                                 con + ".best", false, false);
        if (fName.empty())
        {
            ILOG_INFO("no .best found for " << con);
            continue;
        }
        Impala::Util::IOBuffer* ioBuf = db->GetIOBuffer(fName, true, false, "");
        String buffer = ioBuf->ReadLine();
        Impala::Util::StringParser p(buffer);
        p.Eat(':');
        Real64 ap = p.GetDouble();
        std::cout << con << " " << ap << std::endl;
        map += ap;
        nrMap++;
    }
    std::cout << "map = " << map / nrMap << std::endl << std::endl;
#endif
}

void
DoAPConfusion()
{
#ifndef REPOSITORY_TODO
    typedef Core::Database::RawDataSet RawDataSet;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::AnnotationTable AnnotationTable;
    using Impala::Core::Table::AveragePrecision;

    ILOG_VAR(Impala.Application.Util.DoAPConfusion);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String dataSetName = options.GetArg(1);
    String annoSet = options.GetArg(2);
    String namesPath = options.GetArg(3);
    String mappingName;
    if (options.GetNrArg() > 4)
        mappingName = options.GetArg(4);

    RawDataSet* dataSet = Core::Database::MakeRawDataSet(dataSetName);
    Impala::Util::Database* db = dataSet->GetDatabase();
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(
                     db->GetFilePath(".", namesPath, false, false), db);

    Impala::Util::SimpleMap<String, String> conceptMapping;
    if (!mappingName.empty())
    {
        std::vector<String> lines;
        Impala::Util::DatabaseReadStrings(lines, mappingName, db);
        for (int i=0 ; i<lines.size() ; i++)
        {
            Impala::Util::StringParser p(lines[i]);
            String src = p.GetString(' ');
            String dst = p.GetString(' ', false);
            std::cout << src << " => " << dst << std::endl;
            conceptMapping.Add(src, dst);
        }
    }

    Core::Table::QuidTable* qTable = simSet->GetQuidTable();
    int qClass = QuidClass(qTable->Get1(0));
    Real64 map = 0;
    Real64 mapJ = 0;
    Real64 map2000 = 0;
    Real64 map2000J = 0;
    bool do2000 = (simSet->TableSize() > 2000);
    int nrMap = 0;
    std::cout << namesPath << ", size = " << simSet->TableSize() << std::endl;

    AnnotationTable** allAnnotations = new AnnotationTable*[simSet->NrTables()];

    for (int c=0; c<simSet->NrTables() ; c++){
        String concept = simSet->GetName(c);
        String mappedConcept;
        if (!mappingName.empty())
        {
            if (conceptMapping.Get(concept, mappedConcept))
            {
                concept = mappedConcept;
            }
            else
            {
                continue;
            }
        }
        String fName = dataSet->GetFilePathAnnotation(qClass, annoSet, 
                                                      concept + ".tab",
                                                      false, false);

        if (fName.empty())
        {
            ILOG_ERROR("No groundtruth for " << concept);
            continue;
        }

        allAnnotations[c] = AnnotationTable::MakeFromFile(concept, fName, db);
        allAnnotations[c]->UpdateQuidMap();
    }

    std::cout << "\nConfusion Matrix Average Precision" << std::endl;
    for (int c=0 ; c<simSet->NrTables() ; c++)
    {
        Real64* confEntry = AveragePrecisionConfusion(simSet->GetRankTable(c), allAnnotations,
                                                      c, simSet->NrTables(), -1);

        String concept = simSet->GetName(c);
        std::cout << MakeString(concept, 25) << "\t";

        Real64 total = 0;
        for (int i=0; i<simSet->NrTables() ; i++)
        {
            total += confEntry[i];
            std::cout << confEntry[i] << "\t";
        }

        std::cout << total << std::endl;
    }
 
    std::cout << "\nConfusion Matrix Average Precision Judged" << std::endl;
    for (int c=0 ; c<simSet->NrTables() ; c++)
    {
        Real64* confEntry = AveragePrecisionJudgedConfusion(simSet->GetRankTable(c), allAnnotations,
                                                      c, simSet->NrTables(), -1);

        String concept = simSet->GetName(c);
        std::cout << MakeString(concept, 25) << "\t ";

        Real64 total = 0;
        for (int i=0; i<simSet->NrTables() ; i++)
        {
            total += confEntry[i];
            std::cout << confEntry[i] << "\t ";
        }

        std::cout << total << std::endl;
    }

    for (int i=0; i < simSet->NrTables() ; i++)
        delete allAnnotations[i];
    delete allAnnotations;
    delete simSet;
    delete dataSet;
#endif
}

void
DoScoreConcepts()
{
#ifndef REPOSITORY_TODO
    typedef Core::Database::RawDataSet RawDataSet;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::AnnotationTable AnnotationTable;
    using Impala::Core::Table::AveragePrecision;

    ILOG_VAR(Impala.Application.Util.DoScoreConcept);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 4)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String dataSetName = options.GetArg(1);
    String annoSet = options.GetArg(2);
    String namesPath = options.GetArg(3);
    String mappingName;
    if (options.GetNrArg() > 4)
        mappingName = options.GetArg(4);

    RawDataSet* dataSet = Core::Database::MakeRawDataSet(dataSetName);
    Impala::Util::Database* db = dataSet->GetDatabase();
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(
                     db->GetFilePath(".", namesPath, false, false), db);

    Impala::Util::SimpleMap<String, String> conceptMapping;
    if (!mappingName.empty())
    {
        std::vector<String> lines;
        Impala::Util::DatabaseReadStrings(lines, mappingName, db);
        for (int i=0 ; i<lines.size() ; i++)
        {
            Impala::Util::StringParser p(lines[i]);
            String src = p.GetString(' ');
            String dst = p.GetString(' ', false);
            std::cout << src << " => " << dst << std::endl;
            conceptMapping.Add(src, dst);
        }
    }

    Core::Table::QuidTable* qTable = simSet->GetQuidTable();
    int qClass = QuidClass(qTable->Get1(0));
    Real64 map = 0;
    Real64 mapJ = 0;
    Real64 map2000 = 0;
    Real64 map2000J = 0;
    bool do2000 = (simSet->TableSize() > 2000);
    int nrMap = 0;
    std::cout << namesPath << ", size = " << simSet->TableSize() << std::endl;
    for (int c=0 ; c<simSet->NrTables() ; c++)
    {
        String concept = simSet->GetName(c);
        String mappedConcept;
        if (!mappingName.empty())
        {
            if (conceptMapping.Get(concept, mappedConcept))
            {
                concept = mappedConcept;
            }
            else
            {
                continue;
            }
        }
        String fName = dataSet->GetFilePathAnnotation(qClass, annoSet,
                                                      concept + ".tab",
                                                      false, false);
        if (fName.empty())
        {
            ILOG_ERROR("No groundtruth for " << concept);
            continue;
        }
        AnnotationTable* truth = AnnotationTable::MakeFromFile(concept, fName,
                                                               db);
        truth->UpdateQuidMap();
        Real64 ap = AveragePrecision(simSet->GetRankTable(c), truth, -1);
        Real64 apJ = AveragePrecisionJudged(simSet->GetRankTable(c), truth,
                                            false, -1);
        map += ap;
        mapJ += apJ;
        std::cout << MakeString(concept, 30)
                  << " " << ap << "\t j: " << apJ;

        if (do2000)
        {
            Real64 ap2000 = AveragePrecision(simSet->GetRankTable(c), truth,
                                             2000);
            Real64 ap2000J = AveragePrecisionJudged(simSet->GetRankTable(c),
                                                    truth, false, 2000);
            map2000 += ap2000;
            map2000J += ap2000J;
            std::cout << "\t " << ap2000 << "\t j: " << ap2000J;
        }

        std::cout << std::endl;
        nrMap++;
        delete truth;
    }
    std::cout << "map=" << map/nrMap << ", mapJudged=" << mapJ/nrMap;
    if (do2000)
        std::cout << " map2000=" << map2000/nrMap << ", map2000Judged="
                  << map2000J/nrMap;
    std::cout << std::endl << std::endl;
    delete simSet;
    delete dataSet;
#endif
}

void
DoScoreConceptsVideo()
{
#ifndef REPOSITORY_TODO
    typedef Core::Database::RawDataSet RawDataSet;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::RankTableType RankTableType;
    typedef Core::Table::AnnotationTable AnnotationTable;
    using Impala::Core::Table::AveragePrecision;

    ILOG_VAR(Impala.Application.Util.DoScoreConceptVideo);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String dataSetName = options.GetArg(1);
    String annoSet = options.GetArg(2);
    String namesPath = options.GetArg(3);
    int vidStart = atol(options.GetArg(4));
    int vidEnd = atol(options.GetArg(5));

    RawDataSet* dataSet = Core::Database::MakeRawDataSet(dataSetName);
    Impala::Util::Database* db = dataSet->GetDatabase();
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(namesPath, db);

    Core::Table::QuidTable* qTable = simSet->GetQuidTable();
    int qClass = QuidClass(qTable->Get1(0));
    Real64 mapJ = 0;
    int nrMap = 0;
    std::cout << namesPath << ", size = " << simSet->TableSize() << std::endl;
    std::vector<int> videos;
    std::cout << "videos: ";
    for (int i=vidStart ; i<vidEnd ; i++)
    {
        videos.push_back(i);
        std::cout << i << " ";
    }
    std::cout << std::endl;
    for (int c=0 ; c<simSet->NrTables() ; c++)
    {
        String concept = simSet->GetName(c);
        String fName = dataSet->GetFilePathAnnotation(qClass, annoSet,
                                                      concept + ".tab",
                                                      false, false);
        if (fName.empty())
        {
            ILOG_ERROR("No groundtruth for " << concept);
            continue;
        }
        AnnotationTable* truth = AnnotationTable::MakeFromFile(concept, fName,
                                                               db);
        truth->UpdateQuidMap();
        RankTableType* rank = simSet->GetRankTable(c);
        if (videos.size() > 0)
        {
            Core::Table::CriterionQuidObjectInSet<RankTableType> crit(videos);
            Core::Table::Select(rank, rank, crit, true);
        }
        Real64 apJ = AveragePrecisionJudged(rank, truth, false, -1, true);
        mapJ += apJ;
        std::cout << MakeString(concept, 25) << " " << apJ << std::endl;
        nrMap++;
        delete truth;
    }
    std::cout << "mapJudged=" << mapJ/nrMap << std::endl;
    std::cout << std::endl;
    delete simSet;
    delete dataSet;
#endif
}

void
DoCombineConcepts()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Application.Util.DoCombineConcepts);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 7)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String vidSetName = options.GetArg(1);
    String conceptSet = options.GetArg(2);
    String model = options.GetArg(3);
    String resultName = options.GetArg(4);
    String combi = options.GetArg(5);
    if ((combi != "mul") && (combi != "avg"))
    {
        ILOG_ERROR("Unknown combination method : " << combi);
        return;
    }
    std::vector<String> featureDefs;
    for (int i=6 ; i<options.GetNrArg() ; i++)
        featureDefs.push_back(options.GetArg(i));

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);

    SimilarityTableSet* combinedSet = SimilarityTableSet::MakeFromFile
        (vidSet, conceptSet, model, featureDefs[0]);
    for (int i=1 ; i<featureDefs.size() ; i++)
    {
        SimilarityTableSet* simSet = SimilarityTableSet::MakeFromFile
            (vidSet, conceptSet, model, featureDefs[i]);
        for (int t=0 ; t<simSet->NrTables() ; t++)
        {
            if (combi == "mul")
                MulAssign(combinedSet->GetSimTable(t)->GetColumn1(),
                          simSet->GetSimTable(t)->GetColumn1(), 0, -1);
            else
                AddAssign(combinedSet->GetSimTable(t)->GetColumn1(),
                          simSet->GetSimTable(t)->GetColumn1(), 0, -1);
        }
        delete simSet;
    }
    if (combi == "avg")
        for (int t=0 ; t<combinedSet->NrTables() ; t++)
            DivAssign(combinedSet->GetSimTable(t)->GetColumn1(),
                      featureDefs.size(), 0, -1);
    combinedSet->ComputeRanks(true);
    combinedSet->Save(vidSet, conceptSet, model, resultName, true);
    delete combinedSet;
    delete vidSet;
#endif
}

void
DoMapConcepts2Shots()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::SimTableType SimTableType;

    ILOG_VAR(Impala.Application.Util.DoMapConcepts2Shots);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String vidSetName = options.GetArg(1);
    String conceptSet = options.GetArg(2);
    String model = options.GetArg(3);
    String feature = options.GetArg(4);
    String aggregationMode = "max";
    if (options.GetNrArg() >= 6)
    {
        aggregationMode = options.GetArg(5);
    }

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    SegmentationLocator sLoc(vidSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(vidSet, conceptSet,
                                                      model, feature);

    Core::Table::QuidTable* frameQuids = simSet->GetQuidTable();
    int nrShots = segmentation->GetNrShots();
    SimilarityTableSet* res = new SimilarityTableSet(simSet->GetNames(), nrShots);
    Core::Table::QuidTable* shotQuids = res->GetQuidTable();
//    for (int i=0 ; i<nrShots ; i++)
//    {
//        Quid q = segmentation->GetQuidShot(i);
//        shotQuids->Add(q);
//    }
    for (int t=0 ; t<simSet->NrTables() ; t++)
    {
        SimTableType* frameSimTable = simSet->GetSimTable(t);
        SimTableType* shotSimTable = res->GetSimTable(t);
        int lastShot = -1;
        Real64 lastSim = 0;
        int lastCount = 0;
        for (int i=0 ; i<frameSimTable->Size() ; i++)
        {
            Quid frameQ = frameQuids->Get1(i);
            Real64 frameSim = frameSimTable->Get1(i);
            int shot = segmentation->GetShotId(frameQ);
            if (shot != lastShot)
            {
                if (lastShot != -1)
                {
                    Quid check = segmentation->GetQuidShot(lastShot);
                    int rank = shotSimTable->Size();
                    if (shotQuids->Size() <= rank)
                        shotQuids->Add(check);
                    if (check != shotQuids->Get1(rank))
                    {
                        ILOG_ERROR("Quids do not match");
                    }
                    if(aggregationMode == "avg")
                    {
                        lastSim = lastSim / lastCount;
                    }
                    shotSimTable->Add(lastSim);
                }
                lastSim = frameSim;
                lastShot = shot;
                lastCount = 1;
            }
            else
            {
                lastCount++;
                if(aggregationMode == "max")
                {
                    if (frameSim > lastSim)
                        lastSim = frameSim;
                }
                else if(aggregationMode == "avg")
                {
                    lastSim += frameSim;
                }
            }
        }
        if (lastShot != -1)
        {
            Quid check = segmentation->GetQuidShot(lastShot);
            int rank = shotSimTable->Size();
            if (shotQuids->Size() <= rank)
                shotQuids->Add(check);
            if (check != shotQuids->Get1(rank))
            {
                ILOG_ERROR("Quids do not match");
            }
            if(aggregationMode == "avg")
            {
                lastSim = lastSim / lastCount;
            }
            shotSimTable->Add(lastSim);
        }
    }

    res->ComputeRanks(true);
    res->Save(vidSet, conceptSet, model, "shot" + feature, true);
    delete res;
    delete simSet;
    delete vidSet;
#endif
}

void
DoConcepts2TrecResult()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::RankTableType RankTableType;

    ILOG_VAR(Impala.Application.Util.DoConcepts2TrecResult);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 3)
    {
        ILOG_ERROR("Need more parameters");
        return;
    }
    String vidSetName = options.GetArg(1);
    String namesPath = options.GetArg(2);

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    SegmentationLocator sLoc(vidSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(vidSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
    Impala::Util::Database* db = vidSet->GetDatabase();
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(namesPath, db);

    for (int c=0 ; c<simSet->NrTables() ; c++)
    {
        String concept = simSet->GetName(c);
        ILOG_INFO("Doing concept " << concept);
        std::vector<String> res;
        RankTableType* rTable = simSet->GetRankTable(c);
        for (int i=0 ; i<rTable->Size() ; i++)
        {
            Quid quid = rTable->Get1(i);
            int shot = -1;
            if (QuidClass(quid) == QUID_CLASS_FRAME) // assume it is a keyframe
            {
                int keyfr = keyframes->GetFrameId(quid);
                if (keyfr != -1)
                    shot = keyframes->GetShotId(keyfr);
            }
            if (QuidClass(quid) == QUID_CLASS_SHOT)
            {
                shot = segmentation->GetShotId(quid);
            }
            if (shot == -1)
            {
                ILOG_ERROR("Unable to find shot for " << QuidObj(quid));
                continue;
            }
            String name = segmentation->GetName(shot);
            if (std::find(res.begin(), res.end(), name) == res.end())
                res.push_back(name);
        }
        String fName = concept + ".txt";
        Impala::Util::DatabaseWriteString(fName, db, res.begin(), res.end());
    }
    delete simSet;
    delete vidSet;
#endif
}

void
DoJudgeConcepts()
{
#ifndef REPOSITORY_TODO
    typedef Core::VideoSet::VideoSet VideoSet;
    typedef Core::VideoSet::Segmentation Segmentation;
    typedef Core::VideoSet::Keyframes Keyframes;
    typedef Core::Table::SimilarityTableSet SimilarityTableSet;
    typedef Core::Table::SimilarityTableSet::RankTableType RankTableType;

    ILOG_VAR(Impala.Application.Util.DoJudgeConcepts);
    CmdOptions& options = CmdOptions::GetInstance();
    if (options.GetNrArg() < 5)
    {
        ILOG_ERROR("Need at least 5 arguments");
        return;
    }
    String vidSetName = options.GetArg(1);
    String namesPath = options.GetArg(2);
    String mappingName = options.GetArg(3);
    String judgeFile = options.GetArg(4);

    VideoSet* vidSet = Core::VideoSet::MakeVideoSet(vidSetName);
    SegmentationLocator sLoc(vidSetName, "segmentation", options);
    Segmentation* segmentation = SegmentationRepository().Get(sLoc, vidSet);
    KeyframesLocator kLoc(vidSetName, "keyframes", options);
    Keyframes* keyframes = KeyframesRepository().Get(kLoc, vidSet);
    Impala::Util::Database* db = vidSet->GetDatabase();
    SimilarityTableSet* simSet =
        Core::Table::SimilarityTableSet::MakeFromFile(namesPath, db);

    std::vector<String> lines;
    Impala::Util::SimpleMap<String, String> conceptMapping;
    Impala::Util::DatabaseReadStrings(lines, mappingName, db);
    for (int i=0 ; i<lines.size() ; i++)
    {
        Impala::Util::StringParser p(lines[i]);
        String topic = p.GetString(' ');
        String concept = p.GetString(' ', false);
        std::cout << topic << " = " << concept << std::endl;
        conceptMapping.Add(concept, topic);
    }
    Core::Trec::SearchJudge judge(judgeFile);
    judge.Stats();

    Real64 miap = 0;
    int nrMiap = 0;
    std::cout << namesPath << ", size = " << simSet->TableSize() << std::endl;
    for (int c=0 ; c<simSet->NrTables() ; c++)
    {
        String concept = simSet->GetName(c);
        ILOG_DEBUG("Doing concept " << concept);
        std::vector<String> res;
        RankTableType* rTable = simSet->GetRankTable(c);
        for (int i=0 ; i<rTable->Size() ; i++)
        {
            Quid quid = rTable->Get1(i);
            int shot = -1;
            if (QuidClass(quid) == QUID_CLASS_FRAME) // assume it is a keyframe
            {
                int keyfr = keyframes->GetFrameId(quid);
                if (keyfr != -1)
                    shot = keyframes->GetShotId(keyfr);
            }
            if (QuidClass(quid) == QUID_CLASS_SHOT)
            {
                shot = segmentation->GetShotId(quid);
            }
            if (shot == -1)
            {
                ILOG_ERROR("Unable to find shot for " << QuidObj(quid));
                continue;
            }
            String name = segmentation->GetName(shot);
            if (std::find(res.begin(), res.end(), name) == res.end())
                res.push_back(name);
            if (res.size() >= 2000)
                break;
        }
        String topic;
        if (!conceptMapping.Get(concept, topic))
        {
            ILOG_ERROR("No topic for " << concept);
            continue;
        }
        if (!judge.HasTopic(topic))
        {
            ILOG_INFO("No judge for " << topic << " = " << concept);
            continue;
        }
        Real64 infAP = judge.Judge(topic, res);
        std::cout << concept << " " << infAP << std::endl;
        miap += infAP;
        nrMiap++;
    }
    std::cout << "miap = " << miap / nrMiap << std::endl << std::endl;
    delete simSet;
    delete vidSet;
#endif
}

void
DoPing()
{
#ifndef REPOSITORY_TODO
    typedef Impala::Util::Channel Channel;

    ILOG_VAR(Impala.Application.Util.DoPing);
    Channel* channel = Impala::Util::Database::GetInstance().GetDataChannel();
    if (! (channel && channel->Valid()))
    {
        ILOG_ERROR("No server, use --dataServer to specify server");
        return;
    }
    char* buf = channel->Buffer();
    sprintf(buf, "ping\0");
    channel->SendRequest(strlen(buf)+1);
    if (channel->LastSendHadError())
    {
        ILOG_ERROR("Failed to send message");
        return;
    }
    ILOG_INFO("Server OK");
#endif
}

int
mainUtil(int argc, char* argv[])
{
    CmdOptions& options = CmdOptions::GetInstance();
    options.Initialise(false, false, true);
    options.AddOption(0, "start", "nr", "0");
    options.AddOption(0, "end", "nr", "-1");
    String usageStr = "cmd = \n\n";
    usageStr += "  trectopicdata ...\n";
    usageStr += "  trecset year {devel|test} collection.xml\n";
    usageStr += "  trecjudge qrel.txt shotresult.txt topiccode\n";
    usageStr += "  dumpstills videoset.txt\n";
    usageStr += "  dumpkeyframes videoset.txt\n";
    usageStr += "  dumpsegmentation videoset.txt\n";
    usageStr += "  exportshotmpeg7 videoset.txt {25|30000} generateShotNames withKeyframes\n";
    usageStr += "  exportannompeg7 videoset.txt {25|30000} concepts.txt\n";
    usageStr += "  importannotations videoset.txt {25|30000} concepts.txt\n";
    usageStr += "  exportconceptmpeg7 videoset.txt {25|30000} names.txt\n";
    usageStr += "  dumpparameter videoset.txt concepts.txt model featuredef\n";
    usageStr += "  apconfusion videoset.txt annoset.txt names.txt [mapping]\n";
    usageStr += "  scoreconcepts videoset.txt annoset.txt names.txt [mapping]\n";
    usageStr += "  scoreconceptsvideo videoset.txt annoset.txt names.txt vidStart vidEnd\n";
    usageStr += "  combineconcepts videoset.txt concepts.txt model resultName {avg|mil} {feature}+\n";
    usageStr += "  mapconcepts2shots videoset.txt conceptSet model feature\n";
    usageStr += "  concepts2trecresult videoset.txt names.txt\n";
    usageStr += "  judgeconcepts videoset.txt names.txt conceptMapping.txt qrels\n";
    usageStr += "  ping\n";
    usageStr += "  checknativetypesize\n";
    if (! options.ParseArgs(argc, argv, usageStr, 1))
        return 1;

    ILOG_VAR(Impala.Application.Util.mainUtil);

    String cmd = options.GetArg(0);
    ILOG_DEBUG("cmd = " << cmd);
    if (cmd == "trectopicdata")
        DoTrecTopicData();
    else if (cmd == "trecset")
        DoTrecSet();
    else if (cmd == "trecjudge")
        DoTrecJudge();
    else if (cmd == "dumpstills")
        DoDumpStills();
    else if (cmd == "exportshotmpeg7")
        DoExportShotMpeg7();
    else if (cmd == "exportannompeg7")
        DoExportAnnoMpeg7();
    else if (cmd == "importannotations")
        DoImportAnnotations();
    else if (cmd == "importannotationsork")
        DoImportAnnotationsOrk();
    else if (cmd == "importannotationsactive")
        DoImportAnnotationsActive();
    else if (cmd == "exportconceptmpeg7")
        DoExportConceptMpeg7();
    else if (cmd == "dumpparameter")
        DoDumpParameter();
    else if (cmd == "apconfusion")
        DoAPConfusion();
    else if (cmd == "scoreconcepts")
        DoScoreConcepts();
    else if (cmd == "scoreconceptsvideo")
        DoScoreConceptsVideo();
    else if (cmd == "combineconcepts")
        DoCombineConcepts();
    else if (cmd == "mapconcepts2shots")
        DoMapConcepts2Shots();
    else if (cmd == "concepts2trecresult")
        DoConcepts2TrecResult();
    else if (cmd == "judgeconcepts")
        DoJudgeConcepts();
    else if (cmd == "ping")
        DoPing();
    else if (cmd == "checknativetypesize")
        Impala::Core::Array::Element::CheckNativeTypeSize();
    else ILOG_ERROR("Unknown cmd : " << cmd);

    return 0;
}

} // namespace Util
} // namespace Application
} // namespace Impala

int
main(int argc, char* argv[])
{
    return Impala::Application::Util::mainUtil(argc, argv);
}

---