|| || || || || ||

---

mainRepository.cpp

Go to the documentation of this file.

#include "Persistency/FikSvmRepository.h"
#include "Persistency/AllParameterFileRepository.h"
#include "Persistency/ScoreFileRepository.h"
#include "Persistency/KernelMatrixRepository.h"
#include "Persistency/FoldRepository.h"
#include "Persistency/ImageSetRepository.h"
#include "Persistency/AnnotationTableSetRepository.h"
#include "Persistency/AnnotationTableRepository.h"
#include "Persistency/QuidTableRepository.h"
#include "Persistency/FeatureTableRepository.h"
#include "Persistency/Mpeg7DocRepository.h"
#include "Persistency/ImageArchiveRepository.h"
#include "Persistency/VideoSetRepository.h"
#include "Persistency/SegmentationRepository.h"
#include "Persistency/KeyframesRepository.h"
#include "Persistency/StillsRepository.h"
#include "Persistency/KeywordListRepository.h"
#include "Persistency/SimilarityTableSetRepository.h"
#include "Persistency/BestFileRepository.h"
#include "Persistency/SvmProblemRepository.h"
#include "Persistency/SvmRepository.h"
#include "Persistency/FrameHashesRepository.h"
#include "Core/Table/KeywordListDiff.h"
#include "Core/Training/BestFile.h"
#include "Core/Table/SetQuidSet.h"
#include "Core/Database/MakeRawDataSet.h"
#include "Core/Stream/RgbDataSrc.h"

// since we are not using libraries:
#include "Link/ImpalaLib.cpp"

namespace Impala
{
namespace Application
{
namespace Repository
{


using namespace Persistency;

using Core::Array::ImageArchive;
using Core::Database::RawDataSet;
using Core::ImageSet::ImageSet;
using Core::VideoSet::VideoSet;
using Core::VideoSet::Segmentation;
using Core::VideoSet::Keyframes;
using Core::VideoSet::Stills;
using Core::VideoSet::Mpeg7Doc;
using Core::Table::QuidTable;
using Core::Table::AnnotationTable;
using Core::Table::AnnotationTableSet;
using Core::Table::KeywordList;
using Core::Table::SimilarityTableSet;
using Core::Feature::FeatureTable;
using Core::Training::Svm;
using Core::Training::FikSvm;
using Core::Training::KernelMatrix;
using Core::Stream::RgbDataSrc;

// Convenience functions

RawDataSet*
GetRawDataSet(String locString, String setName, bool required)
{
    ILOG_VAR(Impala.Application.Repository.GetRawDataSet);

    Locator loc(locString, setName);
    RawDataSet* dataSet = 0;
    if (VideoSetsRepository::GetInstance().Contains(setName))
    {
        if (required)
            dataSet = VideoSetRepository().Get(loc);
        else if (VideoSetRepository().Exists(loc))
            dataSet = VideoSetRepository().Get(loc);
    }
    else if (ImageSetsRepository::GetInstance().Contains(setName))
    {
        ImageSetLocator imSetLoc(loc, "");
        if (required)
            dataSet = ImageSetRepository().Get(imSetLoc);
        else if (ImageSetRepository().Exists(imSetLoc))
            dataSet = ImageSetRepository().Get(imSetLoc);
    }
    else if (required)
    {
        ILOG_ERROR("Could not find " << loc);
    }
    return dataSet;
}

RawDataSet*
GetRawDataSet(CmdOptions& options, bool required)
{
    ILOG_VAR(Impala.Application.Repository.GetRawDataSet);

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    return GetRawDataSet(locString, setName, required);
}

ImageSet*
GetImageSet(String locString, String setName, bool required)
{
    ILOG_VAR(Impala.Application.Repository.GetImageSet);

    ImageSetLocator loc(locString, setName);
    ImageSet* imSet = 0;
    if (ImageSetsRepository::GetInstance().Contains(setName))
    {
        if (required)
            imSet = ImageSetRepository().Get(loc);
        else if (ImageSetRepository().Exists(loc))
            imSet = ImageSetRepository().Get(loc);
    }
    else if (required)
    {
        ILOG_ERROR("Could not find " << loc);
    }
    return imSet;
}

ImageSet*
GetImageSet(CmdOptions& options, bool required, bool getSrcLoc = true)
{
    ILOG_VAR(Impala.Application.Repository.GetImageSet);

    int nrArg = 2;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }
    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return GetImageSet(locString, setName, required);
}

VideoSet*
GetVideoSet(String locString, String setName, bool required)
{
    ILOG_VAR(Impala.Application.Repository.GetVideoSet);

    Locator loc(locString, setName);
    VideoSet* vidSet = 0;
    if (VideoSetsRepository::GetInstance().Contains(setName))
    {
        if (required)
            vidSet = VideoSetRepository().Get(loc);
        else if (VideoSetRepository().Exists(loc))
            vidSet = VideoSetRepository().Get(loc);
    }
    else if (required)
    {
        ILOG_ERROR("Could not find " << loc);
    }
    return vidSet;
}

VideoSet*
GetVideoSet(CmdOptions& options, bool required, bool getSrcLoc = true)
{
    ILOG_VAR(Impala.Application.Repository.GetVideoSet);

    int nrArg = 2;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }
    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return GetVideoSet(locString, setName, required);
}

Locator
GetDataSetLocator(CmdOptions& options, bool getSrcLoc = true)
{
    ILOG_VAR(Impala.Application.Repository.GetVideoSet);

    int nrArg = 2;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }
    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return Locator(locString, setName);
}

// ImageSets part

void
DoDumpImageSets()
{
    ImageSetsRepository::GetInstance().Dump();
}

// VideoSets part

void
DoDumpVideoSets()
{
    VideoSetsRepository::GetInstance().Dump();
}

void
DoCopyVideoSets()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyVideoSets);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString = options.GetArg(1);
    Locator dstLoc(options.GetArg(1), "dummy");
    VideoSetsRepository::GetInstance().CopyTo(dstLoc);
}

// ImageSet part

void
DoDumpImageSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpImageSet);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageSet* imSet = GetImageSet(options, true);
    ILOG_INFO("nrDirs = " << imSet->NrDirs());
    ILOG_INFO("nrFiles = " << imSet->NrFiles());
    int minNr = imSet->GetNrFiles(0);
    int maxNr = imSet->GetNrFiles(0);
    for (int i=1 ; i<imSet->NrDirs() ; i++)
    {
        int nr = imSet->GetNrFiles(i);
        if (nr < minNr)
            minNr = nr;
        if (nr > maxNr)
            maxNr = nr;
    }
    ILOG_INFO("minNrFiles/dir = " << minNr << ", maxNrFiles/dir = " << maxNr);
    if (options.GetInt("end") == 0)
        return;
    imSet->Dump(0, -1, options.GetInt("start"), options.GetInt("end"));
    delete imSet;
}

void
DoDiffImageSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffImageSet);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageSet* imSet1 = GetImageSet(options, true);
    ImageSet* imSet2 = GetImageSet(options, true, false);
    imSet1->Diff(imSet2);
    delete imSet2;
    delete imSet1;
}

void
DoCopyImageSet()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyImageSet);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString1 = options.GetArg(1);
    String setName = options.GetArg(2);
    String locString2 = options.GetArg(3);
    ImageSet* imSet = GetImageSet(options, true);
    ImageSetLocator setLoc2(locString2, setName);
    if (options.GetBool("sync"))
        ImageSetRepository().Sync(setLoc2, imSet);
    else
        ImageSetRepository().Add(setLoc2, imSet);
    delete imSet;
}

void
DoDeleteImageSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteImageSet);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString = options.GetArg(1);
    String setName = options.GetArg(2);
    ImageSetLocator loc(locString, setName);
    ImageSetRepository().Delete(loc);
}

void
DoSyncImageFiles()
{
    ILOG_VAR(Impala.Application.Repository.DoSyncImageFiles);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString1 = options.GetArg(1);
    String setName = options.GetArg(2);
    String locString2 = options.GetArg(3);

    ImageSet* imSet = GetImageSet(options, true);
    for (int i=0 ; i<imSet->NrFiles() ; i++)
    {
        String path = FileNameConcat("ImageData", imSet->GetAsPath(i));
        FileLocator srcLoc(locString1, setName, path);
        FileLocator dstLoc(locString2, setName, path);
        RepositoryInFileSystem::GetInstance().CopyFile(srcLoc, dstLoc);
    }
    delete imSet;
}

// ImageSetSpecial part

ImageSetLocator
GetImageSetLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetImageSetLocator);

    int nrArg = 3;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String videoSetName = options.GetArg(a++);
    String imageSetName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return ImageSetLocator(locString, videoSetName, imageSetName);
}

void
DoDumpImageSetSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpImageSetSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageSetLocator loc = GetImageSetLocator(options, true);
    ImageSet* imSet = ImageSetRepository().Get(loc);
    imSet->Dump(0, -1, options.GetInt("start"), options.GetInt("end"));
    delete imSet;
}

void
DoDiffImageSetSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffImageSetSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageSetLocator loc1 = GetImageSetLocator(options, true);
    ImageSet* imSet1 = ImageSetRepository().Get(loc1);
    ImageSetLocator loc2 = GetImageSetLocator(options, false);
    ImageSet* imSet2 = ImageSetRepository().Get(loc2);

    imSet1->Diff(imSet2);

    delete imSet2;
    delete imSet1;
}

void
DoCopyImageSetSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyImageSetSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageSetLocator loc1 = GetImageSetLocator(options, true);
    ImageSet* imSet = ImageSetRepository().Get(loc1);

    ImageSetLocator loc2 = GetImageSetLocator(options, false);
    ImageSetRepository().Add(loc2, imSet);
    delete imSet;
}

void
DoDeleteImageSetSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteImageSetSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageSetLocator loc = GetImageSetLocator(options, true);
    ImageSetRepository().Delete(loc);
}

// VideoSet part

void
DoDumpVideoSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpVideoSet);
    CmdOptions& options = CmdOptions::GetInstance();

    VideoSet* vidSet = GetVideoSet(options, true);
    vidSet->Dump(0, -1, options.GetInt("start"), options.GetInt("end"));
    delete vidSet;
}

void
DoDiffVideoSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffVideoSet);
    CmdOptions& options = CmdOptions::GetInstance();

    VideoSet* vidSet1 = GetVideoSet(options, true);
    VideoSet* vidSet2 = GetVideoSet(options, true, false);
    vidSet1->Diff(vidSet2);
    delete vidSet2;
    delete vidSet1;
}

void
DoCopyVideoSet()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyVideoSet);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString1 = options.GetArg(1);
    String setName = options.GetArg(2);
    String locString2 = options.GetArg(3);
    VideoSet* vidSet = GetVideoSet(options, true);
    Locator setLoc2(locString2, setName);
    if (options.GetBool("sync"))
        VideoSetRepository().Sync(setLoc2, vidSet);
    else
        VideoSetRepository().Add(setLoc2, vidSet);
    delete vidSet;
}

void
DoDeleteVideoSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteVideoSet);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString = options.GetArg(1);
    String setName = options.GetArg(2);
    Locator loc(locString, setName);
    VideoSetRepository().Delete(loc);
}

void
DoSyncVideoFiles()
{
    ILOG_VAR(Impala.Application.Repository.DoSyncVideoFiles);
    CmdOptions& options = CmdOptions::GetInstance();

    String locString1 = options.GetArg(1);
    String setName = options.GetArg(2);
    bool doInfoFiles = StringToBool(options.GetArg(3));
    String locString2 = options.GetArg(4);

    VideoSet* vidSet = GetVideoSet(options, true);
    for (int i=0 ; i<vidSet->NrFiles() ; i++)
    {
        String path = FileNameConcat("VideoData", vidSet->GetAsPath(i));
        FileLocator srcLoc(locString1, setName, path);
        FileLocator dstLoc(locString2, setName, path);
        RepositoryInFileSystem::GetInstance().CopyFile(srcLoc, dstLoc);
        if (doInfoFiles)
        {
            String path = FileNameConcat("VideoData",
                                         vidSet->GetAsPath(i) + ".info");
            FileLocator srcLoc(locString1, setName, path);
            FileLocator dstLoc(locString2, setName, path);
            RepositoryInFileSystem::GetInstance().CopyFile(srcLoc, dstLoc);
        }
    }
    delete vidSet;
}

// Function is not consistent with others, awaiting RgbDataSrc refactoring
void
DoDeleteVideoInfoFiles()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteVideoInfoFiles);
    CmdOptions& options = CmdOptions::GetInstance();

    VideoSet* vidSet = GetVideoSet(options, false);
    if (vidSet)
    {
        for (int i=0 ; i<vidSet->NrFiles() ; i++)
            vidSet->DeleteVideoInfoFile(i);
        delete vidSet;
    }
}

// frame hashes part

void
DoDumpFrameHashes()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpFrameHashes);
    CmdOptions& options = CmdOptions::GetInstance();

    Locator vidSetLocator = GetDataSetLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    if (vidSet == 0)
        return;
    int nrOfVideos = vidSet->NrFiles();
    for (int i = 0; i < nrOfVideos; i++)
    {
        FrameHashesLocator fhLoc(vidSetLocator, vidSet->GetContainer(i));
        Core::VideoSet::FrameHashes* hashTable = 
            FrameHashesRepository().Get(fhLoc);
        if (hashTable != 0)
        {
            hashTable->Dump();
            delete hashTable;
        }
    }
    delete vidSet;
}

void
DoDiffFrameHashes()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffFrameHashes);
    CmdOptions& options = CmdOptions::GetInstance();

    Locator vs1Loc = GetDataSetLocator(options, true);
    Locator vs2Loc = GetDataSetLocator(options, false);
    VideoSet* vidSet1 = GetVideoSet(options, true, true);
    if (vidSet1 == 0)
        return;
    VideoSet* vidSet2 = GetVideoSet(options, true, false);
    if (vidSet2 == 0)
    {
        delete vidSet1;
        return;
    }
    int nrOfVideos1 = vidSet1->NrFiles();
    int nrOfVideos2 = vidSet2->NrFiles();
    if (nrOfVideos1 != nrOfVideos2)
    {
        ILOG_ERROR("Different file counts: " << nrOfVideos1 <<
            " vs. " << nrOfVideos2);
        delete vidSet1;
        delete vidSet2;
        return;
    }
    for (int i = 0; i < nrOfVideos1; i++)
    {
        FrameHashesLocator fh1Loc(vs1Loc, vidSet1->GetContainer(i));
        Core::VideoSet::FrameHashes* hashTable1 = 
            FrameHashesRepository().Get(fh1Loc);
        FrameHashesLocator fh2Loc(vs2Loc, vidSet2->GetContainer(i));
        Core::VideoSet::FrameHashes* hashTable2 = 
            FrameHashesRepository().Get(fh2Loc);
        if (hashTable1 != 0 && hashTable2 != 0)
            if (hashTable1->Diff(hashTable2) != 0)
                ILOG_ERROR("Found difference for: " << vidSet1->GetContainer(i));
        if (hashTable1 != 0)
            delete hashTable1;
        if (hashTable2 != 0)
            delete hashTable2;
    }
    delete vidSet2;
    delete vidSet1;
}

void
DoCopyFrameHashes()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyFrameHashes);
    CmdOptions& options = CmdOptions::GetInstance();

    ILOG_ERROR("Not yet implemented")
}

// Segmentation part

SegmentationLocator
GetSegmentationLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetSegmentationLocator);

    int nrArg = 2;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return SegmentationLocator(locString, setName, "segmentation");
}

void
DoDumpSegmentation()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpSegmentation);
    CmdOptions& options = CmdOptions::GetInstance();

    SegmentationLocator loc = GetSegmentationLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    Segmentation* seg = SegmentationRepository().Get(loc, vidSet);
    ILOG_INFO("nrVideos = " << seg->GetNrVideos());
    ILOG_INFO("nrShots = " << seg->GetNrShots());
    ILOG_INFO("nrFrames = " << seg->GetTotalNrFrames());
    if (options.GetInt("end") == 0)
        return;
    int minNr = 10000000;
    int maxNr = 0;
    for (int i=0 ; i<seg->GetNrVideos() ; i++)
    {
        int nrFrames = seg->GetNrFramesVideo(i);
        std::cout << "  vid " << i << " has " << seg->GetNrShotsVideo(i)
                  << " shots and " << nrFrames << " frames" << std::endl;
        if (nrFrames < minNr)
            minNr = nrFrames;
        if (nrFrames > maxNr)
            maxNr = nrFrames;
    }
    std::cout << "  min nr frames = " << minNr << ", max nr = " << maxNr
              << std::endl;
    seg->Dump(options.GetInt("start"), options.GetInt("end"));
    delete seg;
    delete vidSet;
}

void
DoDiffSegmentation()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffSegmentation);
    CmdOptions& options = CmdOptions::GetInstance();

    SegmentationLocator loc1 = GetSegmentationLocator(options, true);
    VideoSet* vidSet1 = GetVideoSet(options, true);
    Segmentation* seg1 = SegmentationRepository().Get(loc1, vidSet1);

    SegmentationLocator loc2 = GetSegmentationLocator(options, false);
    VideoSet* vidSet2 = VideoSetRepository().Get(loc2);
    Segmentation* seg2 = SegmentationRepository().Get(loc2, vidSet2);

    seg1->Diff(seg2);

    delete seg2;
    delete seg1;
    delete vidSet2;
    delete vidSet1;
}

void
DoCopySegmentation()
{
    ILOG_VAR(Impala.Application.Repository.DoCopySegmentation);
    CmdOptions& options = CmdOptions::GetInstance();

    SegmentationLocator loc1 = GetSegmentationLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    Segmentation* seg = SegmentationRepository().Get(loc1, vidSet);

    SegmentationLocator loc2 = GetSegmentationLocator(options, false);
    if (options.GetBool("sync"))
        SegmentationRepository().Sync(loc2, seg);
    else
        SegmentationRepository().Add(loc2, seg);
    delete seg;
    delete vidSet;
}

void
DoDeleteSegmentation()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteSegmentation);
    CmdOptions& options = CmdOptions::GetInstance();

    SegmentationLocator loc = GetSegmentationLocator(options, true);
    SegmentationRepository().Delete(loc);
}

// Keyframes part

KeyframesLocator
GetKeyframesLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetKeyframesLocator);

    int nrArg = 2;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return KeyframesLocator(locString, setName, "keyframes");
}

void
DoDumpKeyframes()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpKeyframes);
    CmdOptions& options = CmdOptions::GetInstance();

    KeyframesLocator loc = GetKeyframesLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    Keyframes* keys = KeyframesRepository().Get(loc, vidSet);

    ILOG_INFO("nrVideos = " << keys->GetNrVideos());
    ILOG_INFO("nrShots = " << keys->GetNrShots());
    ILOG_INFO("nrKeyframes = " << keys->GetNrKeyframes());
    if (options.GetInt("end") == 0)
        return;
    for (int i=0 ; i<keys->GetNrVideos() ; i++)
        std::cout << "  vid " << i << " has " << keys->GetNrKeyframesVideo(i)
                  << " keyframes" << std::endl;
    //keys.DumpGroups();
    keys->Dump(options.GetInt("start"), options.GetInt("end"));
    delete keys;
    delete vidSet;
}

void
DoDiffKeyframes()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffKeyframes);
    CmdOptions& options = CmdOptions::GetInstance();

    KeyframesLocator loc1 = GetKeyframesLocator(options, true);
    VideoSet* vidSet1 = GetVideoSet(options, true);
    Keyframes* keys1 = KeyframesRepository().Get(loc1, vidSet1);

    KeyframesLocator loc2 = GetKeyframesLocator(options, false);
    VideoSet* vidSet2 = VideoSetRepository().Get(loc2);
    Keyframes* keys2 = KeyframesRepository().Get(loc2, vidSet2);

    keys1->Diff(keys2);

    delete keys2;
    delete keys1;
    delete vidSet2;
    delete vidSet1;
}

void
DoCopyKeyframes()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyKeyframes);
    CmdOptions& options = CmdOptions::GetInstance();

    KeyframesLocator loc1 = GetKeyframesLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    Keyframes* keys = KeyframesRepository().Get(loc1, vidSet);

    KeyframesLocator loc2 = GetKeyframesLocator(options, false);
    if (options.GetBool("sync"))
        KeyframesRepository().Sync(loc2, keys);
    else
        KeyframesRepository().Add(loc2, keys);
    delete keys;
    delete vidSet;
}

void
DoDeleteKeyframes()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteKeyframes);
    CmdOptions& options = CmdOptions::GetInstance();

    KeyframesLocator loc = GetKeyframesLocator(options, true);
    KeyframesRepository().Delete(loc);
}

// Stills part

StillsLocator
GetStillsLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetStillsLocator);

    int nrArg = 2;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return StillsLocator(locString, setName, "stills");
}

void
DoDumpStills()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpStills);
    CmdOptions& options = CmdOptions::GetInstance();

    StillsLocator loc = GetStillsLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    Stills* stills = StillsRepository().Get(loc, vidSet);

    ILOG_INFO("nrVideos = " << stills->GetNrVideos());
    ILOG_INFO("nrShots = " << stills->GetNrShots());
    ILOG_INFO("nrStills = " << stills->GetNrStills());
    //stills->DumpGroups();
    stills->Dump(options.GetInt("start"), options.GetInt("end"));
    delete stills;
    delete vidSet;
}

void
DoDiffStills()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffStills);
    CmdOptions& options = CmdOptions::GetInstance();

    StillsLocator loc1 = GetStillsLocator(options, true);
    VideoSet* vidSet1 = GetVideoSet(options, true);
    Stills* stills1 = StillsRepository().Get(loc1, vidSet1);

    StillsLocator loc2 = GetStillsLocator(options, false);
    VideoSet* vidSet2 = VideoSetRepository().Get(loc2);
    Stills* stills2 = StillsRepository().Get(loc2, vidSet2);

    stills1->Diff(stills2);

    delete stills2;
    delete stills1;
    delete vidSet2;
    delete vidSet1;
}

void
DoCopyStills()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyStills);
    CmdOptions& options = CmdOptions::GetInstance();

    StillsLocator loc1 = GetStillsLocator(options, true);
    VideoSet* vidSet = GetVideoSet(options, true);
    Stills* stills = StillsRepository().Get(loc1, vidSet);

    StillsLocator loc2 = GetStillsLocator(options, false);
    if (options.GetBool("sync"))
        StillsRepository().Sync(loc2, stills);
    else
        StillsRepository().Add(loc2, stills);
    delete stills;
    delete vidSet;
}

void
DoDeleteStills()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteStills);
    CmdOptions& options = CmdOptions::GetInstance();

    StillsLocator loc = GetStillsLocator(options, true);
    StillsRepository().Delete(loc);
}

// QuidTable part

QuidTableLocator
GetQuidTableLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetQuidTableLocator);

    int nrArg = 5;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    String dir = options.GetArg(a++);
    String name = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return QuidTableLocator(locString, setName, dir, name);
}

void
DoDumpQuidTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpQuidTable);
    CmdOptions& options = CmdOptions::GetInstance();

    QuidTableLocator loc = GetQuidTableLocator(options, true);
    QuidTable* tab = QuidTableRepository().Get(loc);
    if (tab == 0)
        return;
    RawDataSet* dataSet = GetRawDataSet(options, false);
    tab->Dump(dataSet, options.GetInt("start"), options.GetInt("end"));
    delete tab;
    if (dataSet)
        delete dataSet;
}

void
DoDiffQuidTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffQuidTable);
    CmdOptions& options = CmdOptions::GetInstance();

    QuidTableLocator loc1 = GetQuidTableLocator(options, true);
    QuidTable* tab1 = QuidTableRepository().Get(loc1);
    if (tab1 == 0)
        return;
    QuidTableLocator loc2 = GetQuidTableLocator(options, false);
    QuidTable* tab2 = QuidTableRepository().Get(loc2);
    tab1->Diff(tab2);
    delete tab2;
    delete tab1;
}

void
DoCopyQuidTable()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyQuidTable);
    CmdOptions& options = CmdOptions::GetInstance();

    QuidTableLocator loc1 = GetQuidTableLocator(options, true);
    QuidTable* tab = QuidTableRepository().Get(loc1);
    if (tab == 0)
        return;
    QuidTableLocator loc2 = GetQuidTableLocator(options, false);
    QuidTableRepository().Add(loc2, tab);
    delete tab;
}

void
DoDeleteQuidTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteQuidTable);
    CmdOptions& options = CmdOptions::GetInstance();

    QuidTableLocator loc = GetQuidTableLocator(options, true);
    QuidTableRepository().Delete(loc);
}

// Mpeg7Doc part

Mpeg7DocLocator
GetMpeg7ShotLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetMpeg7ShotLocator);

    int nrArg = 4;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    String container = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return Mpeg7DocLocator(locString, setName, Mpeg7DocLocator::MPEG7_SHOT,
                           container, "dummy");
}

void
DoDumpMpeg7Shot()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpMpeg7Shot);
    CmdOptions& options = CmdOptions::GetInstance();

    Mpeg7DocLocator loc = GetMpeg7ShotLocator(options, true);
    VideoSet* videoSet = GetVideoSet(options, true);
    String container = loc.GetContainer();
    int nrDocs = (container == "ALL") ? videoSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrDocs ; i++)
    {
        String cur = (container == "ALL") ? videoSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        String name = FileNameBase(cur) + ".xml";
        loc.SetName(name);
        Mpeg7Doc* doc = Mpeg7DocRepository().Get(loc);
        doc->Dump(options.GetInt("start"), options.GetInt("end"));
        delete doc;
    }
    delete videoSet;
}

void
DoDiffMpeg7Shot()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffMpeg7Shot);
    CmdOptions& options = CmdOptions::GetInstance();

    Mpeg7DocLocator loc1 = GetMpeg7ShotLocator(options, true);
    Mpeg7DocLocator loc2 = GetMpeg7ShotLocator(options, false);
    VideoSet* videoSet = GetVideoSet(options, true);
    String container = loc1.GetContainer();
    int nrDocs = (container == "ALL") ? videoSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrDocs ; i++)
    {
        String cur = (container == "ALL") ? videoSet->GetContainer(i) : container;
        String name = FileNameBase(cur) + ".xml";
        ILOG_INFO("Checking " << i << " = " << cur);
        RgbDataSrc* src = videoSet->GetVideo(i);
        loc1.SetContainer(cur);
        loc1.SetName(name);
        loc1.SetFrameRateNum(src->FrameRateNum());
        loc1.SetFrameRateDen(src->FrameRateDen());
        Mpeg7Doc* doc1 = Mpeg7DocRepository().Get(loc1);
        loc2.SetContainer(cur);
        loc2.SetName(name);
        loc2.SetFrameRateNum(src->FrameRateNum());
        loc2.SetFrameRateDen(src->FrameRateDen());
        Mpeg7Doc* doc2 = Mpeg7DocRepository().Get(loc2);
        delete src;
        doc1->Diff(doc2);
        delete doc2;
        delete doc1;
    }
    delete videoSet;
}

void
DoCopyMpeg7Shot()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyMpeg7Shot);
    CmdOptions& options = CmdOptions::GetInstance();

    /*
    Mpeg7DocLocator loc1 = GetMpeg7ShotLocator(options, true);
    Mpeg7DocLocator loc2 = GetMpeg7ShotLocator(options, false);
    VideoSet* videoSet = GetVideoSet(options, true);
    String container = loc1.GetContainer();
    int nrDocs = (container == "ALL") ? videoSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrDocs ; i++)
    {
        String cur = (container == "ALL") ? videoSet->GetContainer(i) : container;
        Quid quid = GetQuidVideo(videoSet, container);
        loc1.RawDataSetContainer(cur);
        loc1.SetQuid(quid);
        Mpeg7Doc* doc = Mpeg7DocRepository().Get(loc1);
        loc2.SetContainer(cur);
        loc2.SetQuid(quid);
        Mpeg7DocRepository().Add(loc2, doc);
        delete doc;
    }
    delete videoSet;
    */
}

void
DoDeleteMpeg7Shot()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteMpeg7Shot);
    CmdOptions& options = CmdOptions::GetInstance();

    Mpeg7DocLocator loc = GetMpeg7ShotLocator(options, true);
    if (loc.GetContainer() == "ALL") // This is a bit rude, use with care
        return Mpeg7DocRepository().Delete(loc);
    VideoSet* videoSet = GetVideoSet(options, false);
    if (!videoSet)
        return;

    int nrDocs = videoSet->GetNrContainers();
    for (int i=0 ; i<nrDocs ; i++)
    {
        String cur = videoSet->GetContainer(i);
        loc.SetContainer(cur);
        String name = FileNameBase(cur) + ".xml";
        loc.SetName(name);
        Mpeg7DocRepository().Delete(loc);
    }
    delete videoSet;
}

void
DoDeleteMpeg7Annotation()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteMpeg7Annotation);
    CmdOptions& options = CmdOptions::GetInstance();

    Mpeg7DocLocator loc = GetMpeg7ShotLocator(options, true);
    loc.SetDocType(Mpeg7DocLocator::MPEG7_ANNOTATION);
    if (loc.GetContainer() == "ALL") // This is a bit rude, use with care
        return Mpeg7DocRepository().Delete(loc);

    ILOG_ERROR("Not fully implemented yet");
}

void
DoDeleteMpeg7Concept()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteMpeg7Concept);
    CmdOptions& options = CmdOptions::GetInstance();

    Mpeg7DocLocator loc = GetMpeg7ShotLocator(options, true);
    loc.SetDocType(Mpeg7DocLocator::MPEG7_CONCEPT);
    if (loc.GetContainer() == "ALL") // This is a bit rude, use with care
        return Mpeg7DocRepository().Delete(loc);

    ILOG_ERROR("Not fully implemented yet");
}

// ImageArchive part

ImageArchiveLocator
GetImageArchiveLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetImageArchiveLocator);

    int nrArg = 6;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    bool frames = StringToBool(options.GetArg(a++));
    String container = options.GetArg(a++);
    String name = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return ImageArchiveLocator(locString, setName, frames, container, name, 0);
}

Quid
GetQuidVideo(RawDataSet* dataSet, String container)
{
    if (dataSet->IsImageSet())
        return QUID_CLASS_UNKNOWN;
    VideoSet* vidSet = dynamic_cast<VideoSet*>(dataSet);
    int v = vidSet->GetFileId(container);
    Quid quid = (v != -1) ? vidSet->GetQuidVideo(v, false) : QUID_CLASS_UNKNOWN;
    return quid;
}

void
DoDumpImageArchive()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpImageArchive);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc = GetImageArchiveLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc.GetContainer();
    int nrArchives = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        Quid quid = GetQuidVideo(dataSet, container);
        loc.SetQuid(quid);
        ImageArchive* archive = ImageArchiveRepository().Get(loc);
        std::cout << loc << " has " << archive->NrImages() << " images"
                  << std::endl;
        if (archive->NrImages() > 0)
        {
            Core::Array::Array2dVec3UInt8* im = archive->ReadImage(0);
            std::cout << "ImageSize = " << im->CW() << "x" << im->CH()
                      << std::endl;
            delete im;
        }
        delete archive;
    }
    delete dataSet;
}

void
DoDiffImageArchive()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffImageArchive);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc1 = GetImageArchiveLocator(options, true);
    ImageArchiveLocator loc2 = GetImageArchiveLocator(options, false);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrArchives = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        ILOG_INFO("Checking " << i << " = " << cur);
        Quid quid = GetQuidVideo(dataSet, container);
        loc1.SetContainer(cur);
        loc1.SetQuid(quid);
        ImageArchive* archive1 = ImageArchiveRepository().Get(loc1);
        loc2.SetContainer(cur);
        loc2.SetQuid(quid);
        ImageArchive* archive2 = ImageArchiveRepository().Get(loc2);
        archive1->Diff(archive2);
        delete archive2;
        delete archive1;
    }
    delete dataSet;
}

void
DoCopyImageArchive()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyImageArchive);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc1 = GetImageArchiveLocator(options, true);
    ImageArchiveLocator loc2 = GetImageArchiveLocator(options, false);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrArchives = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        Quid quid = GetQuidVideo(dataSet, container);
        loc1.SetContainer(cur);
        loc1.SetQuid(quid);
        ImageArchive* archive = ImageArchiveRepository().Get(loc1);
        loc2.SetContainer(cur);
        loc2.SetQuid(quid);
        if (options.GetBool("sync"))
            ImageArchiveRepository().Sync(loc2, archive);
        else
            ImageArchiveRepository().Add(loc2, archive);
        delete archive;
    }
    delete dataSet;
}

void
DoDeleteImageArchive()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteImageArchive);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc = GetImageArchiveLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, false);
    if (!dataSet)
        return;
    String container = loc.GetContainer();
    int nrArchives = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        Quid quid = GetQuidVideo(dataSet, container);
        loc.SetQuid(quid);
        ImageArchiveRepository().Delete(loc);
    }
    delete dataSet;
}

// ImageArchiveSpecial part

ImageArchiveLocator
GetImageArchiveSpecialLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetImageArchiveSpecialLocator);

    int nrArg = 6;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    bool frames = false;
    a++; // This is the imageset.txt, not needed for this locator
    String container = options.GetArg(a++);
    String name = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return ImageArchiveLocator(locString, setName, frames, container, name, 0);
}

void
DoDumpImageArchiveSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpImageArchiveSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc = GetImageArchiveSpecialLocator(options, true);
    ImageSetLocator imSetLoc = GetImageSetLocator(options, true);
    ImageSet* imSet = ImageSetRepository().Get(imSetLoc);
    String container = loc.GetContainer();
    int nrArchives = (container == "ALL") ? imSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? imSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        ImageArchive* archive = ImageArchiveRepository().Get(loc);
        std::cout << loc << " has " << archive->NrImages() << " images"
                  << std::endl;
        if (archive->NrImages() > 0)
        {
            Core::Array::Array2dVec3UInt8* im = archive->ReadImage(0);
            std::cout << "ImageSize = " << im->CW() << "x" << im->CH()
                      << std::endl;
            delete im;
        }
        delete archive;
    }
    delete imSet;
}

void
DoDiffImageArchiveSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffImageArchiveSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc1 = GetImageArchiveSpecialLocator(options, true);
    ImageArchiveLocator loc2 = GetImageArchiveSpecialLocator(options, false);
    ImageSetLocator imSetLoc = GetImageSetLocator(options, true);
    ImageSet* imSet = ImageSetRepository().Get(imSetLoc);
    String container = loc1.GetContainer();
    int nrArchives = (container == "ALL") ? imSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? imSet->GetContainer(i) : container;
        ILOG_INFO("Checking " << i << " = " << cur);
        loc1.SetContainer(cur);
        ImageArchive* archive1 = ImageArchiveRepository().Get(loc1);
        loc2.SetContainer(cur);
        ImageArchive* archive2 = ImageArchiveRepository().Get(loc2);
        archive1->Diff(archive2);
        delete archive2;
        delete archive1;
    }
    delete imSet;
}

void
DoCopyImageArchiveSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyImageArchiveSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc1 = GetImageArchiveSpecialLocator(options, true);
    ImageArchiveLocator loc2 = GetImageArchiveSpecialLocator(options, false);
    ImageSetLocator imSetLoc = GetImageSetLocator(options, true);
    ImageSet* imSet = ImageSetRepository().Get(imSetLoc);
    String container = loc1.GetContainer();
    int nrArchives = (container == "ALL") ? imSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? imSet->GetContainer(i) : container;
        loc1.SetContainer(cur);
        ImageArchive* archive = ImageArchiveRepository().Get(loc1);
        loc2.SetContainer(cur);
        ImageArchiveRepository().Add(loc2, archive);
        delete archive;
    }
    delete imSet;
}

void
DoDeleteImageArchiveSpecial()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteImageArchiveSpecial);
    CmdOptions& options = CmdOptions::GetInstance();

    ImageArchiveLocator loc = GetImageArchiveSpecialLocator(options, true);
    ImageSetLocator imSetLoc = GetImageSetLocator(options, true);
    if (!ImageSetRepository().Exists(imSetLoc))
        return;

    ImageSet* imSet = ImageSetRepository().Get(imSetLoc);
    String container = loc.GetContainer();
    int nrArchives = (container == "ALL") ? imSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrArchives ; i++)
    {
        String cur = (container == "ALL") ? imSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        ImageArchiveRepository().Delete(loc);
    }
    delete imSet;
}

// FeatureTable part

FeatureLocator
GetFeatureLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetFeatureLocator);

    int nrArg = 8;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    bool isCodebook = StringToBool(options.GetArg(a++));
    bool isIndex = StringToBool(options.GetArg(a++));
    String walkType = options.GetArg(a++);
    String feature = options.GetArg(a++);
    String container = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return FeatureLocator(locString, setName, isCodebook, isIndex, walkType,
                          feature, container);
}

void
DoDumpFeatureTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();

    FeatureLocator loc = GetFeatureLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        FeatureTable* tab = FeatureTableRepository().Get(loc);
        tab->Dump(0, options.GetInt("start"), options.GetInt("end"));
        delete tab;
    }
}

void
DoDiffFeatureTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();

    FeatureLocator loc1 = GetFeatureLocator(options, true);
    FeatureLocator loc2 = GetFeatureLocator(options, false);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        ILOG_INFO("Checking " << i << " = " << cur);
        loc1.SetContainer(cur);
        FeatureTable* tab1 = FeatureTableRepository().Get(loc1);
        loc2.SetContainer(cur);
        FeatureTable* tab2 = FeatureTableRepository().Get(loc2);
        tab1->Diff(tab2);
        delete tab2;
        delete tab1;
    }
}

void
DoCopyFeatureTable()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();

    FeatureLocator loc1 = GetFeatureLocator(options, true);
    FeatureLocator loc2 = GetFeatureLocator(options, false);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc1.SetContainer(cur);
        FeatureTable* tab = FeatureTableRepository().Get(loc1);
        loc2.SetContainer(cur);
        if (options.GetBool("sync"))
            FeatureTableRepository().Sync(loc2, tab);
        else
            FeatureTableRepository().Add(loc2, tab);
        delete tab;
    }
}

void
DoDeleteFeatureTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();

    FeatureLocator loc = GetFeatureLocator(options, true);
    if (loc.GetFeatureName() == "ALL") // This is a bit rude, use with care
        return FeatureTableRepository().Delete(loc);
    RawDataSet* dataSet = GetRawDataSet(options, false);
    if (!dataSet)
        return;

    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        FeatureTableRepository().Delete(loc);
    }
}

void
DoSetQuidSetFeatureTable()
{
    ILOG_VAR(Impala.Application.Repository.DoSetQuidSetFeatureTable);
    CmdOptions& options = CmdOptions::GetInstance();

    int a = 8;
    if (options.GetNrArg() < a + 1)
    {
        ILOG_ERROR("Need at least " << a + 1 << " parameters");
    }
    int setId = atol(options.GetArg(a));
    ILOG_INFO("Setting id to " << setId);

    FeatureLocator loc = GetFeatureLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        ILOG_INFO("Doing i=" << i << ", cur=" << cur);
        loc.SetContainer(cur);
        FeatureTable* tab = FeatureTableRepository().Get(loc);
        SetQuidSet(tab, setId);
        FeatureTableRepository().Add(loc, tab); // Should be Replace
        delete tab;
    }
}

// KeywordList part

KeywordListLocator
GetKeywordListLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetKeywordListLocator);

    int nrArg = 3;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    String conceptSet = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return KeywordListLocator(locString, setName, conceptSet);
}

void
DoDumpKeywordList()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpKeywordList);
    CmdOptions& options = CmdOptions::GetInstance();

    KeywordListLocator loc = GetKeywordListLocator(options, true);
    KeywordList keyList = *(KeywordListRepository().Get(loc));
    for (int i=0 ; i<keyList.size() ; i++)
        std::cout << keyList[i] << std::endl;
}

void
DoDiffKeywordList()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffKeywordList);
    CmdOptions& options = CmdOptions::GetInstance();

    KeywordListLocator loc1 = GetKeywordListLocator(options, true);
    KeywordList keyList1 = *(KeywordListRepository().Get(loc1));
    KeywordListLocator loc2 = GetKeywordListLocator(options, false);
    KeywordList keyList2 = *(KeywordListRepository().Get(loc2));
    Core::Table::KeywordListDiff(keyList1, keyList2);    
}

void
DoCopyKeywordList()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyKeywordList);
    CmdOptions& options = CmdOptions::GetInstance();

    KeywordListLocator loc1 = GetKeywordListLocator(options, true);
    KeywordList* keyList = KeywordListRepository().Get(loc1);
    KeywordListLocator loc2 = GetKeywordListLocator(options, false);
    if (options.GetBool("sync"))
        KeywordListRepository().Sync(loc2, keyList);
    else
        KeywordListRepository().Add(loc2, keyList);
    delete keyList;
}

// AnnotationTable part

AnnotationTableLocator
GetAnnotationTableLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetAnnotationTableLocator);

    int nrArg = 6;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    int quidClass = StringToQuidClass(options.GetArg(a++));
    String conceptSet = options.GetArg(a++);
    String keyword = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return AnnotationTableLocator(locString, setName, quidClass, conceptSet,
                                  keyword);
}

void
DoDumpAnnotationTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpAnnotationTable);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableLocator loc = GetAnnotationTableLocator(options, true);
    AnnotationTable* tab = AnnotationTableRepository().Get(loc);
    if (tab == 0)
        return;
    RawDataSet* dataSet = GetRawDataSet(options, false);
    tab->Dump(dataSet, options.GetInt("start"), options.GetInt("end"));
    tab->DumpSummary();
    delete tab;
    if (dataSet)
        delete dataSet;
}

void
DoDiffAnnotationTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffAnnotationTable);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableLocator loc1 = GetAnnotationTableLocator(options, true);
    AnnotationTable* tab1 = AnnotationTableRepository().Get(loc1);
    if (tab1 == 0)
        return;
    AnnotationTableLocator loc2 = GetAnnotationTableLocator(options, false);
    AnnotationTable* tab2 = AnnotationTableRepository().Get(loc2);
    tab1->Diff(tab2);
    delete tab2;
    delete tab1;
}

void
DoCopyAnnotationTable()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyAnnotationTable);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableLocator loc1 = GetAnnotationTableLocator(options, true);
    AnnotationTable* tab = AnnotationTableRepository().Get(loc1);
    if (tab == 0)
        return;
    AnnotationTableLocator loc2 = GetAnnotationTableLocator(options, false);
    AnnotationTableRepository().Add(loc2, tab);
    delete tab;
}

void
DoDeleteAnnotationTable()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteAnnotationTable);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableLocator loc = GetAnnotationTableLocator(options, true);
    AnnotationTableRepository().Delete(loc);
}

// Merges two annotation tables on the same dataset into a new table.
// The new concept is positive if either one of the input tables is positive
void
DoMergeAnnotationTable()
{
    ILOG_VAR(Impala.Application.Repository.DoMergeAnnotationTable);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableLocator loc = GetAnnotationTableLocator(options, true);
    String outputKeyword = loc.GetKeyword();
    AnnotationTable* res = 0;
    int a = 6;
    while (a < options.GetNrArg())
    {
        String inputKeyword = options.GetArg(a++);
        ILOG_INFO("Merging " << inputKeyword);
        loc.SetKeyword(inputKeyword);
        AnnotationTable* tab = AnnotationTableRepository().Get(loc);
        if (tab == 0)
            return;
        if (res == 0)
        {
            res = tab;
            res->SetLabel(outputKeyword);
        }
        else
        {
            res->Merge(tab);
            delete tab;
        }

        ILOG_INFO("Merged " << loc << " : ");
        res->DumpSummary();
    }
    loc.SetKeyword(outputKeyword);
    AnnotationTableRepository().Add(loc, res);
    delete res;
}

// AnnotationTableSet part

AnnotationTableSetLocator
GetAnnotationTableSetLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetAnnotationTableSetLocator);

    int nrArg = 5;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    int quidClass = StringToQuidClass(options.GetArg(a++));
    String conceptSet = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return AnnotationTableSetLocator(locString, setName, quidClass, conceptSet);
}

void
DoDumpAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    int a = 5;
    bool dumpTable = atol(options.GetArg(a++)) != 0;
    int maxV = -1;
    if (options.GetNrArg() > a)
        maxV = atol(options.GetArg(a++));
    int maxP = -1;
    if (options.GetNrArg() > a)
        maxP = atol(options.GetArg(a++));
    int maxN = -1;
    if (options.GetNrArg() > a)
        maxN = atol(options.GetArg(a++));

    AnnotationTableSetLocator loc = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSet* tSet = AnnotationTableSetRepository().Get(loc);
    if (tSet == 0)
        return;
    RawDataSet* dataSet = GetRawDataSet(options, false);

    for (int i=0 ; i<tSet->Size() ; i++)
    {
        AnnotationTable* tab = tSet->GetTable(i);
        tab->DumpSummary();
        //tab->DumpSummaryObject();
        if (dumpTable)
            tab->Dump(dataSet, options.GetInt("start"), options.GetInt("end"));
        if (maxV != -1)
        {
            tab->SelectQuidObjectMaxId(maxV);
            tab->SetLabel(tab->GetLabel() + "_maxV_" + MakeString(maxV));
            tab->DumpSummary();
            tab->DumpSummaryObject();
        }
        if (maxP != -1)
        {
            tab->SelectQuidObjectMaxPositive(maxP);
            tab->SetLabel(tab->GetLabel() + "_maxP_" + MakeString(maxP));
            tab->DumpSummary();
            tab->DumpSummaryObject();
        }
        if (maxN != -1)
        {
            tab->SelectQuidObjectMaxNegative(maxN);
            tab->SetLabel(tab->GetLabel() + "_maxN_" + MakeString(maxN));
            tab->DumpSummary();
            tab->DumpSummaryObject();
        }
    }

    delete tSet;
    if (dataSet)
        delete dataSet;
}

void
DoDiffAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableSetLocator loc1 = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSet* tSet1 = AnnotationTableSetRepository().Get(loc1);
    if (tSet1 == 0)
        return;
    AnnotationTableSetLocator loc2 = GetAnnotationTableSetLocator(options, false);
    AnnotationTableSet* tSet2 = AnnotationTableSetRepository().Get(loc2);
    tSet1->Diff(tSet2);
    delete tSet2;
    delete tSet1;
}

void
DoCopyAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableSetLocator loc1 = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSet* tSet = AnnotationTableSetRepository().Get(loc1);
    if (tSet == 0)
        return;
    AnnotationTableSetLocator loc2 = GetAnnotationTableSetLocator(options, false);
    if (options.GetBool("sync"))
        AnnotationTableSetRepository().Sync(loc2, tSet);
    else
        AnnotationTableSetRepository().Add(loc2, tSet);
    delete tSet;
}

void
DoDeleteAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableSetLocator loc = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSetRepository().Delete(loc);
}

// Merges annotations from different data sets by appending existing tables.
void
DoMergeAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoMergeAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableSet* res = 0;
    int a = 1;
    while (a < options.GetNrArg())
    {
        if (a + 4 > options.GetNrArg())
        {
            ILOG_ERROR("Incorrect number of parameters");
            return;
        }
        String locString = options.GetArg(a++);
        String setName = options.GetArg(a++);
        int quidClass = StringToQuidClass(options.GetArg(a++));
        String conceptSet = options.GetArg(a++);
        AnnotationTableSetLocator loc(locString, setName, quidClass, conceptSet);

        if (a == options.GetNrArg())
        {
            ILOG_INFO("Saving result in " << loc);
            AnnotationTableSetRepository().Add(loc, res);
            delete res;
            return;
        }

        AnnotationTableSet* tSet = AnnotationTableSetRepository().Get(loc);
        if (tSet == 0)
            return;
        if (res == 0)
        {
            res = tSet;
        }
        else
        {
            res->Merge(tSet);
            delete tSet;
        }

        ILOG_INFO("Merged " << loc << " : ");
        for (int i=0 ; i<res->Size() ; i++)
        {
            AnnotationTable* tab = res->GetTable(i);
            tab->DumpSummary();
        }
    }
}

void
DoCorrectAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoCorrectAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    int nrArg = 7;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    AnnotationTableSetLocator loc = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSet* tSet = AnnotationTableSetRepository().Get(loc);
    if (tSet == 0)
        return;

    int a = 5;
    String corrSetName = options.GetArg(a++);
    String resSetName = options.GetArg(a++);
    loc.SetConceptSet(corrSetName);
    AnnotationTableSet* corrSet = AnnotationTableSetRepository().Get(loc);
    if (corrSet == 0)
        return;

    tSet->Correct(corrSet);
    delete corrSet;
    loc.SetConceptSet(resSetName);
    ILOG_INFO("Saving result in " << loc);
    AnnotationTableSetRepository().Add(loc, tSet);
    delete tSet;
}

void
DoSelectAnnotationTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoSelectAnnotationTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    int nrArg = 7;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    AnnotationTableSetLocator loc = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSet* tSet = AnnotationTableSetRepository().Get(loc);
    if (tSet == 0)
        return;

    int a = 5;
    String concept = options.GetArg(a++);
    String iteration = options.GetArg(a++);
    AnnotationTable* src = tSet->GetTable(concept);
    if (!src)
    {
        ILOG_ERROR("No annotations for " << concept);
        return;
    }

    std::vector<String> labels;
    labels.push_back(concept);
    AnnotationTableSet resSet(labels, 0);
    AnnotationTable* res = resSet.GetTable(concept);
    Copy(res, src);
    res->SelectQuidObjectMaxPositive(5 * atol(iteration));
    res->SelectQuidObjectMaxNegative(5 * atol(iteration));
    res->SelectQuidObjectMaxSkip(0);

    String resSetName = FileNameBase(loc.GetConceptSet()) + "_" + concept + "_"
        + iteration + ".txt";
    loc.SetConceptSet(resSetName);
    ILOG_INFO("Saving result in " << loc);
    AnnotationTableSetRepository().Add(loc, &resSet);
    delete tSet;
}

void
DoIndexAnnotatedFeatures()
{
    ILOG_VAR(Impala.Application.Repository.DoIndexAnnotatedFeatures);
    CmdOptions& options = CmdOptions::GetInstance();

    AnnotationTableSetLocator aLoc = GetAnnotationTableSetLocator(options, true);
    AnnotationTableSet* tSet = AnnotationTableSetRepository().Get(aLoc);
    if (tSet == 0)
        return;

    int a = 5;
    String featureDef = options.GetArg(a++);
    ILOG_INFO("FeatureDef = " << featureDef);

    QuidTable* quids = tSet->GetUniqueQuids();
    ILOG_INFO("Read " << quids->Size() << " quids, starting sort...");
    Sort(quids, 1, true);
    ILOG_INFO("Found " << quids->Size() << " unique quids");

    int lastClassId = -1;
    int lastSetId = -1;
    int lastContainerNr = -1;
    RawDataSet* dataSet = 0;
    FeatureLocator fLoc;
    FeatureTable* features = 0;
    FeatureTable* result = 0;
    String locString = aLoc.GetProtocolAndHost();
    for (int i=0 ; i<quids->Size() ; i++)
    {
        Quid q = quids->Get1(i);
        if ((lastClassId != QuidClass(q)) || (lastSetId != QuidSet(q)))
        {
            if (dataSet)
                delete dataSet;
            lastClassId = QuidClass(q);
            lastSetId = QuidSet(q);
            dataSet = Core::Database::MakeRawDataSet(locString, q);
            ILOG_INFO("i=" << i << ": new dataset " << dataSet->GetSetName());
            String walk = (QuidClass(q) == QUID_CLASS_IMAGE) ? "" : "Keyframes";
            fLoc = FeatureLocator(dataSet->GetLocator(), false,
                                  false, walk, featureDef, "<empty>");
            lastContainerNr = -1;
        }

        int containerNr = dataSet->GetContainerNrOfQuid(q);
        if (lastContainerNr != containerNr)
        {
            lastContainerNr = containerNr;
            String container = dataSet->GetContainer(lastContainerNr);
            ILOG_INFO("i=" << i << ": new container " << container);
            fLoc.SetContainer(container);
            if (features)
                delete features;
            features = FeatureTableRepository().Get(fLoc);
        }
        FeatureTable::VectorReal64 vec = features->FindFeature(q);
        if (result == 0)
        {
            result = new FeatureTable(features->GetFeatureDefinition(),
                                      quids->Size(), vec.Size());
        }
        result->Add(q, vec);
    }
    if (features)
        delete features;
    if (dataSet)
        delete dataSet;
    if (result == 0)
    {
        ILOG_ERROR("No features found");
        return;
    }
    if (result->Size() != quids->Size())
    {
        ILOG_ERROR("Found " << result->Size() << " features for " <<
                   quids->Size() << " quids");
    }
    delete quids;

    FeatureLocator rLoc(aLoc, false, true, aLoc.GetConceptSet(), featureDef, "");
    FeatureTableRepository().Add(rLoc, result);
    delete result;
}

// Fold part

FoldLocator
GetFoldLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetFoldLocator);

    int nrArg = 8;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    int quidClass = StringToQuidClass(options.GetArg(a++));
    String conceptSet = options.GetArg(a++);
    String keyword = options.GetArg(a++);
    String foldType = options.GetArg(a++);
    int foldNr = atol(options.GetArg(a++));
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return FoldLocator(locString, setName, quidClass, conceptSet, keyword,
                       foldType, foldNr);
}

void
DoDumpFold()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpFold);
    CmdOptions& options = CmdOptions::GetInstance();

    FoldLocator loc = GetFoldLocator(options, true);
    QuidTable* tab = FoldRepository().Get(loc);
    if (tab == 0)
        return;
    RawDataSet* dataSet = GetRawDataSet(options, false);
    tab->Dump(dataSet, options.GetInt("start"), options.GetInt("end"));
    delete tab;
    if (dataSet)
        delete dataSet;
}

void
DoDiffFold()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffFold);
    CmdOptions& options = CmdOptions::GetInstance();

    FoldLocator loc1 = GetFoldLocator(options, true);
    QuidTable* tab1 = FoldRepository().Get(loc1);
    if (tab1 == 0)
        return;
    FoldLocator loc2 = GetFoldLocator(options, false);
    QuidTable* tab2 = FoldRepository().Get(loc2);
    tab1->Diff(tab2);
    delete tab2;
    delete tab1;
}

void
DoCopyFold()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyFold);
    CmdOptions& options = CmdOptions::GetInstance();

    FoldLocator loc1 = GetFoldLocator(options, true);
    QuidTable* tab = FoldRepository().Get(loc1);
    if (tab == 0)
        return;
    FoldLocator loc2 = GetFoldLocator(options, false);
    FoldRepository().Add(loc2, tab);
    delete tab;
}

void
DoDeleteFold()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteFold);
    CmdOptions& options = CmdOptions::GetInstance();

    FoldLocator loc = GetFoldLocator(options, true);
    FoldRepository().Delete(loc);
}

// SimilarityTableSet part

SimilarityTableSetLocator
GetSimilarityTableSetLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetSimilarityTableSetLocator);

    int nrArg = 9;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    bool inIndex = StringToBool(options.GetArg(a++));
    String walkType = options.GetArg(a++);
    String conceptSet = options.GetArg(a++);
    String model = options.GetArg(a++);
    String feature = options.GetArg(a++);
    String container = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return SimilarityTableSetLocator(locString, setName, inIndex, walkType,
                                     conceptSet, model, feature, container);
}

void
DoDumpSimilarityTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpSimilarityTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    bool doMatrix = false;
    if (options.GetNrArg() > 9)
        doMatrix = StringToBool(options.GetArg(9));
    bool dumpQuid = false;
    if (options.GetNrArg() > 10)
        dumpQuid = StringToBool(options.GetArg(10));

    SimilarityTableSetLocator loc = GetSimilarityTableSetLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        SimilarityTableSet* simSet = SimilarityTableSetRepository().Get(loc);
        if (doMatrix)
        {
            simSet->DumpMatrix(dataSet, dumpQuid);
        }
        else
        {
            simSet->DumpRanking(dataSet, options.GetInt("start"),
                                options.GetInt("end"));
        }
        delete simSet;
    }
}

void
DoDiffSimilarityTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffSimilarityTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    SimilarityTableSetLocator loc1 = GetSimilarityTableSetLocator(options, true);
    SimilarityTableSetLocator loc2 = GetSimilarityTableSetLocator(options, false);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        ILOG_INFO("Checking " << i << " = " << cur);
        loc1.SetContainer(cur);
        SimilarityTableSet* simSet1 = SimilarityTableSetRepository().Get(loc1);

        loc2.SetContainer(cur);
        SimilarityTableSet* simSet2 = SimilarityTableSetRepository().Get(loc2);
        simSet1->Diff(simSet2);
        delete simSet2;
        delete simSet1;
    }
}

void
DoCopySimilarityTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoCopySimilarityTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    SimilarityTableSetLocator loc1 = GetSimilarityTableSetLocator(options, true);
    SimilarityTableSetLocator loc2 = GetSimilarityTableSetLocator(options, false);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc1.SetContainer(cur);
        SimilarityTableSet* simSet = SimilarityTableSetRepository().Get(loc1);

        loc2.SetContainer(cur);
        SimilarityTableSetRepository().Add(loc2, simSet);
        delete simSet;
    }
}

void
DoDeleteSimilarityTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteSimilarityTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    SimilarityTableSetLocator loc = GetSimilarityTableSetLocator(options, true);
    if (loc.GetConceptSet() == "ALL") // This is a bit rude, use with care
        return SimilarityTableSetRepository().Delete(loc);
    RawDataSet* dataSet = GetRawDataSet(options, false);
    if (!dataSet)
        return;

    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        SimilarityTableSetRepository().Delete(loc);
    }
}

void
DoMergeSimilarityTableSet()
{
    ILOG_VAR(Impala.Application.Repository.DoMergeSimilarityTableSet);
    CmdOptions& options = CmdOptions::GetInstance();

    SimilarityTableSetLocator dstLoc =
        GetSimilarityTableSetLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    std::vector<String> srcSets;
    for (int a=9 ; a<options.GetNrArg() ; a++)
    {
        srcSets.push_back(options.GetArg(a));
    }

    String container = dstLoc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        SimilarityTableSet* dstSet = 0;
        SimilarityTableSetLocator srcLoc = dstLoc;
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        srcLoc.SetContainer(cur);
        for (int c=0 ; c<srcSets.size() ; c++)
        {
            srcLoc.SetConceptSet(srcSets[c]);
            ILOG_INFO("Merging " << i << " = " << cur << " for " << srcSets[c]);
            SimilarityTableSet* s = SimilarityTableSetRepository().Get(srcLoc);

            //s->DumpRanking(0, options.GetInt("start"), options.GetInt("end"));
            if (dstSet == 0)
            {
                dstSet = s;
            }
            else
            {
                dstSet->Merge(s);
                delete s;
            }
        }
        dstLoc.SetContainer(cur);
        SimilarityTableSetRepository().Add(dstLoc, dstSet);
        delete dstSet;
    }
}

// Model part

ModelLocator
GetModelLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetModelLocator);

    int nrArg = 7;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    String conceptSet = options.GetArg(a++);
    String model = options.GetArg(a++);
    String feature = options.GetArg(a++);
    String concept = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    return ModelLocator(locString, setName, conceptSet, model, feature, concept);
}

KeywordList
GetModelKeywordList(const ModelLocator& modelLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetModelKeywordList);

    KeywordList res;
    String concept = modelLoc.GetConcept();
    if (concept == "ALL")
    {
        KeywordListLocator keyLoc(modelLoc, modelLoc.GetConceptSet());
        KeywordList* list = KeywordListRepository().Get(keyLoc);
        if(list)
        {
            res = *list;
            delete list;
            if (res.size() == 0)
                ILOG_ERROR("No keywords found");
        }
        else
        {
            ILOG_ERROR("keyword list not found");
        }
    }
    else
    {
        res.push_back(concept);
    }
    return res;
}

void
DoDumpBestFile()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpBestFile);

    CmdOptions& options = CmdOptions::GetInstance();
    ModelLocator loc = GetModelLocator(options, true);
    KeywordList concepts = GetModelKeywordList(loc);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc.SetConcept(concepts[i]);
        Util::PropertySet* bestFile = BestFileRepository().Get(loc);
        if (bestFile)
        {
            bestFile->Print(std::cout);
            std::cout << std::endl;
            delete bestFile;
        }
    }
}

void
DoDiffBestFile()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffBestFile);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        ILOG_INFO("Checking " << i << " = " << concepts[i]);
        loc1.SetConcept(concepts[i]);
        Util::PropertySet* bestFile1 = BestFileRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        Util::PropertySet* bestFile2 = BestFileRepository().Get(loc2);
        Core::Training::BestFileReferenceDiff(bestFile1, bestFile2);
        delete bestFile1;
        delete bestFile2;
    }
}

void
DoCopyBestFile()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyDiffBestFile);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc1.SetConcept(concepts[i]);
        Util::PropertySet* bestFile = BestFileRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        BestFileRepository().Add(loc2, bestFile);
        delete bestFile;
    }
}

void
DoDumpSvmProblem()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpSvmProblem);

    CmdOptions& options = CmdOptions::GetInstance();
    ModelLocator loc = GetModelLocator(options, true);
    KeywordList concepts = GetModelKeywordList(loc);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc.SetConcept(concepts[i]);
        svm_problem* problem = SvmProblemRepository().Get(loc);
        if (problem)
        {
            Dump(problem, -1, 10, std::cout);
            DestroySvmProblem(problem);
        }
    }
}

void
DoDiffSvmProblem()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffSvmProblem);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        ILOG_INFO("Checking " << i << " = " << concepts[i]);
        loc1.SetConcept(concepts[i]);
        svm_problem* problem1 = SvmProblemRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        svm_problem* problem2 = SvmProblemRepository().Get(loc2);
        SvmProblemDiff(problem1, problem2);
        DestroySvmProblem(problem1);
        DestroySvmProblem(problem2);
    }
}

void
DoCopySvmProblem()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyDiffSvmProblem);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc1.SetConcept(concepts[i]);
        svm_problem* problem = SvmProblemRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        SvmProblemRepository().Add(loc2, problem);
        delete problem;
    }
}

void
DoDeleteSvmProblem()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteSvmProblem);

    CmdOptions& options = CmdOptions::GetInstance();
    ModelLocator loc = GetModelLocator(options, true);
    KeywordList concepts = GetModelKeywordList(loc);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc.SetConcept(concepts[i]);
        SvmProblemRepository().Delete(loc);
    }
}

void
DoDumpModel()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpModel);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc = GetModelLocator(options, true);
    KeywordList concepts = GetModelKeywordList(loc);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc.SetConcept(concepts[i]);
        Svm* svm = SvmRepository().Get(loc);
        Dump(svm->GetModel(), -1);
        delete svm;
    }
}

void
DoDiffModel()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffModel);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        ILOG_INFO("Checking " << i << " = " << concepts[i]);
        loc1.SetConcept(concepts[i]);
        Svm* svm1 = SvmRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        Svm* svm2 = SvmRepository().Get(loc2);
        svm1->ReferenceDiff(svm2);
        delete svm1;
        delete svm2;
    }
}

void
DoCopyModel()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyDiffModel);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc1.SetConcept(concepts[i]);
        Svm* svm = SvmRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        SvmRepository().Add(loc2, svm);
        delete svm;
    }
}

void
DoDeleteModel()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteModel);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc = GetModelLocator(options, true);
    if (loc.GetConceptSet() == "ALL") // This is a bit rude, use with care
        return SvmRepository().Delete(loc);

    KeywordList concepts = GetModelKeywordList(loc);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc.SetConcept(concepts[i]);
        SvmRepository().Delete(loc);
    }
}

void
DoDiffAllParameterFile()
{
    typedef Util::PropertySet PropertySet;
    ILOG_VAR(Impala.Application.Repository.DoDiffAllParameterFile);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        ILOG_INFO("Checking " << i << " = " << concepts[i]);
        loc1.SetConcept(concepts[i]);
        std::vector<PropertySet*> v1 = *(AllParameterFileRepository().Get(loc1));
        loc2.SetConcept(concepts[i]);
        std::vector<PropertySet*> v2 = (*AllParameterFileRepository().Get(loc2));
        for (int p=0 ; p<v1.size() ; p++)
        {
            v1[p]->Diff(v2[p]);
            delete v1[p];
            delete v2[p];
        }
    }
}

void
DoDiffScoreFile()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffScoreFile);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        ILOG_INFO("Checking " << i << " = " << concepts[i]);
        loc1.SetConcept(concepts[i]);
        Util::PropertySet* scoreFile1 = ScoreFileRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        Util::PropertySet* scoreFile2 = ScoreFileRepository().Get(loc2);
        scoreFile1->Diff(scoreFile2);
        delete scoreFile1;
        delete scoreFile2;
    }
}

void
DoDumpFikSvm()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpFikSvm);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc = GetModelLocator(options, true);
    KeywordList concepts = GetModelKeywordList(loc);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        loc.SetConcept(concepts[i]);
        FikSvm* svm = FikSvmRepository().Get(loc);
        svm->Dump(i, 3);
        delete svm;
    }
}

void
DoDiffFikSvm()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffFikSvm);
    CmdOptions& options = CmdOptions::GetInstance();

    ModelLocator loc1 = GetModelLocator(options, true);
    ModelLocator loc2 = GetModelLocator(options, false);
    KeywordList concepts = GetModelKeywordList(loc1);
    for (int i=0 ; i<concepts.size() ; i++)
    {
        ILOG_INFO("Checking " << i << " = " << concepts[i]);
        loc1.SetConcept(concepts[i]);
        FikSvm* svm1 = FikSvmRepository().Get(loc1);
        loc2.SetConcept(concepts[i]);
        FikSvm* svm2 = FikSvmRepository().Get(loc2);
        svm1->Diff(svm2);
        delete svm1;
        delete svm2;
    }
}

// KernelMatrix part, work in progress...

KernelMatrixLocator
GetKernelMatrixLocator(CmdOptions& options, bool getSrcLoc)
{
    ILOG_VAR(Impala.Application.Repository.GetKernelMatrixLocator);

    int nrArg = 9;
    if (!getSrcLoc)
        nrArg++;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    int a = 1;
    String locString = options.GetArg(a++);
    String setName = options.GetArg(a++);
    bool isIndex = StringToBool(options.GetArg(a++));
    String walkType = options.GetArg(a++);
    String annoSet = options.GetArg(a++);
    String model = options.GetArg(a++);
    String feature = options.GetArg(a++);
    String container = options.GetArg(a++);
    if (!getSrcLoc)
        locString = options.GetArg(a++);
    if ((annoSet == "nil") || (annoSet == "empty"))
        annoSet = "";
    KernelMatrixLocator loc(locString, setName, isIndex, walkType, annoSet,
                            model, feature, container);
    loc.SetStartNode(0);
    loc.SetNodeCount(1);
    return loc;
}

void
DoDumpKernelMatrix()
{
    ILOG_VAR(Impala.Application.Repository.DoDumpKernelMatrix);
    CmdOptions& options = CmdOptions::GetInstance();

    KernelMatrixLocator loc = GetKernelMatrixLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        KernelMatrix* mat = KernelMatrixRepository().Get(loc);
        mat->Dump(0, options.GetInt("start"), options.GetInt("end"));
        delete mat;
    }
}

void
DoDiffKernelMatrix()
{
    ILOG_VAR(Impala.Application.Repository.DoDiffKernelMatrix);
    CmdOptions& options = CmdOptions::GetInstance();

    KernelMatrixLocator loc1 = GetKernelMatrixLocator(options, true);
    KernelMatrixLocator loc2 = GetKernelMatrixLocator(options, false);
    int a = 10;
    if (options.GetNrArg() > a)
    {
        String walkType2 = options.GetArg(a++);
        ILOG_INFO("walkType2 = " << walkType2);
        loc2.SetWalkType(walkType2);
    }
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        ILOG_INFO("Checking " << i << " = " << cur);
        loc1.SetContainer(cur);
        KernelMatrix* mat1 = KernelMatrixRepository().Get(loc1);
        loc2.SetContainer(cur);
        KernelMatrix* mat2 = KernelMatrixRepository().Get(loc2);
        mat1->Diff(mat2);
        delete mat2;
        delete mat1;
    }
}

void
DoCopyKernelMatrix()
{
    ILOG_VAR(Impala.Application.Repository.DoCopyKernelMatrix);
    CmdOptions& options = CmdOptions::GetInstance();

    KernelMatrixLocator loc1 = GetKernelMatrixLocator(options, true);
    KernelMatrixLocator loc2 = GetKernelMatrixLocator(options, false);
    int a = 10;
    if (options.GetNrArg() > a)
    {
        String walkType2 = options.GetArg(a++);
        ILOG_INFO("walkType2 = " << walkType2);
        loc2.SetWalkType(walkType2);
    }
    if (options.GetNrArg() > a)
    {
        bool writeMerged = StringToBool(options.GetArg(a++));
        ILOG_INFO("writeMerged = " << writeMerged);
        if (writeMerged)
            loc2.SetDoParts(3);
    }
    RawDataSet* dataSet = GetRawDataSet(options, true);
    String container = loc1.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        ILOG_INFO("Checking " << i << " = " << cur);
        loc1.SetContainer(cur);
        KernelMatrix* mat = KernelMatrixRepository().Get(loc1);
        loc2.SetContainer(cur);
        if (options.GetBool("sync"))
            KernelMatrixRepository().Sync(loc2, mat);
        else
            KernelMatrixRepository().Add(loc2, mat);
        delete mat;
    }
}

void
DoDeleteKernelMatrix()
{
    ILOG_VAR(Impala.Application.Repository.DoDeleteKernelMatrix);
    CmdOptions& options = CmdOptions::GetInstance();

    KernelMatrixLocator loc = GetKernelMatrixLocator(options, true);
    if (loc.GetFeature() == "ALL") // This is a bit rude, use with care
        return KernelMatrixRepository().Delete(loc);
    RawDataSet* dataSet = GetRawDataSet(options, false);
    if (!dataSet)
        return;

    String container = loc.GetContainer();
    int nrContainers = (container == "ALL") ? dataSet->GetNrContainers() : 1;
    for (int i=0 ; i<nrContainers ; i++)
    {
        String cur = (container == "ALL") ? dataSet->GetContainer(i) : container;
        loc.SetContainer(cur);
        KernelMatrixRepository().Delete(loc);
    }
}

void
DoKernelMatrix2SimilarityTable()
{
    ILOG_VAR(Impala.Application.Repository.DoKernelMatrix2SimilarityTable);
    CmdOptions& options = CmdOptions::GetInstance();

    KernelMatrixLocator kLoc = GetKernelMatrixLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    KernelMatrix* mat = KernelMatrixRepository().Get(kLoc);
    int topN = 100;
    int a = 9;
    if (options.GetNrArg() > a)
    {
        topN = atol(options.GetArg(a++));
        ILOG_INFO("topN = " << topN);
    }
    int start = options.GetInt("start");
    int end = options.GetInt("end");
    if (end == -1)
        end = mat->NrRow();

    std::vector<String> names;
    for (int i=start ; i<end ; i++)
    {
        String name = "kernelRow_" + MakeString(i);
        names.push_back(name);
    }
    SimilarityTableSet simSet(names, topN);
    QuidTable* quidTab = simSet.GetQuidTable();
    for (int i=0 ; i<topN ; i++)
        quidTab->Add(0); // quids should not be used!

    std::vector<String> tmpNames;
    tmpNames.push_back("tmp");
    int rowSize = mat->NrCol();
    SimilarityTableSet tmpSet(tmpNames, rowSize);
    QuidTable* tmpQuidTab = tmpSet.GetQuidTable();
    Core::Table::Copy(tmpQuidTab, mat->GetQuids());
    SimilarityTableSet::SimTableType* tmpSimTab = tmpSet.GetSimTable(0);
    for (int i=start ; i<end ; i++)
    {
        Real64* simData = tmpSimTab->GetColumn1()->GetData();
        int res = mat->GetRow(i, simData, rowSize);
        tmpSimTab->SetSize(res);

        int tabNr = i - start;
        SimilarityTableSet::SimTableType* simTab = simSet.GetSimTable(tabNr);
        SimilarityTableSet::RankTableType* rankTab = simSet.GetRankTable(tabNr);
        simTab->SetSize(topN);
        rankTab->SetSize(topN);
        Core::Column::SortDescendingTopN(tmpQuidTab->GetColumn1(),
                                         tmpSimTab->GetColumn1(),
                                         rankTab->GetColumn1(),
                                         simTab->GetColumn1());
    }
    delete mat;

    SimilarityTableSetLocator sLoc(kLoc, kLoc.GetIsIndex(), kLoc.GetWalkType(),
                                   "kernel.txt", "direct", kLoc.GetFeature(),
                                   kLoc.GetContainer());
    SimilarityTableSetRepository().Add(sLoc, &simSet);
}

void
DoKernelMatrix2Text()
{
    ILOG_VAR(Impala.Application.Repository.DoKernelMatrix2Text);
    CmdOptions& options = CmdOptions::GetInstance();

    KernelMatrixLocator kLoc = GetKernelMatrixLocator(options, true);
    RawDataSet* dataSet = GetRawDataSet(options, true);
    KernelMatrix* mat = KernelMatrixRepository().Get(kLoc);
    int topN = 100;
    int a = 9;
    if (options.GetNrArg() > a)
    {
        topN = atol(options.GetArg(a++));
        ILOG_INFO("topN = " << topN);
    }
    int start = options.GetInt("start");
    int end = options.GetInt("end");
    if (end == -1)
        end = mat->NrRow();

    RepositoryInFileSystem& repFS = RepositoryInFileSystem::GetInstance();
    String fName = dataSet->GetSetNameBase() + "_top" + MakeString(topN);
    if ((start != 0) || (end != mat->NrRow()))
        fName += "_rows" + MakeString(start) + "-" + MakeString(end);
    fName += ".txt";
    FileLocator loc(fName);
    Persistency::File file = repFS.GetFile(loc, true, false);
    if (!file.Valid())
        return;
    Util::IOBuffer* ioBuf = file.GetWriteBuffer();
    if (!ioBuf)
        return;

    int rowSize = mat->NrCol();
    QuidTable* allQuids = mat->GetQuids();
    SimilarityTableSet::SimTableType rowSims(rowSize);
    QuidTable topQuids(topN);
    SimilarityTableSet::SimTableType topSims(topN);
    ILOG_INFO("Processing " << start << "-" << end);
    for (int i=start ; i<end ; i++)
    {
        if (i % 100 == 0)
            ILOG_INFO("doing row " << i);
        Real64* simData = rowSims.GetColumn1()->GetData();
        int res = mat->GetRow(i, simData, rowSize);
        rowSims.SetSize(res);

        Core::Column::SortDescendingTopN(allQuids->GetColumn1(),
                                         rowSims.GetColumn1(),
                                         topQuids.GetColumn1(),
                                         topSims.GetColumn1());
        String s = MakeString(i) + " : ";
        for (int j=0 ; j<topN ; j++)
        {
            Quid q = topQuids.Get1(j);
            int fileId = dataSet->GetFileIdOfQuid(q);
            String path = dataSet->GetAsPath(fileId);
            s += "\"" + path + "\", ";
        }
        ioBuf->Puts(s);
    }
    ILOG_INFO("Done");
    delete ioBuf;
    delete mat;
}

// misc part

void
DoSyncFile()
{
    ILOG_VAR(Impala.Application.Repository.DoSyncFile);
    CmdOptions& options = CmdOptions::GetInstance();

    int nrArg = 4;
    if (options.GetNrArg() < nrArg)
    {
        ILOG_ERROR("Need at least " << nrArg << " parameters");
    }

    String locString1 = options.GetArg(1);
    String setName = options.GetArg(2);
    String fileName = options.GetArg(3);
    String locString2 = options.GetArg(4);

    FileLocator srcLoc(locString1, setName, fileName);
    FileLocator dstLoc(locString2, setName, fileName);
    RepositoryInFileSystem::GetInstance().CopyFile(srcLoc, dstLoc);
}

int
mainRepository(int argc, char* argv[])
{
    CmdOptions& options = CmdOptions::GetInstance();
    options.Initialise(true, false, true);
    options.AddOption(0, "start", "nr", "0");
    options.AddOption(0, "end", "nr", "-1");
    options.AddOption(0, "tolerance", "tolerance for difference in double values", "1e-14");
    options.AddOption(0, "sync", "", "0");
    String usageStr = "cmd = \n\n";
    usageStr += "  dumpimagesets\n";
    usageStr += "  dumpvideosets\n";
    usageStr += "  copyvideosets dstLocation\n";
    usageStr += "  dumpimageset location imageset.txt\n";
    usageStr += "  diffimageset location1 imageset.txt location2\n";
    usageStr += "  copyimageset location1 imageset.txt location2\n";
    usageStr += "  deleteimageset location imageset.txt\n";
    usageStr += "  syncimagefiles location1 imageset.txt location2\n";
    usageStr += "  dumpspecialimageset location videoset.txt imageset.txt\n";
    usageStr += "  diffspecialimageset location1 videoset.txt imageset.txt location2\n";
    usageStr += "  copyspecialimageset location1 videoset.txt imageset.txt location2\n";
    usageStr += "  dumpvideoset location videoset.txt\n";
    usageStr += "  diffvideoset location1 videoset.txt location2\n";
    usageStr += "  copyvideoset location1 videoset.txt location2\n";
    usageStr += "  deletevideoset location videoset.txt\n";
    usageStr += "  syncvideofiles location1 videoset.txt doInfoFiles location2\n";
    usageStr += "  deletevideoinfofiles location videoset.txt\n";
    usageStr += "  dumpframehashes location videoset.txt\n";
    usageStr += "  diffframehashes location1 videoset.txt location2\n";
    usageStr += "  copyframehashes location1 videoset.txt location2\n";
    usageStr += "  dumpsegmentation location videoset.txt\n";
    usageStr += "  diffsegmentation location1 videoset.txt location2\n";
    usageStr += "  copysegmentation location1 videoset.txt location2\n";
    usageStr += "  deletesegmentation location videoset.txt\n";
    usageStr += "  dumpkeyframes location videoset.txt\n";
    usageStr += "  diffkeyframes location1 videoset.txt location2\n";
    usageStr += "  copykeyframes location1 videoset.txt location2\n";
    usageStr += "  deletekeyframes location videoset.txt\n";
    usageStr += "  dumpstills location videoset.txt\n";
    usageStr += "  diffstills location1 videoset.txt location2\n";
    usageStr += "  copystills location1 videoset.txt location2\n";
    usageStr += "  deletestills location videoset.txt\n";
    usageStr += "  dumpquidtable location dataset.txt dir name\n";
    usageStr += "  diffquidtable location1 dataset.txt dir name location2\n";
    usageStr += "  copyquidtable location1 dataset.txt dir name location2\n";
    usageStr += "  deletequidtable location dataset.txt dir name\n";
    usageStr += "  dumpmpeg7shot location videoset.txt container\n";
    usageStr += "  diffmpeg7shot location1 videoset.txt container location2\n";
    usageStr += "  copympeg7shot location1 videoset.txt container location2\n";
    usageStr += "  deletempeg7shot location videoset.txt container\n";
    usageStr += "  deletempeg7annotation location videoset.txt container\n";
    usageStr += "  deletempeg7concept location videoset.txt container\n";
    usageStr += "  dumpimagearchive location dataset.txt isFrames container name.raw\n";
    usageStr += "  diffimagearchive location1 dataset.txt isFrames container name.raw location2\n";
    usageStr += "  copyimagearchive location1 dataset.txt isFrames container name.raw location2\n";
    usageStr += "  deleteimagearchive location dataset.txt isFrames container name.raw\n";
    usageStr += "  dumpimagearchivespecial location videoset.txt imageset.txt container name.raw\n";
    usageStr += "  diffimagearchivespecial location1 videoset.txt imageset.txt container name.raw location2\n";
    usageStr += "  copyimagearchivespecial location1 videoset.txt imageset.txt container name.raw location2\n";
    usageStr += "  deleteimagearchivespecial location videoset.txt imageset.txt container name.raw\n";
    usageStr += "  dumpfeaturetable location dataset.txt isCodebook isIndex walkType feature container\n";
    usageStr += "  difffeaturetable location1 dataset.txt isCodebook isIndex walkType feature container location2\n";
    usageStr += "  copyfeaturetable location1 dataset.txt isCodebook isIndex walkType feature container location2\n";
    usageStr += "  deletefeaturetable location dataset.txt isCodebook isIndex walkType feature container\n";
    usageStr += "  setquidsetfeaturetable location dataset.txt isCodebook isIndex walkType feature container setId\n";
    usageStr += "  dumpkeywordlist location videoset.txt concepts.txt\n";
    usageStr += "  diffkeywordlist location1 videoset.txt concepts.txt location2\n";
    usageStr += "  copykeywordlist location1 videoset.txt concepts.txt location2\n";
    usageStr += "  dumpannotationtable location dataset.txt quidClass concepts.txt keyword\n";
    usageStr += "  diffannotationtable location1 dataset.txt quidClass concepts.txt keyword location2\n";
    usageStr += "  copyannotationtable location1 dataset.txt quidClass concepts.txt keyword location2\n";
    usageStr += "  deleteannotationtable location dataset.txt quidClass concepts.txt keyword\n";
    usageStr += "  mergeannotationtable location dataset.txt quidClass concepts.txt outputkeyword [inputkeyword]+ \n";
    usageStr += "  dumpannotationtableset location dataset.txt quidClass concepts.txt doTable [maxVidId maxPos maxNeg]\n";
    usageStr += "  diffannotationtableset location1 dataset.txt quidClass concepts.txt location2\n";
    usageStr += "  copyannotationtableset location1 dataset.txt quidClass concepts.txt location2\n";
    usageStr += "  deleteannotationtableset location dataset.txt quidClass concepts.txt\n";
    usageStr += "  mergeannotationtableset [inputlocation inputdataset.txt inputquidClass inputconcepts.txt]+ outputlocation outputdataset.txt outputquidClass outputconcepts.txt \n";
    usageStr += "  correctannotationtableset location dataset.txt quidClass conceptsOriginal.txt conceptsCorrections.txt conceptsResult.txt\n";
    usageStr += "  selectannotationtableset location dataset.txt quidClass concepts.txt concept iteration\n";
    usageStr += "  indexannotatedfeatures location dataset.txt quidClass concepts.txt feature\n";
    usageStr += "  dumpfold location dataset.txt quidClass concepts.txt keyword foldType foldNr\n";
    usageStr += "  difffold location1 dataset.txt quidClass concepts.txt keyword foldType foldNr location2\n";
    usageStr += "  copyfold location1 dataset.txt quidClass concepts.txt keyword foldType foldNr location2\n";
    usageStr += "  deletefold location dataset.txt quidClass concepts.txt keyword foldType foldNr\n";
    usageStr += "  dumpsimilaritytableset location dataset.txt isIndex walkType conceptSet.txt model feature container [dumpAsMatrix dumpQuid]\n";
    usageStr += "  diffsimilaritytableset location1 dataset.txt isIndex walkType conceptSet.txt model feature container location2\n";
    usageStr += "  copysimilaritytableset location1 dataset.txt isIndex walkType conceptSet.txt model feature container location2\n";
    usageStr += "  deletesimilaritytableset location dataset.txt isIndex walkType conceptSet.txt model feature container\n";
    usageStr += "  mergesimilaritytableset location1 dataset.txt isIndex walkType dstConceptSet.txt model feature container srcConceptSet1 srcConceptSet2 [srcConceptSetN]\n";
    usageStr += "  dumpbestfile location dataset.txt conceptSet.txt model feature concept\n";
    usageStr += "  diffbestfile location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  copybestfile location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  dumpsvmproblem location dataset.txt conceptSet.txt model feature concept\n";
    usageStr += "  diffsvmproblem location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  copysvmproblem location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  deletesvmproblem location dataset.txt conceptSet.txt model feature concept\n";
    usageStr += "  dumpmodel location dataset.txt conceptSet.txt model feature concept\n";
    usageStr += "  diffmodel location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  copymodel location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  deletemodel location dataset.txt conceptSet.txt model feature concept\n";
    usageStr += "  diffallparameterfile location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  diffscorefile location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  dumpfiksvm location dataset.txt conceptSet.txt model feature concept\n";
    usageStr += "  difffiksvm location1 dataset.txt conceptSet.txt model feature concept location2\n";
    usageStr += "  dumpkernelmatrix location dataset.txt isIndex walkType annoSet.txt model feature container\n";
    usageStr += "  diffkernelmatrix location1 dataset.txt isIndex walkType annoSet.txt model feature container location2\n";
    usageStr += "  copykernelmatrix location1 dataset.txt isIndex walkType annoSet.txt model feature container location2 [walkType2] [writeMerged2]\n";
    usageStr += "  deletekernelmatrix location dataset.txt isIndex walkType annoSet.txt model feature container\n";
    usageStr += "  kernelmatrix2similaritytable location dataset.txt isIndex walkType annoSet.txt model feature container [topN]\n";
    usageStr += "  kernelmatrix2text location dataset.txt isIndex walkType annoSet.txt model feature container [topN]\n";
    usageStr += "  syncfile location1 dataset.txt filename location2\n";
    if (! options.ParseArgs(argc, argv, usageStr, 1))
        return 1;

    ILOG_VAR(Impala.Application.Repository.mainRepository);

    String cmd = options.GetArg(0);
    ILOG_DEBUG("cmd = " << cmd);
    if (cmd == "dumpimagesets")
        DoDumpImageSets();
    else if (cmd == "dumpvideosets")
        DoDumpVideoSets();
    else if (cmd == "copyvideosets")
        DoCopyVideoSets();
    else if (cmd == "dumpimageset")
        DoDumpImageSet();
    else if (cmd == "diffimageset")
        DoDiffImageSet();
    else if (cmd == "copyimageset")
        DoCopyImageSet();
    else if (cmd == "deleteimageset")
        DoDeleteImageSet();
    else if (cmd == "syncimagefiles")
        DoSyncImageFiles();
    else if (cmd == "dumpimagesetspecial")
        DoDumpImageSetSpecial();
    else if (cmd == "diffimagesetspecial")
        DoDiffImageSetSpecial();
    else if (cmd == "copyimagesetspecial")
        DoCopyImageSetSpecial();
    else if (cmd == "deleteimagesetspecial")
        DoDeleteImageSetSpecial();
    else if (cmd == "dumpvideoset")
        DoDumpVideoSet();
    else if (cmd == "diffvideoset")
        DoDiffVideoSet();
    else if (cmd == "copyvideoset")
        DoCopyVideoSet();
    else if (cmd == "deletevideoset")
        DoDeleteVideoSet();
    else if (cmd == "syncvideofiles")
        DoSyncVideoFiles();
    else if (cmd == "deletevideoinfofiles")
        DoDeleteVideoInfoFiles();
    else if (cmd == "dumpframehashes")
        DoDumpFrameHashes();
    else if (cmd == "diffframehashes")
        DoDiffFrameHashes();
    else if (cmd == "copyframehashes")
        DoCopyFrameHashes();
    else if (cmd == "dumpsegmentation")
        DoDumpSegmentation();
    else if (cmd == "diffsegmentation")
        DoDiffSegmentation();
    else if (cmd == "copysegmentation")
        DoCopySegmentation();
    else if (cmd == "deletesegmentation")
        DoDeleteSegmentation();
    else if (cmd == "dumpkeyframes")
        DoDumpKeyframes();
    else if (cmd == "diffkeyframes")
        DoDiffKeyframes();
    else if (cmd == "copykeyframes")
        DoCopyKeyframes();
    else if (cmd == "deletekeyframes")
        DoDeleteKeyframes();
    else if (cmd == "dumpstills")
        DoDumpStills();
    else if (cmd == "diffstills")
        DoDiffStills();
    else if (cmd == "copystills")
        DoCopyStills();
    else if (cmd == "deletestills")
        DoDeleteStills();
    else if (cmd == "dumpquidtable")
        DoDumpQuidTable();
    else if (cmd == "diffquidtable")
        DoDiffQuidTable();
    else if (cmd == "copyquidtable")
        DoCopyQuidTable();
    else if (cmd == "deletequidtable")
        DoDeleteQuidTable();
    else if (cmd == "dumpmpeg7shot")
        DoDumpMpeg7Shot();
    else if (cmd == "diffmpeg7shot")
        DoDiffMpeg7Shot();
    else if (cmd == "copympeg7shot")
        DoCopyMpeg7Shot();
    else if (cmd == "deletempeg7shot")
        DoDeleteMpeg7Shot();
    else if (cmd == "deletempeg7annotation")
        DoDeleteMpeg7Annotation();
    else if (cmd == "deletempeg7concept")
        DoDeleteMpeg7Concept();
    else if (cmd == "dumpimagearchive")
        DoDumpImageArchive();
    else if (cmd == "diffimagearchive")
        DoDiffImageArchive();
    else if (cmd == "copyimagearchive")
        DoCopyImageArchive();
    else if (cmd == "deleteimagearchive")
        DoDeleteImageArchive();
    else if (cmd == "dumpimagearchivespecial")
        DoDumpImageArchiveSpecial();
    else if (cmd == "diffimagearchivespecial")
        DoDiffImageArchiveSpecial();
    else if (cmd == "copyimagearchivespecial")
        DoCopyImageArchiveSpecial();
    else if (cmd == "deleteimagearchivespecial")
        DoDeleteImageArchiveSpecial();
    else if (cmd == "dumpfeaturetable")
        DoDumpFeatureTable();
    else if (cmd == "difffeaturetable")
        DoDiffFeatureTable();
    else if (cmd == "copyfeaturetable")
        DoCopyFeatureTable();
    else if (cmd == "deletefeaturetable")
        DoDeleteFeatureTable();
    else if (cmd == "setquidsetfeaturetable")
        DoSetQuidSetFeatureTable();
    else if (cmd == "dumpkeywordlist")
        DoDumpKeywordList();
    else if (cmd == "diffkeywordlist")
        DoDiffKeywordList();
    else if (cmd == "copykeywordlist")
        DoCopyKeywordList();
    else if (cmd == "dumpannotationtable")
        DoDumpAnnotationTable();
    else if (cmd == "diffannotationtable")
        DoDiffAnnotationTable();
    else if (cmd == "copyannotationtable")
        DoCopyAnnotationTable();
    else if (cmd == "deleteannotationtable")
        DoDeleteAnnotationTable();
    else if (cmd == "mergeannotationtable")
        DoMergeAnnotationTable();
    else if (cmd == "dumpannotationtableset")
        DoDumpAnnotationTableSet();
    else if (cmd == "diffannotationtableset")
        DoDiffAnnotationTableSet();
    else if (cmd == "copyannotationtableset")
        DoCopyAnnotationTableSet();
    else if (cmd == "deleteannotationtableset")
        DoDeleteAnnotationTableSet();
    else if (cmd == "mergeannotationtableset")
        DoMergeAnnotationTableSet();
    else if (cmd == "correctannotationtableset")
        DoCorrectAnnotationTableSet();
    else if (cmd == "selectannotationtableset")
        DoSelectAnnotationTableSet();
    else if (cmd == "indexannotatedfeatures")
        DoIndexAnnotatedFeatures();
    else if (cmd == "dumpfold")
        DoDumpFold();
    else if (cmd == "difffold")
        DoDiffFold();
    else if (cmd == "copyfold")
        DoCopyFold();
    else if (cmd == "deletefold")
        DoDeleteFold();
    else if (cmd == "dumpsimilaritytableset")
        DoDumpSimilarityTableSet();
    else if (cmd == "diffsimilaritytableset")
        DoDiffSimilarityTableSet();
    else if (cmd == "copysimilaritytableset")
        DoCopySimilarityTableSet();
    else if (cmd == "deletesimilaritytableset")
        DoDeleteSimilarityTableSet();
    else if (cmd == "mergesimilaritytableset")
        DoMergeSimilarityTableSet();
    else if (cmd == "dumpbestfile")
        DoDumpBestFile();
    else if (cmd == "diffbestfile")
        DoDiffBestFile();
    else if (cmd == "copybestfile")
        DoCopyBestFile();
    else if (cmd == "dumpsvmproblem")
        DoDumpSvmProblem();
    else if (cmd == "diffsvmproblem")
        DoDiffSvmProblem();
    else if (cmd == "copysvmproblem")
        DoCopySvmProblem();
    else if (cmd == "deletesvmproblem")
        DoDeleteSvmProblem();
    else if (cmd == "dumpmodel")
        DoDumpModel();
    else if (cmd == "diffmodel")
        DoDiffModel();
    else if (cmd == "copymodel")
        DoCopyModel();
    else if (cmd == "deletemodel")
        DoDeleteModel();
    else if (cmd == "diffallparameterfile")
        DoDiffAllParameterFile();
    else if (cmd == "diffscorefile")
        DoDiffScoreFile();
    else if (cmd == "dumpfiksvm")
        DoDumpFikSvm();
    else if (cmd == "difffiksvm")
        DoDiffFikSvm();
    else if (cmd == "dumpkernelmatrix")
        DoDumpKernelMatrix();
    else if (cmd == "diffkernelmatrix")
        DoDiffKernelMatrix();
    else if (cmd == "copykernelmatrix")
        DoCopyKernelMatrix();
    else if (cmd == "deletekernelmatrix")
        DoDeleteKernelMatrix();
    else if (cmd == "kernelmatrix2similaritytable")
        DoKernelMatrix2SimilarityTable();
    else if (cmd == "kernelmatrix2text")
        DoKernelMatrix2Text();
    else if (cmd == "syncfile")
        DoSyncFile();
    else ILOG_ERROR("Unknown cmd : " << cmd);

    int nrOfErrors = ILOG_ERROR_COUNT;
    ILOG_INFO_HEADNODE("Done, total nr error = " << nrOfErrors);
    return nrOfErrors;
}

} // namespace Repository
} // namespace Application
} // namespace Impala

int
main(int argc, char* argv[])
{

#ifdef NO_DIALOGONCRASH
#ifdef WIN32
    DWORD oldMode = ::SetErrorMode(SEM_NOGPFAULTERRORBOX);
    ::SetErrorMode(oldMode | SEM_NOGPFAULTERRORBOX);
#endif
#endif

    return Impala::Application::Repository::mainRepository(argc, argv);
}

---