|| || || || || || || ||

---

TrecEngine.h

Go to the documentation of this file.

/***********************************************************************************
 *
 * Application: Investigator's Dashboard (MultimediaN project 6)
 * 
 * Semantic Video Search Engine
 *
 * Primary author(s): Ork
 *                  : 
 *                  : 
 ***********************************************************************************/

#ifndef Impala_Application_IDash_TrecEngine_h
#define Impala_Application_IDash_TrecEngine_h

#include "Basis/Logger.h"
#include "Basis/CmdOptions.h"

#include "Core/Trec/ThreadSet.h"
#include "Core/VideoSet/VideoSet.h"
#include "Core/VideoSet/Segmentation.h"
#include "Core/VideoSet/Stills.h"
#include "Core/VideoSet/SegmentationDocument.h"
#include "Core/VideoSet/Keyframes.h"
#include "Core/ImageSet/ImageSet.h"
#include "Core/ImageSet/MakeImageSet.h"
#include "Core/VideoSet/MakeVideoSet.h"

#include "Application/idash/IDashQueryEngine.h"

namespace Impala {
namespace Application {
namespace IDash {

class TrecEngine
{
public:
    typedef Impala::Core::Table::TableTem
            <Impala::Core::Column::ColumnTem<Real64> >      SimTableType;
    typedef Core::Trec::IDashQueryEngine                    IDashQueryEngine;
    typedef Core::Trec::KeyframeResult                      KeyframeResult;

    TrecEngine()
    {
        ILOG_SYSTEM("Loading Semantic Video Search Engine...");

        ILOG_STARTACTION("loading configuration", ILOG_LEVEL_INFO);
        LoadConfig();
        ILOG_ENDACTION("loading configuration");

        ILOG_STARTACTION("loading data", ILOG_LEVEL_INFO);
        LoadData();
        ILOG_ENDACTION("loading data");

        mQueryEngine = new IDashQueryEngine(ThreadSet());

    }


    // Engine Retrieval Functions:
    //
    // these functions map the engine to the GUI
    //
    Core::Trec::ThreadSet*
    ThreadSet()
    {
        return mThreadSet;
    }

    Core::VideoSet::SegmentationDocument*
    SegmentationDocument()
    {
        return mSegDoc;
    }

    Core::Table::SimilarityTableSet*
    GetConcepts()
    {
        return mConcepts;
    }

    Core::VideoSet::Segmentation*
    GetShots()
    {
        return mShots;
    }
        
    Core::VideoSet::Keyframes*
    GetKeyframes()
    {
        return mKeyframes;
    }

    Core::ImageSet::ImageSet*
    GetKeyframeImageSet()
    {
        return mKeyframeImages;
    }

    double
    GetTableSimilarityScore(int table, int row)
    {
        SimTableType *t = mConcepts->GetSimTable(table);
        return (double)t->Get1(row);
    }

    std::string
    GetTableName(int table)
    {
        return mConcepts->GetName(table);
    }

    int
    GetNrTables()
    {
        return mConcepts->NrTables();
    }

    int ImWidth()        { return mThumbWidth; }
    int ImHeight()       { return mThumbHeight; }
    int ImScale()        { return mThumbScale; }


    int
    GetNumberOfStills(int row)
    {
        if (!OpenStill(row)) return 0;
        return mStillEnd;
    }

    Core::Array::Array2dVec3UInt8*
    GetStill(int row, int frame)
    {
        ILOG_DEBUG("GetStill " << row << " " << frame);
        if (!OpenStill(row)) return 0;

        return mStillImages->GetImage(mStillStart + frame);

    }

    Core::Array::Array2dVec3UInt8*
    GetNextStill(int row)
    {
        if (!OpenStill(row)) return 0;

        mStillFrame++;
        if (mStillFrame >= mStillEnd)
            mStillFrame = 0;

        return GetStill(row, mStillFrame);
    }

    bool
    OpenStill(int row)
    {
        if (!mStills || !mStillImages)
            return false;

        if (row != mStillsRow)
        {
            mStillsRow = row;
            mStillFrame = 0;
            mStillStart = mStills->GetFirstStillShot(row);
            mStillEnd   = mStills->GetNrStillsShot(row);
            ILOG_DEBUG("Opening stills for ID=" << row << " " << mStillEnd << " frames.");
        }
        return true;
    }

    // Engine Actions:
    //
    void
    PerformQuery(std::list<std::string> querycomponents)
    {

        ILOG_DEBUG("Preparing query...");
        mQueryEngine->Clear();
        mQueryEngine->ProcessIDashQueryList(querycomponents);

        if (mQueryEngine->HasComponents()) {
            mThreadSet->RemoveThread("shots_initialquery");
    
            ILOG_DEBUG("Performing query...");
            mQueryEngine->GenerateThreadFromQuery("initialquery");
        }
    }

    void
    PerformVisualQuery(std::string threadname, int keyframe)
    {
        ILOG_DEBUG("Preparing visual query for " << threadname << " keyframe " << keyframe << "...");
        mQueryEngine->Clear();
        std::list<KeyframeResult> visuals = mVisualQueryEngine->QueryVisual(threadname.substr(7), keyframe, true);
        mThreadSet->RemoveThread("shots_initialquery");
        mThreadSet->AddThreadShots("initialquery", visuals);
    }

private:

    void
    LoadConfig()
    {
        CmdOptions& options = CmdOptions::GetInstance();
        std::string yearStr = options.GetString("year");
        mYear = atol(yearStr);

        mVideoSetName = options.GetString("videoSet");
        mImageStills = options.GetBool("imageStills");
        mKeyServer = options.GetString("keyServer");
        mConceptSetName = options.GetString("conceptSet");
        mConceptModel = options.GetString("conceptModel");
        mConceptFeature = options.GetString("conceptFeature");

        mStillsRow = -1;
        mStillFrame = 0;

        mStills = 0; 
        mStillImages = 0;

    }

    void
    LoadData()
    {
        typedef Core::ImageSet::ImageSet ImageSet;

        CmdOptions& options = CmdOptions::GetInstance();

        // read the keyframes
        ILOG_STARTACTION("reading keyframes", ILOG_LEVEL_INFO);;
        ImageSet* imSetKeyfr =
            Core::ImageSet::MakeImageSet(options.GetString("imageSetKeyframes"));

        if (imSetKeyfr)
            imSetKeyfr->SetImageSrc(false, false, true);
        else
            ILOG_WARN( "No keyframes available" );
        ILOG_ENDACTION("reading keyframes");

        mKeyframeImages = imSetKeyfr;

        // read the thumbnails
        ILOG_STARTACTION("reading thumbnails", ILOG_LEVEL_INFO);;
        ImageSet* imSetThumb = 
            Core::ImageSet::MakeImageSet(options.GetString("imageSetThumbnails"));
        if (imSetThumb)
        {
            imSetThumb->SetImageSrc(! options.GetBool("noArchive"),
                                    options.GetBool("imFileArchive"),
                                    options.GetBool("imSplitArchive"));

            //Core::Array::Array2dVec3UInt8* im = imSetThumb->GetImage(0);
            //mThumbWidth = im->CW();
            //mThumbHeight = im->CH();
            //delete im;

            // we set this manually to allow for different sizes:
            mThumbWidth = 360 / 1.5; //im->CW();
            mThumbHeight = 240 / 1.5; //im->CH();

            mKeyframeImages = imSetThumb;

            mThumbScale = 1.0;
        }
        else
            ILOG_WARN( "No thumbnails available" );
        ILOG_ENDACTION("reading thumbnails");
    

        // read the stills
        ILOG_STARTACTION("reading stills", ILOG_LEVEL_INFO);;
        ImageSet* imSetStill =
            Core::ImageSet::MakeImageSet(options.GetString("imageSetStills"));
        if (imSetStill)
            imSetStill->SetImageSrc(false, false, true);
        else
            ILOG_WARN( "No stills available" );
        ILOG_ENDACTION("reading stills");

        mStillImages = imSetStill;

        // read the videoset
        ILOG_STARTACTION("reading videoset", ILOG_LEVEL_INFO);
        mVidSet = Core::VideoSet::MakeVideoSet(mVideoSetName);
        ILOG_ENDACTION("reading videoset");

        // read the shots
        ILOG_STARTACTION("reading shots", ILOG_LEVEL_INFO);
        mShots = new Core::VideoSet::Segmentation(mVidSet, "segmentation");
        ILOG_ENDACTION("reading shots");

        // read the keyframe set
        ILOG_STARTACTION("reading keyframe set", ILOG_LEVEL_INFO);
        mKeyframes = new Core::VideoSet::Keyframes(mVidSet, "keyframes");
        ILOG_ENDACTION("reading keyframe set");

        ILOG_INFO("Found " << mKeyframes->GetNrKeyframes() << " keyframes.");

        ILOG_STARTACTION("reading stills set", ILOG_LEVEL_INFO);
        mStills = new Core::VideoSet::Stills(mVidSet, "stills");
        ILOG_ENDACTION("reading stills set");

        //mSegDoc = new Core::VideoSet::SegmentationDocument
        //    (mVidSet, mShots, mKeyframes, imSetKeyfr, imSetThumb, imSetStill);

  /*      mThreadSet = new Core::Trec::ThreadSet(mShots, mKeyframes, imSetThumb, imSetKeyfr);
        mThreadSet->AddThreadTime();
        mThreadSet->AddThreadBookmarked(mSegDoc->GetBookmarked());
        mThreadSet->AddThreadHistory();
*/
        // read the concepts
        if (mConceptSetName != "") {
            ILOG_STARTACTION("reading concept ranking", ILOG_LEVEL_INFO);
            try {
                mConcepts =
                    Core::Table::SimilarityTableSet::MakeFromFile(mVidSet,
                                                                  mConceptSetName, 
                                                                  mConceptModel, 
                                                                  mConceptFeature);
            } catch (std::length_error) {
                ILOG_ERROR("MakeFromFile threw std::length_error");
                ILOG_ERROR("Probable cause: there are no concepts in set '" << mConceptSetName << "'.");
                mConceptSetName = "";
                mConcepts = 0;
            }

            if (mConcepts) {
                ILOG_INFO("Found " << mConcepts->NrTables() << " concepts for this set.");
                for (int i=0; i< mConcepts->NrTables(); i++) {
                    //mThreadSet->AddThreadRank(mConceptRanking, i);
                }
            }


            ILOG_ENDACTION("reading concept ranking");

        } else {
            ILOG_WARN("concept reading disabled. No concepts available.");
            mConcepts = 0;
        }
/*
        // read the visual similarities:
        ILOG_STARTACTION("adding visual similarity", ILOG_LEVEL_INFO);
        
//         ILOG_STARTACTION("adding visual similarity - Lab histograms", ILOG_LEVEL_INFO);
//         mThreadSet->AddThreadVisualSimilarity("labhistogram");
//         ILOG_ENDACTION("adding visual similarity - Lab histograms");

        ILOG_STARTACTION("adding visual similarity - Weibull", ILOG_LEVEL_INFO);
        mThreadSet->AddThreadVisualSimilarity("vissem");
        ILOG_ENDACTION("adding visual similarity - Weibull");

        ILOG_STARTACTION("adding visual similarity - Gabor", ILOG_LEVEL_INFO);
        mThreadSet->AddThreadVisualSimilarity("vissemgabor");
        ILOG_ENDACTION("adding visual similarity - Gabor");

//         ILOG_STARTACTION("adding visual similarity - Fusion", ILOG_LEVEL_INFO);
//         mThreadSet->AddThreadVisualSimilarity("fusionvissemgabor");
//         ILOG_ENDACTION("adding visual similarity - Fusion");

        ILOG_ENDACTION("adding visual similarity");

*/
        //mVisualQueryEngine = new Core::Trec::VisualQueryEngine(mThreadSet, 400);


    }

    // Data
    Core::VideoSet::VideoSet*        mVidSet;
    Core::VideoSet::Segmentation*    mShots;
    Core::VideoSet::Keyframes*       mKeyframes;
    Core::Trec::ThreadSet*           mThreadSet;
    Core::Table::SimilarityTableSet* mConcepts;
    Core::VideoSet::SegmentationDocument* mSegDoc; // controls current shot

    Core::ImageSet::ImageSet*        mKeyframeImages;
    Core::ImageSet::ImageSet*        mStillImages;
    Core::VideoSet::Stills*          mStills;

    int mStillsRow, mStillStart, mStillEnd, mStillFrame;


    int                             mThumbWidth;
    int                             mThumbHeight;
    double                          mThumbScale;
    std::string                     mConceptSetName;
    std::string                     mConceptModel;
    std::string                     mConceptFeature;

    // Computation
    IDashQueryEngine*               mQueryEngine;
    Core::Trec::VisualQueryEngine*  mVisualQueryEngine;

    // Configuration
    std::string                     mVideoSetName;
    bool                            mImageStills;
    std::string                     mKeyServer;
    int                             mYear;


    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(TrecEngine, Application.IDash);



}
}
}

#endif

---