|| || || || || || || ||

---

IDashQueryEngine.h

Go to the documentation of this file.

#ifndef Application_IDash_IDashQueryEngine_h
#define Application_IDash_IDashQueryEngine_h

#include "Util/StringParser.h"
#include "Core/Trec/ThreadSet.h"
#include "Core/Trec/ShotResult.h"
#include "Core/Trec/VisualQueryEngine.h"
#include "Core/Table/SimilarityTableSet.h"
#include "Core/Table/TableTem.h"
#include "Core/Table/Sort.h"

/* TODO:
  remove all std::list / std::map usage and replace with
  Core::Impala::Table constructions.
*/

namespace Impala {
//namespace Application {
//namespace IDash {
namespace Core {
namespace Trec {

// RvB: Should not this typedef be made local to the class?
typedef Core::Table::TableTem
        <Core::Column::ColumnTem<Int32>,
         Core::Column::ColumnTem<double> >          ShotWithWeight;

//using namespace Core::Trec;
//using namespace Core;

class IDashQueryEngine
{
public:
    typedef std::list<ShotResult>                   ShotResultList;

    IDashQueryEngine(ThreadSet *set)
    {
        Init(set);
    }

    void ProcessIDashQueryList(std::list<String> qlist)
    {
        std::list<String>::iterator qlisti;
        for (qlisti=qlist.begin(); qlisti!=qlist.end(); qlisti++)
        {
            String q = *qlisti;
            ILOG_DEBUG("Processing query component: " << q);
            // query starts with CONCEPT:
            if (q.find("CONCEPT:") == 0 )
                AddConcept(q.substr(8));
        }
    }

    bool HasComponents()
    {
        return mSelectedConcepts.size() > 0 || mSelectedShots.size() > 0;
    }

    void AddConcept(String conceptname, double amount = 1.0)
    {
        ILOG_DEBUG( "QueryEngine::AddConcept(" << conceptname << ")..." );
        if (mSelectedConcepts.find(conceptname) != mSelectedConcepts.end())
        {
            ILOG_DEBUG( " already selected. skipping." );
            return;
        }
        mSelectedConcepts[conceptname] = amount;
    }

    void RemoveConcept(String conceptname)
    {
        if (mSelectedConcepts.find(conceptname) != mSelectedConcepts.end())
            mSelectedConcepts.erase(conceptname);
    }

    void AddShots(String identifier, std::list<KeyframeResult> keyframes)
    {
        ILOG_DEBUG("QueryEngine::AddShots(" << identifier << ", " <<
                   keyframes.size() << " keyframes)..." );
        if (mSelectedShots.find(identifier) != mSelectedShots.end())
        {
            ILOG_DEBUG( " already selected. replacing." );
        }

        ShotResultList shots = ConvertToShotResult(keyframes);
        mSelectedShots[identifier] = shots;
    }

    void AddShots(String identifier, ShotResultList shots)
    {
        ILOG_DEBUG("QueryEngine::AddShots(" << identifier << ", " <<
                   shots.size() << " shots)..." );
        if (mSelectedShots.find(identifier) != mSelectedShots.end())
        {
            ILOG_DEBUG( " already selected. replacing." );
        }
        mSelectedShots[identifier] = shots;
    }

    void RemoveShots(String identifier)
    {
        if (mSelectedShots.find(identifier) != mSelectedShots.end())
            mSelectedShots.erase(identifier);
    }

    void Clear()
    {
        mSelectedShots.clear();
        mSelectedConcepts.clear();
    }


    // GOAL of this function
    // - create individual threads for special searches
    //   (text, query by region, ...)
    // - create a new thread 'threadname' combining all selected from QueryGui.
    void GenerateThreadFromQuery(String threadname)
    {
        ILOG_SYSTEM("generating thread from query: " << threadname << ":");
        std::list<ShotResultList > concepts;
        std::map<String, double>::iterator c;
        for (c = mSelectedConcepts.begin(); c != mSelectedConcepts.end(); c++)
        {
            ILOG_SYSTEM("contains concept: "<<c->first<<" ("<<c->second<<")");
            ShotResultList concept = ConvertToShotResult(mThreadSet, c->first);
            concepts.push_back(concept);
        }

        std::map<String, ShotResultList >::iterator s;
        for (s = mSelectedShots.begin(); s != mSelectedShots.end(); s++)
        {
            ILOG_DEBUG("contains shots: " << s->first << " (" <<
                       s->second.size() << " shots)" );
            // make threads for each individual text search or QbyR search:
            if (mThreadSet->GetThreadByName(s->first))
            {
                ILOG_DEBUG("textual thread " << s->first <<
                           " already exists. skipping." );
            }
            else
                mThreadSet->AddThreadShots(s->first, s->second);
            concepts.push_back(s->second);
        }
        ILOG_SYSTEM("end of content list");

        if (concepts.size() == 0)
        {
            ILOG_WARN("QueryEngine::GenerateThreadFromQuery: " <<
                      "no content to generate. Abort." );
            return;
        }

        ShotResultList merged = MergeResults(concepts);
        mThreadSet->AddThreadShots(threadname, merged);
    }


    /* note: true = keep, false = delete */
    bool FilterByLanguage(ShotResult in)
    {
        if (!mDoLanguageFilter) return true;
        int shotID = in.shotid;
        int language = GetLanguage(shotID);
        if (language == LANGUAGE_ENGLISH && mLanguageAllowEng) return true;
        if (language == LANGUAGE_CHINESE && mLanguageAllowChi) return true;
        if (language == LANGUAGE_ARABIC  && mLanguageAllowArb) return true;
        return false;
   }

    ShotResultList
    ProcessList(ShotResultList in, bool (IDashQueryEngine::*f)(ShotResult))
    {
        ShotResultList out;
        for (ShotResultList::iterator i=in.begin();i!=in.end();i++)
            if((*this.*f)(*i))
                out.push_back(*i);
        return out;
    }

    ShotResultList
    MergeResults(std::list<ShotResultList > lists,int mergeMethod=MERGE_DEFAULT)
    {
        switch (mergeMethod)
        {
            case MERGE_ROUNDROBIN:
                return MergeResultsByRoundRobin(lists);
            case MERGE_BORDAFUSION:
                return MergeResultsByBordaFusion(lists);
            default:
                ILOG_ERROR("QueryEngine::MergeResults: invalid merge function: "
                           << mergeMethod);
                return MergeResultsByBordaFusion(lists);
        }
    }

    ShotResultList
    MergeResultsByBordaFusion(std::list<ShotResultList > lists)
    {
        ShotResultList          merged;
        std::map<int, double>   fusionvalue;
        std::map<int, double>   weights;

        // note:
        // using fixed N = 1000 here, because of unequal list lengths.
        // We assume the number of shots required (for trecvid) = max. 1000.
        int N = 1000;

        // phase 1: assign equal weights to all lists
        // upgrade: assign unequal weights. :-)
        int listidx = 0;
        std::list<ShotResultList >::iterator i;
        for (i = lists.begin(); i != lists.end(); i++)
            weights[listidx++] = 1.0 / (double)lists.size();

        // phase 2: itterate through all shots from all lists,
        // and increment fusionvalue
        listidx = 0;
        for (i = lists.begin(); i != lists.end(); i++) {

            double weight = weights[listidx++];
            int rank = 0;
            for (ShotResultList::iterator j = i->begin(); j != i->end(); j++)
            {
                int shotid = (*j).shotid;
                double weightedbordacc = weight * (double)(N - rank++);
                if (fusionvalue.find(shotid) == fusionvalue.end())
                    fusionvalue[shotid] = weightedbordacc;
                else
                    fusionvalue[shotid] = fusionvalue[shotid] + weightedbordacc;
            }
        }

        // phase 3: sort by highest fusionvalues, return top N
        // note: all this std::map//list//Core::Table swapping might be a
        // wee bit expensive....
        ShotWithWeight* sww = new ShotWithWeight(fusionvalue.size());
        std::map<int, double>::iterator mapIt;
        for (mapIt = fusionvalue.begin(); mapIt != fusionvalue.end(); mapIt++)
            sww->Add(mapIt->first, mapIt->second);

        double maxscore = sww->Get2(sww->Size() - 1);
        int rank = 1;
        //Table::Sort(sww, 2); // sort on weights
        Table::Sort(sww, 2, true); // ascending
        for (int i=sww->Size()-1; i>=0; i--)
        {
            ShotResult t;
            t.shotid = sww->Get1(i);
            t.score = sww->Get2(i) / maxscore;
            t.rank = rank++;
            merged.push_back(t);
        }
        return merged;
    }

    ShotResultList
    MergeResultsByRoundRobin(std::list<ShotResultList > lists)
    {
        ShotResultList      merged;
        std::map<int, int>  counts;
        std::list<int>      lengths;
        std::list<int>      positions;

        std::list<ShotResultList::iterator> iterators;

        // round robin: (BAD!)
        std::list<ShotResultList >::iterator i;
        for (i = lists.begin(); i != lists.end(); i++)
        {
            iterators.push_back(i->begin());
            lengths.push_back(i->size());
            positions.push_back(0);
        }

        for (int nr = 0; nr < mMaxResults; nr++)
        {
            std::list<int>::iterator length   = lengths.begin();
            std::list<int>::iterator position = positions.begin();
            std::list<ShotResultList::iterator>::iterator it;
            for (it = iterators.begin(); it != iterators.end(); it++)
            {
                if (*position < *length)
                {
                    int shotid = (*(*it)).shotid;
                    if (counts.find(shotid) == counts.end())
                    {
                        merged.push_back(*(*it));
                        counts[shotid] = 1;
                    }
                    else
                    {
                        counts[shotid] = counts[shotid] + 1;
                    }
                    (*it)++;
                    (*position)++;
                }
                position++;
                length++;
            }
        }

        merged = ProcessFilters(merged);

        return merged;
    }


    ShotResultList ProcessFilters(ShotResultList in)
    {
        ILOG_DEBUG( "FILTERING: original  " << in.size() << " shots." );
        in = ProcessList(in, &IDashQueryEngine::FilterByLanguage);
        ILOG_DEBUG( "FILTERING: language  " << in.size() << " shots." );
        return in;
    }

    void SetLanguageFilter(bool enableEng, bool enableChi, bool enableArb)
    {
        mLanguageAllowEng = enableEng;
        mLanguageAllowArb = enableArb;
        mLanguageAllowChi = enableChi;
        if (mDoLanguageFilter = !(enableEng && enableArb && enableChi))
        {
            ILOG_SYSTEM("Language filter enabled. " <<
                        "English: " << mLanguageAllowEng << ", " <<
                        "Chinese: " << mLanguageAllowChi << ", " <<
                        "Arabic: " << mLanguageAllowArb);
        }
        else
            ILOG_SYSTEM( "Language filter disabled." );
    }

private:
    ShotResultList
    ConvertToShotResult(std::list<KeyframeResult> keyframes)
    {
        ShotResultList shots;
        std::list<KeyframeResult>::iterator i;
        for (i=keyframes.begin(); i != keyframes.end(); i++)
        {
            ShotResult r;
            int frameID = i->keyframeid;
            if (frameID == -1) {
                ILOG_WARN("QueryGui::ConvertToShotResult: " <<
                          "WARNING: erroneous frame in set." );
                continue;
            }
            int shotID = mThreadSet->GetKeyframes()->GetShotId(frameID);
            r.shotid = shotID;
            r.score = i->score;
            r.rank = i->rank;
            shots.push_back(r);
        }
        return shots;
    }

    ShotResultList
    ConvertToShotResult(ThreadSet *threadset, String concept)
    {
        // find the Thread using the "concept" name
        Thread *t = NULL;
        for (int i=0; i < threadset->GetNrThreads(); i++)
        {
            if (threadset->GetThread(i)->GetName() == String("rank_") + concept)
            {
                t = threadset->GetThread(i);
                break;
            }
        }

        // make a ShotResult list
        ShotResultList out;
        if (t != NULL)
        {
            // int is NOT used for ThreadRank objects... TODO
            int firstShot = t->GetFirstShot();
            for (int i=0; i<mMaxResults; i++)
            {
                ShotResult r;
                r.shotid = t->GetShotAt(firstShot, i);
                r.rank = i+1;
                r.score = 1.0;
                out.push_back(r);
            }
        }
        return out;
    }

    int GetLanguage(int shotID)
    {
        int videoID = mThreadSet->GetSegmentation()->GetVideoId(shotID);
        String videofile =
            mThreadSet->GetSegmentation()->GetVideoSet()->GetFile(videoID);
        
        if (videofile.find("_ENG") != String::npos)
            return LANGUAGE_ENGLISH;
        if (videofile.find("_CHN") != String::npos)
            return LANGUAGE_CHINESE;
        if (videofile.find("_ARB") != String::npos)
            return LANGUAGE_ARABIC;

        ILOG_WARN("QueryEngine::GetLanguage(" << shotID <<
                  "): WARNING: could not determine language of " << videofile );
        return -1;
    }

    void Init(ThreadSet *set)
    {
        mThreadSet = set;
        mDoLanguageFilter = false;
        mLanguageAllowEng = mLanguageAllowArb = mLanguageAllowChi = true;
        mMaxResults = 2000;
    }

    int                                     mMaxResults;

    bool                                    mDoLanguageFilter;
    bool                                    mLanguageAllowEng;
    bool                                    mLanguageAllowArb;
    bool                                    mLanguageAllowChi;

    ThreadSet*                              mThreadSet;
    std::map<String, double>                mSelectedConcepts;
    std::map<String, ShotResultList >       mSelectedShots;


    static const int LANGUAGE_ENGLISH    = 1;
    static const int LANGUAGE_CHINESE    = 2;
    static const int LANGUAGE_ARABIC     = 4;

    static const int MERGE_ROUNDROBIN    = 1;
    static const int MERGE_BORDAFUSION   = 2;

    static const int MERGE_DEFAULT       = MERGE_BORDAFUSION;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(IDashQueryEngine, Application.IDash);

}
}
}

#endif

---