|| || || || || || || ||

---

ThreadVisualSimilarity.h

Go to the documentation of this file.

#ifndef Impala_Core_Trec_ThreadVisualSimilarity_h
#define Impala_Core_Trec_ThreadVisualSimilarity_h

#include "Basis/NativeTypes.h"
#include "Core/Table/TableTem.h"
#include "Core/Vector/NormalizationMax.h"
#include "Core/Vector/Similarity.h"
#include "Core/Vector/Norm1Dist.h"
#include "Core/Feature/FeatureTable.h"
#include "Core/Table/SimilarityTableSet.h"
#include "Core/Table/Read.h"
#include "Core/Column/Find.h"
#include "Core/Trec/Thread.h"
#include "Core/VideoSet/Keyframes.h"
#include "Core/Feature/VisSem.h"

namespace Impala
{
namespace Core
{
namespace Trec
{

// table: shotID, rank, similarity to original
typedef Table::TableTem<Column::ColumnTem<Int32>,
                        Column::ColumnTem<Int32>,
                        Column::ColumnTem<Real64> > TableShotsIndexScore;

class ThreadVisualSimilarity : public Thread
{
public:
   typedef Vector::VectorTem<Real64> VectorReal64;
   
    ThreadVisualSimilarity(CString feature, VideoSet::Keyframes* keyframes)
        : Thread(String("visual_") + feature)
    {
        String qualName = feature;
        if (feature == "vissem" || feature == "vissemgabor" ||
            feature == "fusionvissemgabor")
                qualName += "_proto_annotation_nrScales_2_nrRects_130";
        Feature::FeatureDefinition def(qualName);
        mFeatureTable = new Feature::FeatureTable(def);

        mKeyframes = keyframes;
        String fName =
            mKeyframes->GetVideoSet()->GetFilePathFeatureIndex(def, "", false,
                                                               false);
        if (fName.empty())
        {
            ILOG_DEBUG("fName is empty, attempting alternative.");
            Feature::FeatureDefinition def_alt(feature + "_proto_clusters_keyframes_all_maxlen_120_sp_0");
            mFeatureTable = new Feature::FeatureTable(def_alt);
            fName =
                mKeyframes->GetVideoSet()->GetFilePathFeatureIndex(def_alt, "", false,
                                                                   false);
            if (fName.empty()) {
                ILOG_DEBUG("fName is empty, failed.");
                mValid = false;
                return;
            }
        }

        mShotsCache = new TableShotsIndexScore(0);

        Table::Read(mFeatureTable, fName,
                    mKeyframes->GetVideoSet()->GetDatabase());
        mKeyEqualShot = (mFeatureTable->Size() == mKeyframes->GetNrShots());
        Vector::NormalizationMax(mFeatureTable->GetColumn2());
        mOrigin = -1;
        mTopN = Min(mFeatureTable->Size() - 1, 1000);
        ILOG_DEBUG("mTopN for " << feature << " = " << mTopN);
        std::vector<String> names;
        names.push_back("test");
        mSimSet = new Table::SimilarityTableSet(names, mFeatureTable->Size());
        Core::Table::Copy(mSimSet->GetQuidTable(), mFeatureTable);
        mValid = true;
    }

    bool
    GetValid() const // refers to the similarity data, not the origin
    {
        return mValid;
    }

    virtual void
    SetOrigin(int shot)
    {
        // ORK: temporarily disabled to find bugs.
        // && mSimSet->GetRankTable(0)->Size() >= mTopN) // SK: only skip recalc if sufficient results are available
            
        if (shot == mOrigin) {
            ILOG_DEBUG("SetOrigin: " << shot << " length: " << mTopN << " NO RECALC");
            return;
        }

        int key = Shot2Key(shot);
        Quid quid = mKeyframes->GetQuidFrame(key);
        ILOG_DEBUG("SetOrigin: " << shot << " key=" << key << " quid=" <<
                   quid << " length: " << mTopN);
        //ILOG_STARTACTION("computing visual similarity", ILOG_LEVEL_DEBUG);
        Table::SimilarityTableSet::SimTableType* sTable =
            mSimSet->GetSimTable(0);
        Real64* sim = sTable->GetColumn1()->GetData();
        Vector::Similarity(sim, mFeatureTable->FindFeature(quid),
                           mFeatureTable->GetColumn2(), 0, 1,
                           Vector::Norm1Dist<Real64>);
        sTable->SetSize(mFeatureTable->Size());
        int nr = Min(mFeatureTable->Size(), mTopN);
        mSimSet->ComputeRanksTopN(nr, false);

        // cache these results:
        mShotsCache->SetEmpty();
        std::map<int,bool> noduplicates;
        Table::SimilarityTableSet::RankTableType* ranktable = mSimSet->GetRankTable(0);
        Table::SimilarityTableSet::SimTableType* scoretable = mSimSet->GetSimTable(0);
        for (int i = 0; i < nr; i ++)
        {
            Quid thisresult = ranktable->Get1(i);
            int keyframe = mKeyframes->GetFrameId(thisresult);
            Real64 score = scoretable->Get1(mSimSet->FindQuid(thisresult));

            if (keyframe < 0)
                continue;

            int shot = mKeyframes->GetShotId(keyframe);

            if (shot >= 0 &&
                noduplicates.find(shot) == noduplicates.end())
            {
                mShotsCache->Add(shot, i, score);
                noduplicates[shot] = true;
            }
        }
        //ILOG_ENDACTION("computing visual similary");
        mOrigin = shot;
    }


    // added for experiment Bouke - 13-1-2008
/*    Quid
    GetNearestFrameQuid(Quid quid)
    {
        Table::SimilarityTableSet::SimTableType* sTable =
            mSimSet->GetSimTable(0);
        Real64* sim = sTable->GetColumn1()->GetData();
        Vector::Similarity(sim, mFeatureTable->FindFeature(quid),
                           mFeatureTable->GetColumn2(), 0, 1,
                           Vector::Norm1Dist<Real64>);
        sTable->SetSize(mFeatureTable->Size());
        int nr = Min(mFeatureTable->Size(), 8);

        mSimSet->ComputeRanksTopN(nr, false);

        // return best result:
        Table::SimilarityTableSet::RankTableType* ranktable = mSimSet->GetRankTable(0);
        Table::SimilarityTableSet::SimTableType* scoretable = mSimSet->GetSimTable(0);

        Quid rquid = ranktable->Get1(0);
        ILOG_INFO("GetNearestFrameQuid: IN:" << QuidObj(quid) << " OUT:" << QuidObj(rquid));
        mOrigin = -1;
        return rquid;
    } */

    Quid
    GetNearestFrameQuid(VectorReal64 exampleFeatureVector)
    {
        Table::SimilarityTableSet::SimTableType* sTable =
            mSimSet->GetSimTable(0);
        Real64* sim = sTable->GetColumn1()->GetData();
        Vector::Similarity(sim, exampleFeatureVector,
                           mFeatureTable->GetColumn2(), 0, 1,
                           Vector::Norm1Dist<Real64>);
        sTable->SetSize(mFeatureTable->Size());
        int nr = Min(mFeatureTable->Size(), 8);

        mSimSet->ComputeRanksTopN(nr, false);

        // return best result:
        Table::SimilarityTableSet::RankTableType* ranktable = mSimSet->GetRankTable(0);
        Table::SimilarityTableSet::SimTableType* scoretable = mSimSet->GetSimTable(0);

        Quid rquid = ranktable->Get1(0);
       

        // verification
        int ver_keyframe = mKeyframes->GetFrameId(rquid);
       ILOG_INFO("GetNearestFrameQuid:" << QuidObj(rquid) << " keyframe: " << ver_keyframe );
       if (mKeyframes->IsRKF(ver_keyframe)) 
         {
            ILOG_INFO("is rkf");
         }
       
        if (ver_keyframe > mKeyframes->GetNrKeyframes()) 
        {
            ILOG_ERROR("Keyframe does not exist!");
        }
       
       // only take RKF's:
       int shot = mKeyframes->GetShotId(ver_keyframe);
       int rkf_keyframe = Shot2Key(shot);
       Quid rrquid = mKeyframes->GetQuidFrame(rkf_keyframe);
       if (rrquid != rquid) 
          ILOG_INFO("Quid " << QuidObj(rrquid) << " differs fromr result!");
        mOrigin = -1;
       return rrquid;
    }

    void
    SetMaxReturned(int nr)
    {
        mTopN = Min(nr, mFeatureTable->Size() - 1);
    }

    int
    GetMaxReturned()
    {
        return mTopN;
    }

    Real64
    GetSimilarity(int shot)
    {
        int rankIndex = GetShotPosition(shot);
        if (rankIndex < 0)
            return -1.0;

        return mShotsCache->Get3(rankIndex);
    }

    virtual int
    GetShot(int position)
    {
        if ((position < 0) || (position >= mShotsCache->Size()))
            return -1;
        return mShotsCache->Get1(position);
    }

    /*Real64
    GetShotScore(int position)
    {
        Quid shot = mSimSet->GetRankTable(0)->Get1(position);
*/

    virtual int
    GetLength()
    {
        return mShotsCache->Size();
    }

    virtual int
    GetShotAt(int shot, int offset)
    {
        int r = Column::Find(mShotsCache->GetColumn1(), shot, 0, mShotsCache->Size());
        int dst = r + offset;
        if ((dst < 0) || (dst >= mShotsCache->Size())) {
            return -1;
        }
        return mShotsCache->Get1(dst);
    }

    virtual int
    GetShotPosition(int shot)
    {
        int r = Column::Find(mShotsCache->GetColumn1(), shot, 0, mShotsCache->Size());
        if (r == mShotsCache->Size())
            return -1;
        return r;
    }

    virtual int
    GetType()
    {
        return VISUAL;
    }

private:

    int
    Shot2Key(int shot)
    {
        //return (mKeyEqualShot) ? shot : mKeyframes->GetShotRKF(shot);
        return mKeyframes->GetShotRKF(shot);
    }

    int
    Key2Shot(int key)
    {
        return (mKeyEqualShot) ? key : mKeyframes->GetShotId(key);
    }

    VideoSet::Keyframes*       mKeyframes;
    Feature::FeatureTable*     mFeatureTable;
    int                        mOrigin;
    int                        mTopN;
    Table::SimilarityTableSet* mSimSet;
    bool                       mValid;
    bool                       mKeyEqualShot;

    TableShotsIndexScore*      mShotsCache;

    ILOG_VAR_DEC;

};

ILOG_VAR_INIT(ThreadVisualSimilarity, Core.Trec);


} // namespace Trec
} // namespace Core
} // namespace Impala

#endif

---