|| || || || || || || ||

---

IndexAnnotation.h

Go to the documentation of this file.

#ifndef Impala_Core_VideoSet_IndexAnnotation_h
#define Impala_Core_VideoSet_IndexAnnotation_h

#include "Persistency/AnnotationTableSetRepository.h"
#include "Persistency/Mpeg7DocRepository.h"
#include "Persistency/QuidTableRepository.h"
#include "Core/Table/AnnotationTableSet.h"
#include "Core/Table/Sort.h"
#include "Core/VideoSet/Reporter.h"
#include "Core/VideoSet/Mpeg7Doc.h"

namespace Impala
{
namespace Core
{
namespace VideoSet
{


class IndexAnnotation : public Listener
{
public:

    IndexAnnotation(Reporter* reporter, CmdOptions& options)
    {
        mReporter = reporter;
        if (options.GetNrArg() < 3)
        {
            ILOG_ERROR("Missing argument");
            return;
        }
        mConceptSet = options.GetArg(2);
        mConceptExtra = "";
        if (options.GetNrArg() > 3)
            mConceptExtra = options.GetArg(3);
        mNrExtra = 0;
        if (options.GetNrArg() > 4)
            mNrExtra = atol(options.GetArg(4));
        mKeyframes = 0;
        mSegmentation = 0;
        mRkfMask = options.GetBool("rkfMask");
    }

    virtual void
    HandleNewWalk(VideoSet* vs)
    {
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        mAnnoShot = Table::AnnotationTableSet::MakeFromFile(vs, mConceptSet,
                                                            false, 0);
        mAnnoFrame = Table::AnnotationTableSet::MakeFromFile(vs, mConceptSet,
                                                             false, 0);
#else // REPOSITORY_USED
        Persistency::KeywordListLocator loc(vs->GetLocator(), mConceptSet);
        Table::KeywordList keyList =
            *(Persistency::KeywordListRepository().Get(loc));
        mAnnoShot = new Table::AnnotationTableSet(keyList, 0);
        mAnnoFrame = new Table::AnnotationTableSet(keyList, 0);
#endif // REPOSITORY_USED
        if (! (mAnnoShot && mAnnoFrame))
        {
            ILOG_ERROR("Unable to read conceptSet");
        }
        mAnnoExtra = 0;
        mQuidExtra = 0;
        if (!mConceptExtra.empty())
        {
            ILOG_INFO("Reading extra : " << mConceptExtra);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            mAnnoExtra = Table::AnnotationTableSet::MakeFromFile(vs,
                                                                 mConceptExtra,
                                                                 false, 0);
#else // REPOSITORY_USED
            loc.SetConceptSet(mConceptExtra);
            keyList = *(Persistency::KeywordListRepository().Get(loc));
            mAnnoExtra = new Table::AnnotationTableSet(keyList, 0);
#endif // REPOSITORY_USED
            if (!mAnnoExtra)
            {
                ILOG_ERROR("Unable to read conceptExtra");
            }
            mQuidExtra = new Table::QuidTable(100);
        }
        int nr = mAnnoShot->Size();
        mTotalNrAnno.resize(nr);
        mTotalNrSingleFrame.resize(nr);
        mTotalNrFrames.resize(nr);
        mTotalNrAnnoPos.resize(nr);
        mTotalNrSingleFramePos.resize(nr);
        mTotalNrFramesPos.resize(nr);
    }

    virtual void
    HandleNewWalk(VideoSet* vs, Keyframes* keyframes, bool* keyframeMask)
    {
        mKeyframes = keyframes;
    }

    virtual void
    HandleNewWalk(VideoSet* vs, Segmentation* segmentation)
    {
        mSegmentation = segmentation;
    }

    virtual void
    HandleNewFile(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        if (mAnnoExtra && (!(mKeyframes && mSegmentation)))
        {
            ILOG_ERROR("Need keyframes and segmentation for extra annotations");
            delete mAnnoExtra;
            mAnnoExtra = 0;
            delete mQuidExtra;
            mQuidExtra = 0;
        }

        for (int c=0 ; c<mAnnoShot->Size() ; c++)
        {
            String concept = mAnnoShot->GetLabel(c);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            String fName;
            fName = vs->GetFilePathMpeg7Anno(fileId, mConceptSet,
                                             concept + ".xml", false, true);
            if (fName.empty())
                continue;

            Mpeg7Doc mp7(fName, vs->GetDatabase());
            for (int s=0 ; s<mp7.NrShots() ; s++)
            {
                mTotalNrAnno[c]++;
                int len = mp7.EndFrame(s) - mp7.StartFrame(s);
                if (len <= 1)
                    mTotalNrSingleFrame[c]++;
                else
                    mTotalNrFrames[c] += len;
                if (mp7.IsPositiveAnno(s))
                {
                    mTotalNrAnnoPos[c]++;
                    if (len <= 1)
                        mTotalNrSingleFramePos[c]++;
                    else
                        mTotalNrFramesPos[c] += len;
                }
            }
            if (mKeyframes)
                DoKeyframes(vs, fileId, c, &mp7);
            if (mSegmentation)
                DoShots(vs, fileId, c, &mp7);
#else // REPOSITORY_USED
            String container = mConceptSet + "/" + vs->GetContainerFile(fileId);
            typedef Persistency::Mpeg7DocLocator MP7Loc;
            MP7Loc loc(vs->GetLocator(), MP7Loc::MPEG7_ANNOTATION, container,
                       concept + ".xml");
            if (!Persistency::Mpeg7DocRepository().Exists(loc))
                continue;

            Mpeg7Doc* mp7 = Persistency::Mpeg7DocRepository().Get(loc);
            for (int s=0 ; s<mp7->NrShots() ; s++)
            {
                mTotalNrAnno[c]++;
                int len = mp7->EndFrame(s) - mp7->StartFrame(s);
                if (len <= 1)
                    mTotalNrSingleFrame[c]++;
                else
                    mTotalNrFrames[c] += len;
                if (mp7->IsPositiveAnno(s))
                {
                    mTotalNrAnnoPos[c]++;
                    if (len <= 1)
                        mTotalNrSingleFramePos[c]++;
                    else
                        mTotalNrFramesPos[c] += len;
                }
            }
            if (mKeyframes)
                DoKeyframes(vs, fileId, c, mp7);
            if (mSegmentation)
                DoShots(vs, fileId, c, mp7);
            delete mp7;
#endif // REPOSITORY_USED
        }
    }

    virtual void
    HandleDoneWalk(VideoSet* vs)
    {
        Persistency::AnnotationTableSetLocator loc(vs->GetLocator(), 0,
                                                   mConceptSet);
        for (int c=0 ; c<mAnnoShot->Size() ; c++)
        {
            String concept = mAnnoShot->GetLabel(c);
            int nrShot = mTotalNrAnno[c] - mTotalNrSingleFrame[c];
            double avg = (double) mTotalNrFrames[c] / nrShot;
            if (nrShot == 0)
                avg = 1;
            ILOG_INFO("Indexed " << mTotalNrAnno[c] << " annotations for "
                      << concept);
            ILOG_INFO("total of " << mTotalNrSingleFrame[c] << " frames and "
                      << nrShot << " shots with avg length of " << avg);
            nrShot = mTotalNrAnnoPos[c] - mTotalNrSingleFramePos[c];
            avg = (double) mTotalNrFramesPos[c] / nrShot;
            if (nrShot == 0)
                avg = 1;
            ILOG_INFO("postives: total of " << mTotalNrSingleFramePos[c] <<
                      " frames and " << nrShot << " shots with avg length of "
                      << avg);
        }
        if (mSegmentation)
        {
            ILOG_INFO("Number of shots " << mSegmentation->GetNrShots());
            ILOG_INFO("Saving shot annotations");
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            mAnnoShot->SaveTables(vs, mConceptSet, true);
#else // REPOSITORY_USED
            Persistency::AnnotationTableSetRepository().Add(loc, mAnnoShot);
#endif // REPOSITORY_USED
        }
        if (mKeyframes)
        {
            ILOG_INFO("Number of keyframes " << mKeyframes->GetNrKeyframes());
            ILOG_INFO("Saving keyframe annotations");
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            mAnnoFrame->SaveTables(vs, mConceptSet, true);
#else // REPOSITORY_USED
            Persistency::AnnotationTableSetRepository().Add(loc, mAnnoFrame);
#endif // REPOSITORY_USED
        }
        if (mAnnoExtra)
        {
            ILOG_INFO("Saving extra annotations");
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            mAnnoExtra->SaveTables(vs, mConceptExtra, true);
#else // REPOSITORY_USED
            loc.SetConceptSet(mConceptExtra);
            Persistency::AnnotationTableSetRepository().Add(loc, mAnnoExtra);
#endif // REPOSITORY_USED
            ILOG_INFO("Sorting and saving quids");
            Table::Sort(mQuidExtra, 1, true);
            String qName = FileNameBase(mConceptExtra) + "_quids.tab";
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
            String fName = vs->GetFilePathAnnotation(qName, true, false);
            if (!fName.empty())
                Table::Write(mQuidExtra, fName, vs->GetDatabase(), true);
#else // REPOSITORY_USED
            Persistency::QuidTableLocator qLoc(vs->GetLocator(), "Annotations",
                                               qName);
            Persistency::QuidTableRepository().Add(qLoc, mQuidExtra);
#endif // REPOSITORY_USED
        }
    }

private:

    void
    AddExtra(VideoSet* vs, int fileId, int concept, Mpeg7Doc* mp7, int key,
             int idx)
    {
        Table::AnnotationTable* frameExtra = mAnnoExtra->GetTable(concept);
        int shot = mKeyframes->GetShotId(key);
        int begin = Max(mSegmentation->GetStart(shot), mp7->StartFrame(idx));
        int end = Max(mSegmentation->GetEnd(shot), mp7->EndFrame(idx));
        int step = (end - begin) / (mNrExtra + 2);  // skip begin and end parts
        int frame = begin + step;
        while (frame <= end - step)
        {
            Quid q = MakeQuidFrame(vs->GetSetId(), fileId, frame);
            if (!frameExtra->IsAnnotated(q))
            {
                frameExtra->AddPositive(q);
                if (!mQuidExtra->Contains(q))
                    mQuidExtra->Add(q);
            }
            frame += step;
        }
    }

    void
    DoKeyframes(VideoSet* vs, int fileId, int concept, Mpeg7Doc* mp7)
    {
        Table::AnnotationTable* frameTable = mAnnoFrame->GetTable(concept);
        Table::AnnotationTable* frameExtra = 0;
        if (mAnnoExtra)
            frameExtra = mAnnoExtra->GetTable(concept);
        int startKey = mKeyframes->GetFirstKeyframeVideo(fileId);
        int endKey = startKey + mKeyframes->GetNrKeyframesVideo(fileId);
        for (int k=startKey ; k<endKey ; k++)
        {
            if (mRkfMask && (!mKeyframes->IsRKF(k)))
                continue;
            int frame = mKeyframes->GetFrameNr(k);
            int idx = mp7->FindShot(frame);
            if (idx != -1)
            {
                Quid q = MakeQuidFrame(vs->GetSetId(), fileId, frame);
                if (mQuidExtra && (!mQuidExtra->Contains(q)))
                    mQuidExtra->Add(q);
                if (mp7->IsPositiveAnno(idx))
                {
                    frameTable->AddPositive(q);
                    if (frameExtra)
                    {
                        frameExtra->AddPositive(q);
                        int len = mp7->EndFrame(idx) - mp7->StartFrame(idx);
                        if (len > 1)
                        {
                            AddExtra(vs, fileId, concept, mp7, k, idx);
                        }
                    }
                }
                else if (mp7->IsNegativeAnno(idx))
                {
                    frameTable->AddNegative(q);
                    if (frameExtra)
                        frameExtra->AddNegative(q);
                }
                else
                {
                    frameTable->AddSkip(q);
                    if (frameExtra)
                        frameExtra->AddSkip(q);
                }
            }
        }
    }

    void
    DoShots(VideoSet* vs, int fileId, int concept, Mpeg7Doc* mp7)
    {
        Table::AnnotationTable* shotTable = mAnnoShot->GetTable(concept);
        int startShot = mSegmentation->GetFirstShotVideo(fileId);
        int endShot = startShot + mSegmentation->GetNrShotsVideo(fileId);
        for (int s=startShot ; s<endShot ; s++)
        {
            int startFrame = mSegmentation->GetStart(s);
            int endFrame = mSegmentation->GetEnd(s);
            std::vector<int> idx = mp7->FindShots(startFrame, endFrame);

            if (idx.size() > 0)
            {
                Quid q = MakeQuidShot(vs->GetSetId(), fileId, s - startShot);

                bool hasPos = false;
                bool hasNeg = false;
                for (int i=0 ; i<idx.size() ; i++)
                {
                    if (mp7->IsPositiveAnno(idx[i]))
                        hasPos = true;
                    else if (mp7->IsNegativeAnno(idx[i]))
                        hasNeg = true;
                }

                if (hasPos)
                    shotTable->AddPositive(q);
                else if (hasNeg)
                    shotTable->AddNegative(q);
                else
                    shotTable->AddSkip(q);
            }
        }
    }

    Reporter*                  mReporter;
    String                     mConceptSet;
    String                     mConceptExtra;
    int                        mNrExtra;
    Segmentation*              mSegmentation;
    Keyframes*                 mKeyframes;
    bool                       mRkfMask;
    std::vector<String>        mConcepts;
    Table::AnnotationTableSet* mAnnoShot;
    Table::AnnotationTableSet* mAnnoFrame;
    Table::AnnotationTableSet* mAnnoExtra;
    Table::QuidTable*          mQuidExtra;

    std::vector<int> mTotalNrAnno;
    std::vector<int> mTotalNrSingleFrame;
    std::vector<int> mTotalNrFrames;
    std::vector<int> mTotalNrAnnoPos;
    std::vector<int> mTotalNrSingleFramePos;
    std::vector<int> mTotalNrFramesPos;

    ILOG_VAR_DEC;

};

ILOG_VAR_INIT(IndexAnnotation, Impala.Core.VideoSet);

} // namespace VideoSet
} // namespace Core
} // namespace Impala

#endif

---