|| || || || || ||

---

ShotSegmenter.h

Go to the documentation of this file.

#ifndef Impala_Core_VideoSet_ShotSegmenter_h
#define Impala_Core_VideoSet_ShotSegmenter_h

#include "Persistency/SimilarityTableSetRepository.h"
#include "Core/Histogram/MakeHistogram1d.h"
#include "Core/Array/SetVal.h"
#include "Core/Array/ProjectRange.h"
#include "Core/Array/MakeGaussian1d.h"
#include "Core/Array/RecGauss.h"
#include "Core/Geometry/RectanglePyramid.h"
#include "Core/Vector/ElemMin.h"
#include "Core/Vector/HistogramIntersection.h"
#include "Core/VideoSet/Reporter.h"
#include "Core/VideoSet/Mpeg7DocWrite.h"


namespace Impala
{
namespace Core
{
namespace VideoSet
{


class ShotSegmenter : public Listener
{
public:

    typedef Core::Histogram::Histogram1dTem<Real64> HistType;
    typedef Array::Array2dScalarReal64 ArrayType;
    typedef Array::Array2dScalarReal64 Array2dScalarReal64;

    ShotSegmenter(Reporter* reporter, CmdOptions& options)
    {
        mBlurSigma   = 1.0;
        mRadius      = 6;
        mBinCount    = 32;
        mProbThresh  = options.GetDouble("SSProbThresh");
        mDoMpeg7     = ! options.GetBool("SSNoMpeg7");
        mDoSim       = ! options.GetBool("SSNoSim");
        mDoNrkf      = ! options.GetBool("SSNoNrkf");

        mWindowSize  = 2 * mRadius;
        mPyramid = Geometry::RectanglePyramid(0, 1, 2, 1, -1, -1);
        mHistPerFrame = 3 * mPyramid.TotalNrRects();
        // Prepare the 1-d gaussian filter to be used for scaling in time
        mGauss = Array::MakeGaussian1d(18, 0, 0, mWindowSize*2+1,
                                       mWindowSize*2+1);
        mWeights = Array::ArrayCreate<Array2dScalarReal64>(mWindowSize,
                                                           mWindowSize);
        mFeatures = new Vector::VectorTem<Real64>(mWindowSize);
        mHistograms = new HistType*[mWindowSize*mHistPerFrame];
        for (int i=0 ; i<mWindowSize*mHistPerFrame ; i++)
            mHistograms[i] = new HistType(0, 256, mBinCount, 0);

        mScores = 0;
        mPredictions = 0;
    }

    virtual
    ~ShotSegmenter()
    {
        delete mGauss;
        delete mWeights;
        delete mFeatures;

        for (int i=0 ; i<mWindowSize*mHistPerFrame ; i++)
            delete mHistograms[i];
        delete[] mHistograms;

        CleanUpScoresAndPredictions();
    }

    void
    HandleNewFile(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        if (src->LastFrame() < 0)
        {
            ILOG_ERROR("HandleNewFile: src->LastFrame() = " << src->LastFrame()
                       << " (fileId = " << fileId << ")");
            return;
        }
        int lastFrame = src->LastFrame();

        // Reset data for the next file
        SetVal(mWeights, 0);

        CleanUpScoresAndPredictions();

        mPredictions = Array::ArrayCreate<Array2dScalarReal64>(lastFrame,1,0,0);
        SetVal(mPredictions, 0);

        mScores = Array::ArrayCreate<Array2dScalarReal64>(lastFrame,1,0,0);
        SetVal(mScores, 0);

        mPyramid.SetImageSize(src->FrameWidth(), src->FrameHeight());
    }

    void
    HandleNewFrame(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        int processFrame = src->FrameNr();
        int scoreFrame = processFrame - mRadius;
        int decisionFrame = processFrame - mWindowSize;
        int predictFrame = (decisionFrame > 0) ? decisionFrame : 0;

        UpdateHistograms(src);

        if (processFrame >= mWindowSize - 1)
        {
            ComputeSSWeights();
            Real64 score = CalculateScores();
            mScores->SetValue(score, scoreFrame, 0);
        }

        Predict(predictFrame);
    }

    void
    HandleDoneFile(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        WriteMp7ShotSeg(vs, fileId, src);
        WriteShotSimilarity(vs, fileId, src);
    }


protected:

    Mpeg7Doc* 
    ConstructMpeg7Doc(CString id, CString fileName, int realId, 
                      int nrOfFrames, int frameRateNum, int frameRateDen) const
    {
        int fractions = 0;
        Mpeg7Doc* mp7 = new Mpeg7Doc(id, fileName, nrOfFrames, fractions, true, 
                                     "", "", frameRateNum, frameRateDen);

        int firstFrameInShot = 0;
        int frameAtSemiCut = -1;
        int lastFrame = nrOfFrames - 1;
        int shotNr = 0;

        for (int i = 0; i <= lastFrame; i++)
        {
            // If the probability is >mProbThresh then there was a cut 
            // between previous and current frame
            if ((mPredictions->Value(i, 0) > mProbThresh) ||
                (i == lastFrame))
            {
                String shotName = "shot" + MakeString(realId + 1) + "_"
                    + MakeString(shotNr + 1);
                std::vector<String> keyNames;
                keyNames.push_back(shotName + "_RKF");
                if (frameAtSemiCut != -1)
                    keyNames.push_back(shotName + "_NRKF");
                std::vector<int> keyFrames;
                if (frameAtSemiCut == -1)
                {
                    keyFrames.push_back((firstFrameInShot + i) / 2);
                }
                else
                {
                    keyFrames.push_back((firstFrameInShot + frameAtSemiCut) / 2);
                    keyFrames.push_back((frameAtSemiCut + i) / 2);
                }
                mp7->AddShot(shotName, firstFrameInShot, i, keyNames, keyFrames);
                firstFrameInShot = i + 1;
                frameAtSemiCut = -1;
                shotNr++;
            }
            else if (mDoNrkf &&
                     (mPredictions->Value(i, 0) > 0.8 * mProbThresh) &&
                     (i - firstFrameInShot > 25))
            {
                frameAtSemiCut = i;
            }
        }
        return mp7;
    }


private:

    void
    BlurImage(Array2dScalarReal64*& channel, double sigma)
    {
        Array2dScalarReal64* blurred = 0;
        RecGauss(blurred, channel, sigma, sigma, 0, 0, 1);
        delete channel;
        channel = blurred;
    }

    void
    UpdateHistograms(Stream::RgbDataSrc* src)
    {
        typedef Array::Array2dVec3Real64 Array2dVec3Real64;
        typedef Array::Array2dVec3UInt8 Array2dVec3UInt8;

        HistType** tmpHist;
        tmpHist = new HistType*[mHistPerFrame];
        // Back up the first frame's histogram pointers
        for (int i=0; i<mHistPerFrame ;i++)
            tmpHist[i] = mHistograms[i];
        // Shift the rest of the histograms
        for (int i=0 ; i<(mWindowSize-1)*mHistPerFrame ; i++)
            mHistograms[i] = mHistograms[i+mHistPerFrame];

        for (int i=(mWindowSize-1)*mHistPerFrame ; i<mWindowSize*mHistPerFrame ;
             i++)
        {
            mHistograms[i] = tmpHist[i - (mWindowSize - 1) * mHistPerFrame];
        }
        delete [] tmpHist;

        Array2dVec3Real64* srcWrap =
            Array::ArrayCreate<Array2dVec3Real64>(src->FrameWidth(),
                                                  src->FrameHeight(), 0, 0);

        Array::MakeFromData2<Array2dVec3Real64,Array2dVec3UInt8>(srcWrap,
                                                                 src->DataPtr());

        Array::Array2dScalarReal64* srcO1 = 0;
        Array::Array2dScalarReal64* srcO2 = 0;
        Array::Array2dScalarReal64* srcO3 = 0;

        Array::ProjectRange(srcO1, srcWrap, 1);
        Array::ProjectRange(srcO2, srcWrap, 2);
        Array::ProjectRange(srcO3, srcWrap, 3);
    
        BlurImage(srcO1, mBlurSigma);
        BlurImage(srcO2, mBlurSigma);
        BlurImage(srcO3, mBlurSigma);

        int k = (mWindowSize - 1) * mHistPerFrame - 1;
        for (int i=0 ; i<mPyramid.NrLevels() ; i++)
        {
            for (int j=0 ; j<mPyramid.NrRects(i) ; j++)
            {
                MakeHistogram1d(mHistograms[++k], srcO1, mPyramid.Rect(i, j));
                mHistograms[k]->Normalize();
                
                MakeHistogram1d(mHistograms[++k], srcO2, mPyramid.Rect(i, j));
                mHistograms[k]->Normalize();

                MakeHistogram1d(mHistograms[++k], srcO3, mPyramid.Rect(i, j));
                mHistograms[k]->Normalize();
            }
        }

        delete srcWrap;
        delete srcO1;
        delete srcO2;
        delete srcO3;
    }

    void
    ComputeSSWeights()
    {   
        for (int i=0 ; i<mWindowSize ; i++)
        {
            for (int j=0 ; j<mWindowSize ; j++)
            {
                Real64 weight=0;
                for (int k=0 ; k<mHistPerFrame ; k++)
                {
                    weight += Core::Vector::HistogramIntersection
                        (*mHistograms[i*mHistPerFrame+k],
                         *mHistograms[j*mHistPerFrame+k]);
                }
                mWeights->SetValue(weight*mGauss->Value(mWindowSize+i-j,0),i,j);
            }
        }
    }

    Real64
    CalculateScores()
    {
        int center = mRadius;
        Real64 disSimA = 0;
        Real64 disSimB = 0;
        Real64 simA = 0;
        Real64 simB = 0;

        for (int i=0 ; i<mRadius ; i++)
        {
            for (int j=0 ; j<mRadius ; j++)
            {
                simA += mWeights->Value(i, j);
                disSimA += mWeights->Value(i, j + center);
            }
        }

        for (int i=center ; i<mWindowSize ; i++)
        {
            for (int j=0 ; j<mRadius ; j++)
            {
                simB += mWeights->Value(i, j + center);
                disSimB += mWeights->Value(i, j);
            }
            
        }

        if ((simA * simB == 0) || (disSimA * disSimB == 0))
            return 4;
        Real64 cutAB = disSimA + disSimB;
        Real64 s = cutAB / simA + cutAB / simB;
        return s;
    }
    
    void
    Predict(int predictFrame)
    {
        int predictBase = predictFrame - mRadius + 1; 
        int size = mWindowSize;
        if (predictBase < 0)
        {
            mPredictions->SetValue(0.0, predictFrame, 0);
            return;
        }
        
        // Update the feature vector
        for (int i=0 ; i<size ; i++)
        {
            (*mFeatures)[i] = mScores->Value(predictBase + i, 0);
        }

        Real64 minScore = Vector::ElemMin(*mFeatures);
        if (mFeatures->Elem(mRadius-1) == minScore)
        {
            mPredictions->SetValue((4.0-minScore) / 2.0, predictFrame, 0);
        }
        else
        {
            mPredictions->SetValue(0.0, predictFrame, 0);
        }
    }

    void
    WriteMp7ShotSeg(VideoSet* vs, int fileId, Stream::RgbDataSrc* src) const
    {
        if (!mDoMpeg7)
            return;
        if ((!src) || (src->FrameRateNum() < 0) || (src->FrameRateDen() < 0))
        {
            ILOG_ERROR("WriteMp7ShotSeg: need src with known frame rate");
            return;
        }

        String fileName = vs->GetFile(fileId);
        String id = FileNameBase(fileName);
        int realId = vs->GetFileId(fileId);

        Mpeg7Doc* mp7 = 
            ConstructMpeg7Doc(id, fileName, realId, src->LastFrame() + 1, 
                              src->FrameRateNum(), src->FrameRateDen());

        Mpeg7DocWrite(mp7, vs, fileId);
        delete mp7;
    }

    void
    WriteShotSimilarity(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        if (!mDoSim)
            return;
        int lastFrame = src->LastFrame();
        std::vector<String> names;
        names.push_back("ShotBoundary");
        Table::SimilarityTableSet simSet(names, lastFrame + 1);
        Table::QuidTable* qTable = simSet.GetQuidTable();
        int idx = 0;
        Table::SimilarityTableSet::SimTableType* sTable = simSet.GetSimTable(idx);
        for (int i=0 ; i<=lastFrame ; i++)
        {
            Quid q = vs->GetQuidFrame(fileId, i);
            qTable->Add(q);
            Real64 v = mPredictions->Value(i,0) / 2; // to get between 0 and 1
            sTable->Add(v);
        }
        simSet.ComputeRanks(true);
        Persistency::SimilarityTableSetLocator loc
            (vs->GetLocator(), false, "Frames", "streamConcepts.txt", "no_model",
             "direct", vs->GetAsPath(fileId));
        Persistency::SimilarityTableSetRepository().Add(loc, &simSet);
    }

    void
    CleanUpScoresAndPredictions()
    {
        if (mScores)
        {
            delete mScores;
            mScores = 0;
        }
        if (mPredictions)
        {
            delete mPredictions;
            mPredictions = 0;
        }
    }

    double mBlurSigma;
    int    mRadius;
    int    mBinCount;
    double mProbThresh;
    bool   mDoMpeg7;
    bool   mDoSim;
    bool   mDoNrkf;
    int    mWindowSize;
    int    mHistPerFrame;

    Geometry::RectanglePyramid mPyramid;
    
    Array2dScalarReal64* mGauss;
    Array2dScalarReal64* mWeights;
    Array2dScalarReal64* mScores;
    Array2dScalarReal64* mPredictions;

    Vector::VectorTem<Real64>* mFeatures;

    HistType** mHistograms;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(ShotSegmenter, Impala.Core.VideoSet);

} // namespace VideoSet
} // namespace Core
} // namespace Impala

#endif

---