|| || || || || || || ||

---

ShotSegmenter.h

Go to the documentation of this file.

#ifndef Impala_Core_VideoSet_ShotSegmenter_h
#define Impala_Core_VideoSet_ShotSegmenter_h

#include "Persistency/SimilarityTableSetRepository.h"
#include "Core/Stream/RgbDataSrc.h"
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
#include "Core/Training/Svm.h"
#endif // REPOSITORY_USED
#include "Core/Histogram/Histogram1dTem.h"
#include "Core/Histogram/MakeHistogram1d.h"
#include "Core/Array/SetVal.h"
#include "Core/Array/PixMin.h"
#include "Core/Array/PixMax.h"
#include "Core/Array/MulVal.h"
#include "Core/Array/AddVal.h"
#include "Core/Array/PixSum.h"
#include "Core/Array/Sqrt.h"
#include "Core/Array/Minimum.h"
#include "Core/Array/Maximum.h"
#include "Core/Array/Statistics.h"
#include "Core/Array/LessThanVal.h"
#include "Core/Array/ProjectRange.h"
#include "Core/Array/MakeGaussian1d.h"
#include "Core/Array/RGB2Intensity.h"
#include "Core/Array/ColorSpace.h"
#include "Core/Array/ArraySystem.h"
#include "Core/Array/RecGauss.h"

#include "Core/Vector/HistogramIntersection.h"
#include "Core/Vector/KullbackDivergence.h"

#include "Core/VideoSet/Reporter.h"
#include "Core/VideoSet/Listener.h"
#include "Core/VideoSet/Mpeg7DocWrite.h"
#include "Core/Table/SimilarityTableSet.h"
#include "Persistency/XmlFileReader.h"

#include "Core/Geometry/RectanglePyramid.h"
#include "Basis/FileExists.h"
#include "Core/Vector/ElemMin.h"

namespace Impala
{
namespace Core
{
namespace VideoSet
{


class ShotSegmenter : public Listener
{
public:

    typedef Core::Histogram::Histogram1dTem<Real64> HistType;
    typedef Array::Array2dScalarReal64 ArrayType;
    typedef Array::Array2dScalarReal64 Array2dScalarReal64;

    ShotSegmenter(Reporter* reporter,CmdOptions& options)
    {
        //Walk Listener setup////////////////////
        mReporter = reporter;
        
        mScores=0;
        mPredictions=0;
        mFeatures=0;
        mFeaturesReversed=0;
        mHistograms=0;
        
        //Similarity Types
        mSim         = options.GetString ("SSSim","+");
        mHistSim     = options.GetString ("SSHistSim","Intersection");
        mSymmetry    = options.GetString ("SSSymmetry","None");

        //Preprocessing
        mHistEq      = options.GetBool   ("SSHistEq",false);
        mHistNorm    = options.GetBool   ("SSHistNorm",true);
        mBlur        = options.GetBool   ("SSBlur",true);
        mBlurDer     = options.GetBool   ("SSBlurDer",0);
        mBlurSigma   = options.GetDouble ("SSBlurSigma",1.);
        mNormW       = options.GetBool   ("SSNormW",false);

        //GPM Parameters///////////////////////////////////////
        mThreshold   = options.GetDouble ("SSThreshold",3.4);
        mRadius      = options.GetInt    ("SSRadius",6);
        mSigma       = options.GetDouble ("SSSigma",18);
        mBinCount    = options.GetInt    ("SSBinCount",32);
        mCentered    = options.GetBool   ("SSCentered",false);
        mPyrL        = options.GetInt    ("SSPyrL",1);
        mPyrU        = options.GetInt    ("SSPyrU",2);
        mProbThresh  = options.GetDouble ("SSProbThresh",0.5);
        mShortGrad   = options.GetInt    ("SSShortGrad",3); //Anything

        //RunModes/////////////////////////////////////////////
        mMode         = options.GetString ("SSMode","Run");          //Test,Train,EvalTrain,EvalTest,Run
        mRunName      = options.GetString ("SSRunName","NoRunName"); //Anything
        mGUI          = options.GetBool   ("SSGUI",false);
        mThresholdOnly= options.GetBool   ("SSThresholdOnly",false);
        mPredictOnly  = options.GetBool   ("SSPredictOnly",false);
        mPredictOutput= options.GetBool   ("SSPredictOutput",false);
        mDoProcessing = options.GetBool   ("SSDoProcessing",true);
        mDoProcessing = mDoProcessing&&(!(mThresholdOnly||mPredictOnly));
        
        //Directories//////////////////////////////////////////
        mDataDir      = options.GetString ("data");
        mRefDir       = options.GetString ("SSRefDir");
        if(mRefDir==""){
            mRefDir=mDataDir+"/ShotSegmentation/ref";
        }
        
        //Other
        mRefPrefix   = options.GetString ("SSRefPrefix","/ref_");
        mSvmModelFile= options.GetString ("SSModelFile","default");
        mVerbose     = options.GetInt   ("SSVerbose",0);
        
        mWindowSize  = mCentered ? 2 * mRadius + 1 : 2 * mRadius;

        ILOG_INFO("Shot Segmentation Parameters:");
        ILOG_INFO("---------------------------------");
        ILOG_INFO("  Radius    :"<<mRadius);
        ILOG_INFO("  Sigma     :"<<mSigma);
        ILOG_INFO("  BinCount  :"<<mBinCount);
        ILOG_INFO("  Threshold :"<<mThreshold);
        ILOG_INFO("  HistEq    :"<<(mHistEq ? "yes" : "no"));
        ILOG_INFO("  HistNorm  :"<<(mHistNorm ? "yes" : "no"));
        ILOG_INFO("  Sim       :"<<mSim);
        ILOG_INFO("  HistSim   :"<<mHistSim);
        ILOG_INFO("  Symmetry  :"<<mSymmetry);
        ILOG_INFO("  Centered  :"<<(mCentered?"yes":"no"));
        ILOG_INFO("  Blur      :"<<(mBlur?"yes":"no"));
        ILOG_INFO("  BlurDer   :"<<mBlurDer);
        ILOG_INFO("  BlurSigma :"<<mBlurSigma);
        ILOG_INFO("  mNormW    :"<<(mNormW?"yes":"no"));
        ILOG_INFO("  ProbThresh:"<<mProbThresh);
        ILOG_INFO("  ShortGrad :"<<mShortGrad);
        ILOG_INFO("  Pyr. Lower:"<<mPyrL);
        ILOG_INFO("  Pyr. Upper:"<<mPyrU);
        ILOG_INFO("");
        ILOG_INFO("Shot Segmentation Modes:");
        ILOG_INFO("---------------------------------");
        ILOG_INFO("  Mode      :"<<mMode);
        ILOG_INFO("  GUI       :"<<(mGUI ? "yes" : "no"));
        ILOG_INFO("  RunName   :"<<mRunName);
        ILOG_INFO("  ThreshOnly:"<<(mThresholdOnly?"yes":"no"));
        ILOG_INFO("  PredctOnly:"<<(mPredictOnly?"yes":"no"));
        ILOG_INFO("  PredctOut :"<<(mPredictOutput?"yes":"no"));
        ILOG_INFO("  Processing:"<<(mDoProcessing?"yes":"no"));
        ILOG_INFO("");
        ILOG_INFO("Shot Segmentation Directories:");
        ILOG_INFO("---------------------------------");
        ILOG_INFO("  Data      :"<<mDataDir);
        ILOG_INFO("  Ref       :"<<mRefDir);
        ILOG_INFO("  ModelFile :"<<mSvmModelFile);
            
        mPyramid=Geometry::RectanglePyramid(0,mPyrL,mPyrU,1,-1,-1);

        ILOG_INFO("Pyramid has "<< mPyramid.TotalNrRects()<<" rectangles.");
        mHistPerFrame = 3 * mPyramid.TotalNrRects();

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        if(((mMode=="Run")||(mPredictOnly))&&(mSvmModelFile==""))
        {
            ILOG_INFO("You should supply a model file for prediction!");
            ILOG_INFO("Prediction will not be possible!");
            return;
        }
        else if((mMode=="Run")||(mPredictOnly))
        {
            mSvm=new Core::Training::Svm;
            
            Util::Database& Db = Util::Database::GetInstance();
            String FileName=Db.GetFilePath("ShotSegmentation","Models",mRunName,mSvmModelFile,false,true);
            if (FileName!=""){
                ILOG_INFO("Loading Model");
                mSvm->LoadModel(FileName, &Db);
                ILOG_INFO("Loaded Model");
            }else{
                delete mSvm;
                mSvm=0;
                ILOG_INFO("SVM file not opened");
            }
            //TODO:get the SS parameters from the file name
            //standardize file names
        }else{
            ILOG_INFO("No SVM");
            mSvm=0;
        }
#else // REPOSITORY_USED
        ILOG_INFO("No SVM");
#endif // REPOSITORY_USED
        
        //Prepare the 1-d gaussian filter to be used for scaling in time
        mGauss = Array::MakeGaussian1d(mSigma,0,0,mWindowSize*2+1,mWindowSize*2+1);
        
    }

    virtual
    ~ShotSegmenter()
    {
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        if(mSvm)
            delete mSvm;
#endif // REPOSITORY_USED
        //Clean the walk listener pointers
        if(mScores)
            delete mScores;
        if(mPredictions)
            delete mPredictions;
        if(mHistograms){
            for(int i=0;i<mWindowSize*mHistPerFrame;i++)
            {   
               delete mHistograms[i];
            }
            delete[] mHistograms;
        }
        if(mFeatures)
            delete mFeatures;
        if(mFeaturesReversed)
            delete mFeaturesReversed;
    }

    void
    HandleNewFile(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        mSrc=src;
        mShotNr=0;
        //Allocate the arrays for the next file
        mWeights = Array::ArrayCreate<Array2dScalarReal64>(mWindowSize,mWindowSize);
        SetVal(mWeights,0);

        if (src->LastFrame() < 0)
        {
            ILOG_ERROR("[HandleNewFile] src->LastFrame() = " << src->LastFrame()
                       << " (fileId = " << fileId << ")");
            exit(-1);
        }

        int lastFrame = src->LastFrame();
        mPredictions = Array::ArrayCreate<Array2dScalarReal64>(lastFrame,1,0,0);
        SetVal(mPredictions,0);

        mScores = Array::ArrayCreate<Array2dScalarReal64>(src->LastFrame(),1,0,0);
        SetVal(mScores,0);

        mFeatures=new Vector::VectorTem<Real64>(mCentered?mWindowSize+1:mWindowSize);
        mFeaturesReversed=new Vector::VectorTem<Real64>(mCentered?mWindowSize+1:mWindowSize);
        mHistograms = new HistType*[mWindowSize*mHistPerFrame];
        for(int i=0;i<mWindowSize*mHistPerFrame;i++)
        {
            mHistograms[i]=new HistType(0, 256, mBinCount, 0);
        }
        
        mGroundTruth= Array::ArrayCreate<Array2dScalarReal64>(mSrc->LastFrame(),4,0,0);
        mCUTs= Array::ArrayCreate<Array2dScalarReal64>(mSrc->LastFrame(),1,0,0);
        mDISs= Array::ArrayCreate<Array2dScalarReal64>(mSrc->LastFrame(),1,0,0);
        mOTHs= Array::ArrayCreate<Array2dScalarReal64>(mSrc->LastFrame(),1,0,0);
        mFOIs= Array::ArrayCreate<Array2dScalarReal64>(mSrc->LastFrame(),1,0,0);

        Array::SetVal(mGroundTruth,0);
        Array::SetVal(mCUTs,0);
        Array::SetVal(mDISs,0);
        Array::SetVal(mOTHs,0);
        Array::SetVal(mFOIs,0);
        
        mPyramid.SetImageSize(mSrc->FrameWidth(),mSrc->FrameHeight());
     
        //if((mMode=="Train")||(mMode=="Test")||(mMode=="EvalTrain")||(mMode=="EvalTest"))
        ReadGroundTruth();
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        if(!mDoProcessing){
            char OutFileName[300];
            Util::Database D;
            String SrcName = mSrc->GetName();
            SrcName=Impala::FileNameBase(SrcName);

            //Scores File
            sprintf(OutFileName,"%s_%d-%g-%d.txt",SrcName.c_str(),
                                        mRadius,mSigma,mBinCount);
            Util::Database& Db = Util::Database::GetInstance();
            String FileName=Db.GetFilePath("ShotSegmentation","Scores",mMode,mRunName,OutFileName,false,false);
            Util::IOBuffer* Input=Db.GetIOBuffer(FileName,true,false,"tmp");
            if(Input && Input->Valid())
            {
                ILOG_INFO("No Processing will be done,");
                ILOG_INFO("Reading Scores from "<<FileName);
                int frame;
                double score;
                std::strstream oss;
                for (int i=0;i<mScores->CW();i++){
                    oss<<Input->ReadLine();
                    oss>>frame;
                    oss>>score;
                    //Output->ReadNativeTypeRead(&frame);
                    //Output->ReadNativeTypeRead(&score);
                    mScores->SetValue(score,frame,0);
                }
            }else
                mDoProcessing=true;
        }
#endif // REPOSITORY_USED
    }

    void
    HandleNewFrame(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {       
        int ProcessFrame=mSrc->FrameNr();
        int ScoreFrame=ProcessFrame-mRadius;
        int PredictFrame=GetPredictionFrame();

        if (mDoProcessing)
        {
            UpdateHistograms(mSrc->DataPtr());

            if (ProcessFrame>=mWindowSize-1)
            {
                ComputeSSWeights();
                Real64 score=CalculateScores();
                mScores->SetValue(score,ScoreFrame,0);
            }
        }
        
        Predict(PredictFrame);
        
        if ((mMode=="Train") || (mMode=="Test") || (mMode=="EvalTrain") ||
            (mMode=="EvalTest"))
        {
            return;
        }
    
        if (mGUI && (ProcessFrame==mSrc->LastFrame()))
        {
            HandleDoneFile(vs, fileId, src);
        }
    }

    void
    WritePredictions()
    {
        //TRECVID Evaluation output
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        ILOG_INFO("Writing Predictions");
        String SrcName = mSrc->GetName();
        SrcName=Impala::FileNameBase(SrcName);

        char OutFileName[300];
        sprintf(OutFileName,"%s_%d-%g-%d.xml",SrcName.c_str(),
                                    mRadius,mSigma,mBinCount);
        Util::Database& Db = Util::Database::GetInstance();
        Db.MakeDir("ShotSegmentation","Predict",mRunName);
        String FileName=Db.GetFilePath("ShotSegmentation","Predict",mRunName,OutFileName,true,false);
        Util::IOBuffer* Output=Db.GetIOBuffer(FileName,false,false,"tmp");
        if(Output && Output->Valid())
        {
            ILOG_INFO("Writing Results to "<<FileName);
            Output->Puts(" <seg src=\""+SrcName+".mpg\">\n");
            int start=0;
            int cut;
            std::ostringstream oss;

            for (int i=mRadius+1;i<mPredictions->CW();i++){
                
                if(mPredictions->Value(i,0)>=mProbThresh){
                    start=i;
                    while(mPredictions->Value(i,0)>=mProbThresh)
                        i++;
//                        if(i-start<6)
//                            continue;
                    cut=(start+i)/2;
                    oss<<"  <trans type=\"CUT\" preFNum=\""<<cut<<"\" postFNum=\""<<cut+1<<"\"/>\n";
                    Output->Puts(oss.str());
                }
            }
            Output->Puts(" </seg>\n");
            delete Output;

        }
#endif // REPOSITORY_USED
    }

    void
    WriteMp7ShotSeg(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        //IMPALA Segmentation and keyframes///////////////////////
        String fileName = vs->GetFile(fileId);
        String id = FileNameBase(fileName);
        int lastFrame = src->LastFrame();
        // todo: get fractions from RgbDataSrc
        //int fractions = 30000;
        int fractions = 25;
        Mpeg7Doc mp7(id, fileName, lastFrame + 1, fractions, true, "", "");
        int firstFrameInShot=0;
        int frameAtSemiCut = -1;
        //for(int i=0;i<mPredictions->CW();i++)
        for (int i=0 ; i<=lastFrame ; i++)
        {
            //If the probability is >mProbThresh then there was a cut 
            //between previous and current frame
            if ((mPredictions->Value(i,0) > mProbThresh) ||
                (i == lastFrame))
            {
                int realId = vs->GetFileId(fileId);
                //String shotName = "shot" + MakeString(fileId+1) +
                String shotName = "shot" + MakeString(realId+1) +
                    "_" + MakeString(mShotNr+1);
                std::vector<String> keyNames;
                keyNames.push_back(shotName + "_RKF");
                if (frameAtSemiCut != -1)
                    keyNames.push_back(shotName + "_NRKF");
                std::vector<int> keyFrames;
                if (frameAtSemiCut == -1)
                {
                    keyFrames.push_back((firstFrameInShot+i) / 2);
                }
                else
                {
                    keyFrames.push_back((firstFrameInShot+frameAtSemiCut) / 2);
                    keyFrames.push_back((frameAtSemiCut+i) / 2);
                }
                mp7.AddShot(shotName, firstFrameInShot, i, keyNames,
                            keyFrames);
                firstFrameInShot=i+1;
                frameAtSemiCut = -1;
                mShotNr++;
            }
            else if ((mPredictions->Value(i,0) > 0.8 * mProbThresh) &&
                     (i - firstFrameInShot > 25))
            {
                frameAtSemiCut = i;
            }
        }
        Mpeg7DocWrite(&mp7, vs, fileId);
    }

    void
    WriteShotSimilarity(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        int lastFrame = src->LastFrame();
        std::vector<String> names;
        names.push_back("ShotBoundary");
        Table::SimilarityTableSet simSet(names, lastFrame+1);
        Table::QuidTable* qTable = simSet.GetQuidTable();
        int idx = 0;
        Table::SimilarityTableSet::SimTableType* sTable = simSet.GetSimTable(idx);
        for (int i=0 ; i<=lastFrame ; i++)
        {
            Quid q = vs->GetQuidFrame(fileId, i);
            qTable->Add(q);
            Real64 v = mPredictions->Value(i,0) / 2; // to get between 0 and 1
            sTable->Add(v);
        }
        simSet.ComputeRanks(true);
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        simSet.Save(vs, "Frames", "streamConcepts.txt", "no_model", "direct",
                    fileId, true);
#else // REPOSITORY_USED
        Persistency::SimilarityTableSetLocator loc
            (vs->GetLocator(), false, "Frames", "streamConcepts.txt", "no_model",
             "direct", vs->GetAsPath(fileId));
        Persistency::SimilarityTableSetRepository().Add(loc, &simSet);
#endif // REPOSITORY_USED
    }

    void
    WriteScores(String SrcName)
    {
    
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        std::ostringstream oss;

        //Scores File
        char OutFileName[300];
        sprintf(OutFileName,"%s_%d-%g-%d.txt",SrcName.c_str(),
                                    mRadius,mSigma,mBinCount);
        Util::Database& Db = Util::Database::GetInstance();
        Db.MakeDir("ShotSegmentation","Scores",mMode,mRunName);
        String FileName=Db.GetFilePath("ShotSegmentation","Scores",mMode,mRunName,OutFileName,true,false);
        Util::IOBuffer* Output=Db.GetIOBuffer(FileName,false,false,"tmp");
        if(Output && Output->Valid())
        {
            ILOG_INFO("Writing Scores to "<<FileName);
            for (int i=0;i<mScores->CW();i++)
            {
                oss<<i<<" "<<mScores->Value(i,0);
                Output->Puts(oss.str());
                oss.str( "" );
            }
            delete Output;
        }
#endif // REPOSITORY_USED

    }

    void
    WriteLfvAndSvm(String SrcName, int fileId)
    {
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        //First find all values below our treshold:
        //mScores<mTreshold
        Array::Array2dScalarInt32* TrainSet = 0;
        LessThanVal(TrainSet,mScores,mThreshold);
        
        std::ostringstream osssvm;
        std::ostringstream osslfv;
        
        //LFV File
        char OutFileName[300];
        sprintf(OutFileName,"%s_%s_%g_%d-%g-%d.lfv", SrcName.c_str(),
            mRunName.c_str(),mThreshold,mRadius,mSigma,mBinCount);
        Util::Database& Db = Util::Database::GetInstance();
        Db.MakeDir("ShotSegmentation","Lfv",mMode,mRunName);
        String FileName=Db.GetFilePath("ShotSegmentation","Lfv",mMode,mRunName,OutFileName,true,false);
        Util::IOBuffer* LfvOutput=Db.GetIOBuffer(FileName,false,false,"tmp");
        ILOG_INFO("Writing LFV to "<<FileName);
        //SVM File
        sprintf(OutFileName,"%s_%s_%g_%d-%g-%d.svm",SrcName.c_str(),
            mRunName.c_str(),mThreshold,mRadius,mSigma,mBinCount);
        Db.MakeDir("ShotSegmentation","Svm",mMode,mRunName);
        FileName=Db.GetFilePath("ShotSegmentation","Svm",mMode,mRunName,OutFileName,true,false);
        ILOG_INFO("Writing SVM to "<<FileName);
        Util::IOBuffer* SvmOutput=Db.GetIOBuffer(FileName,false,false,"tmp");
        if(LfvOutput && SvmOutput && LfvOutput->Valid() && SvmOutput->Valid())
        {
            ILOG_INFO("Writing Training Files!");

            //Find the local minimums, discard non-minimums
            //Make ROI to be i-R:i+R =>>TODO: Check Validity of borders
            for(int i = mCentered ? mRadius : mRadius - 1 ;
                    i < ( mCentered ? mScores->CW() - mRadius - 1
                                    :mScores->CW() - mRadius) ; i++)
            {
                ILOG_INFO("Checking if LocalMin");

                if ((TrainSet->Value(i,0) == 1)  ||  (((mMode=="Test")||(mMode=="EvalTest"))&&
                                                     (mCUTs->Value(i,0)==1)&&
                                                     (mCUTs->Value(i-1,0)==1)))
                {
                    ILOG_INFO("Either LocalMin, or Creating Test set");
                    //Centered has 3 more features
                    int left  = mCentered ? i - mRadius : i - mRadius + 1;
                    int right = mCentered ? i + mRadius + 1 : i + mRadius ; 
                    int center = mCentered ? mRadius : mRadius-1;
                    Array2dScalarReal64* roi =
                      MakeRoi(mScores,Geometry::Rectangle(left,0,right,0));
                    //if it's a local min, or a CUT in test run
                    if((roi->Value(center,0)==PixMin(roi)) 
                                            ||(    ((mMode=="Test")||(mMode=="EvalTest"))
                                                && (mCUTs->Value(i,0)   ==1)
                                                && (mCUTs->Value(i-1,0) ==1)) )

                    {
                        bool isCut = ((mCUTs->Value(i-1,0) == 1) |
                                      (mCUTs->Value(i,0)   == 1) |
                                      (mCUTs->Value(i+1,0) == 1) );

                        ILOG_INFO("LocalMin is "<<(isCut?"a cut":"not a cut"));

                        if(mSymmetry!="Inv")
                        {
                            osssvm<<(isCut?1:-1);
                            osslfv<<fileId<<"_"<<i<<" "<<(isCut?1:-1);
                            
                            for(int j=0;j<roi->CW();j++)
                            {
                                //if we are not going to use symmetry
                                osslfv<<" "<<j+1<<":"<<roi->Value(j,0);
                                osssvm<<" "<<j+1<<":"<<roi->Value(j,0);
                            }
                            SvmOutput->Puts(osssvm.str());
                            LfvOutput->Puts(osslfv.str());
                            osssvm.str("");
                            osslfv.str("");
                        }
                        if(mSymmetry!="None")
                        {
                            osssvm<<(isCut?1:-1);
                            osslfv<<fileId<<"_"<<i<<"_sym "<<(isCut?1:-1);
                            for(int j=0;j<roi->CW();j++)
                            {
                                //if we are going to use symmetry,output reverse also
                                osslfv<<" "<<j+1<<":"<<roi->Value(roi->CW()-j-1,0);
                                osssvm<<" "<<j+1<<":"<<roi->Value(roi->CW()-j-1,0);
                            }
                            SvmOutput->Puts(osssvm.str());
                            LfvOutput->Puts(osslfv.str());
                            osssvm.str("");
                            osslfv.str("");
                        }
                    }
                }
            }
            delete SvmOutput;
            delete LfvOutput;
        }
#endif // REPOSITORY_USED
    }

    void
    HandleDoneFile(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
    {
        if (mPredictOnly)
        {
            for (int i=0; i < mScores->CW() - mRadius - 1 ; i++)
                Predict(i);
        }
        
        if (((mMode=="Run") || (mPredictOnly)) && mPredictOutput)
        { 
            WritePredictions();
        }

        //Storing Keyframes and Segmentation is handled in this function

        if (mMode=="Run")
        {
            if (!mGUI)   // If we're not in the walker mode, we do not output
            {            // impala shot data
                WriteMp7ShotSeg(vs, fileId, src);
                WriteShotSimilarity(vs, fileId, src);
            }
        }

        if ((mMode=="Train") || (mMode=="Test") ||
            (mMode=="EvalTrain") || (mMode=="EvalTest"))
        {
            String SrcName = mSrc->GetName();
            SrcName=Impala::FileNameBase(SrcName);
            //Write the scores data 
            WriteScores(SrcName);
            //We should also output the tresholded local minimum values
            WriteLfvAndSvm(SrcName, fileId);
        }

        //Clear the variables for the next file
        if (mScores != 0)
        {
            delete mScores;
            mScores = 0;
        }
        if (mPredictions != 0)
        {
            delete mPredictions;
            mPredictions=0;
        }
        if (mHistograms!=0)
        {
            for (int i=0 ; i<mWindowSize*mHistPerFrame ; i++)
            {   
               delete mHistograms[i];
            }
            delete[] mHistograms;
            mHistograms = 0;
        }
        if (mFeatures != 0)
        {
            delete mFeatures;
            mFeatures = 0;
        }
        if (mFeaturesReversed != 0)
        {
            delete mFeaturesReversed;
            mFeaturesReversed = 0;
        }
        if (mCUTs)
        {
            delete mCUTs;
            delete mFOIs;
            delete mDISs;
            delete mOTHs;
            delete mGroundTruth;
        }
    }

    //Accessors 

    int
    GetPredictionFrame()
    {
        //Return the earliest possible frame we can predict
        int DecisionFrame = mSrc->FrameNr()-mWindowSize;
        return (DecisionFrame>0) ? DecisionFrame : 0;
    }
    
    bool
    Blur()
    {
        return  mBlur;
    }

    Array2dScalarReal64* 
    GetCUTs()
    {
        return mCUTs;
    }

    Array2dScalarReal64* 
    GetFOIs()
    {
        return mFOIs;
    }

    Array2dScalarReal64* 
    GetDISs()
    {
        return mDISs;
    }

    Array2dScalarReal64* 
    GetOTHs()
    {
        return mOTHs;
    }

    Array2dScalarReal64* 
    GetScores()
    {
        return mScores;
    }
    
    Array2dScalarReal64* 
    GetPredictions()
    {
        return mPredictions;
    }
    
    Array2dScalarReal64* 
    GetWeights()
    {
        return mWeights;
    }

    double
    GetProbThresh()
    {
        return mProbThresh;
    }
 
    double
    GetScore(int i)
    {
        if(i<0)
            i=0;
        return mScores->Value(i,0);
    }
    
    int
    GetWindowSize()
    {
        return mWindowSize;
    }

    Vector::VectorTem<Real64>*
    GetFeatures()
    {
        return mFeatures;
    } 
    
    void
    Predict(int PredictFrame)
    {
        int PredictBase = mCentered? PredictFrame-mRadius : PredictFrame-mRadius+1; 
        int size = mCentered ? mWindowSize + 1 : mWindowSize;
        if (PredictBase<0)
        {
            mPredictions->SetValue(0.0, PredictFrame, 0);
            return;
        }
        
        //Update the feature vector
        for(int i=0;i<size;i++)
        {
            (*mFeatures)[i]=mScores->Value(PredictBase+i,0);
            (*mFeaturesReversed)[size-i-1]=mScores->Value(PredictBase+i,0);
            
        }
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        if (mSvm)
        {
            double pf = mSvm->Predict(mFeatures);
            double pr = mSvm->Predict(mFeaturesReversed);
            double p = 0;

            if(mSymmetry=="None")
                p=pf;
            else if(mSymmetry=="Avg")
                p=(pr+pf)/2;
            else if(mSymmetry=="Or")
                p= (pf > pr) ? pf : pr;
            else if(mSymmetry=="And")
                p= (pf < pr) ? pf : pr;

            
            mPredictions->SetValue(1-p,PredictFrame,0);
        }
        else
        {
 #endif // REPOSITORY_USED
           Real64 minScore = Vector::ElemMin(*mFeatures);
            if (mFeatures->Elem(mCentered ? mRadius : mRadius-1) == minScore)
            {
                mPredictions->SetValue((4.0-minScore) / 2.0, PredictFrame, 0);
            }
            else
            {
                mPredictions->SetValue(0.0, PredictFrame, 0);
            }
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        }
#endif // REPOSITORY_USED
    }

private:

    void
    BlurImage(Array2dScalarReal64*& channel,double sigma)
    {

        Array2dScalarReal64* blurred=0;
        RecGauss(blurred,channel,sigma,sigma,mBlurDer,mBlurDer,1);
        if(mBlurDer>0){
            Mul(channel,channel,blurred);
            DivVal(channel,channel,PixMax(blurred));
            RecGauss(blurred,channel,sigma,sigma,0,0,1);
        }

        delete channel;
        channel=blurred;
    }

    void
    UpdateHistograms(unsigned char* data)
    {
        HistType** tmpHist;
        tmpHist = new HistType*[mHistPerFrame];
        //Back up the first frame's histogram pointers
        for(int i=0; i<mHistPerFrame ;i++)
            tmpHist[i] = mHistograms[i];
        //Shift the rest of the histograms
        for(int i=0 ; i<(mWindowSize-1)*mHistPerFrame ; i++)
            mHistograms[i] = mHistograms[i+mHistPerFrame];

        for(int i=(mWindowSize-1)*mHistPerFrame; i<mWindowSize*mHistPerFrame ;i++)
            mHistograms[i]=tmpHist[i-(mWindowSize-1)*mHistPerFrame];
        delete [] tmpHist;

        Array::Array2dVec3Real64* srcWrap =
            Array::ArrayCreate<Array::Array2dVec3Real64>(mSrc->FrameWidth(),
                                                         mSrc->FrameHeight(),
                                                         0, 0);

        Array::MakeFromData2<Array::Array2dVec3Real64,Array::Array2dVec3UInt8>(srcWrap,data);
        
        //OOO
        //Array2dVec3Real64* srcOOO=0;
        //ColorSpace(srcOOO,srcWrap,Element::RGB,Element::OOO);

        Array::Array2dScalarReal64* srcO1 = 0;
        Array::Array2dScalarReal64* srcO2 = 0;
        Array::Array2dScalarReal64* srcO3 = 0;

        Array::ProjectRange(srcO1,srcWrap,1);
        Array::ProjectRange(srcO2,srcWrap,2);
        Array::ProjectRange(srcO3,srcWrap,3);
    
        if (mBlur)
        {
            BlurImage(srcO1,mBlurSigma);
            BlurImage(srcO2,mBlurSigma);
            BlurImage(srcO3,mBlurSigma);
        }

        int k = (mWindowSize-1)*mHistPerFrame-1;

        for(int i=0;i<mPyramid.NrLevels();i++){
            for (int j=0;j<mPyramid.NrRects(i);j++){

                MakeHistogram1d(mHistograms[++k],srcO1,mPyramid.Rect(i,j));
                if(mHistEq)
                    EqualizeHist(mHistograms[k]);
                if(mHistNorm)
                    mHistograms[k]->Normalize();
                
                MakeHistogram1d(mHistograms[++k],srcO2,mPyramid.Rect(i,j));
                if(mHistEq)
                    EqualizeHist(mHistograms[k]);
                if(mHistNorm)
                    mHistograms[k]->Normalize();

                MakeHistogram1d(mHistograms[++k],srcO3,mPyramid.Rect(i,j));
                if(mHistEq)
                    EqualizeHist(mHistograms[k]);
                if(mHistNorm)
                    mHistograms[k]->Normalize();
            }
        }

        delete srcWrap;
        //delete srcOOO;

        delete srcO1;
        delete srcO2;
        delete srcO3;
    }

    void
    EqualizeHist(HistType* h)
    {
        for(int i=0;i<h->Size();i++){
            Real64 tot = h->TotalWeight();
            Real64 cum = 0;
            for(int i=0 ; i<h->Size() ; i++)
            {
                cum += h->Elem(i);
                h->Elem(i) = cum / tot;
            }
        }
    }

    void
    ComputeSSWeights()
    {   
    
        for (int i=0;i<mWindowSize;i++)
        {
            for (int j=0;j<mWindowSize;j++)
            {
                //OOO
                Real64 weight=0;
                for(int k=0;k<mHistPerFrame;k++){
                    if(mHistSim == "Intersection")
                        weight += Core::Vector::HistogramIntersection
                                        (*mHistograms[i*mHistPerFrame+k],
                                         *mHistograms[j*mHistPerFrame+k]);
                    else if(mHistSim == "Kullback")
                        weight += Core::Vector::KullbackDivergence
                                        (*mHistograms[i*mHistPerFrame+k],
                                         *mHistograms[j*mHistPerFrame+k]);
                }
                
                if(mHistSim != "Kullback")
                    mWeights->SetValue(weight*mGauss->Value(mWindowSize+i-j,0),i,j);
                else
                    mWeights->SetValue(pow(2.0,-weight)*mGauss->Value(mWindowSize+i-j,0),i,j);
            }
        }

    }

    Real64
    CalculateScores()
    {
        if(mNormW)
        {
            //Scale the weights for visualization
            MulVal(mWeights,mWeights,1/PixMax(mWeights));
        }
        int center = mCentered ? mRadius +1 : mRadius;

        Real64 DisSimA=0;
        Real64 DisSimB=0;
        Real64 SimA=0;
        Real64 SimB=0;
        //include ==mRadius also
        for(int i=0;i<mRadius;i++)
        {
            for(int j=0;j<mRadius;j++)
            {
                SimA+=mWeights->Value(i,j);
                DisSimA+=mWeights->Value(i,j+center);
            }
        }

        for(int i=center;i<mWindowSize;i++)
        {
            for(int j=0;j<mRadius;j++)
            {
                SimB+=mWeights->Value(i,j+center);
                DisSimB+=mWeights->Value(i,j);
            }
            
        }
        Real64 CutAB=DisSimA+DisSimB;
        Real64 NCutAB=SimA+SimB;

        Real64 s = CutAB/SimA+CutAB/SimB;
        Real64 Ns=NCutAB/DisSimA+NCutAB/DisSimB;

        if((SimA*SimB==0)||(DisSimA*DisSimB==0))
        {
            return 4;
        }

        if(mSim=="+")
            return s;
        else if(mSim=="-")
            return Ns;
        else if(mSim=="log+")
            return log(s);
        else if(mSim=="log-")
            return log(Ns);
        return 0;
    }
    
    void 
    ReadGroundTruth()
    {

        String XMLName=Impala::FileNameBase(mSrc->GetName());
        XMLName= mRefDir + mRefPrefix + XMLName + String(".xml");

        //Create a new xml reader
        Persistency::XmlFileReader* xfr = new Persistency::XmlFileReader;
        //If the file exists or not
        if(!FileExists(XMLName))
        {
            //there is no groundtruth data, so that means it is 
            //not one of the training videos.
            ILOG_INFO("File Does not exist "<<XMLName);
            return;
        }
        if(mVerbose>1)
            ILOG_INFO("Reading Ground truth from "<<XMLName);

        XERCES_CPP_NAMESPACE_QUALIFIER DOMDocument* doc=
        xfr->Read(XMLName);

        //Find all nodes named trans
        XERCES_CPP_NAMESPACE_QUALIFIER DOMNodeList* elems=
        xfr->FindElementsByName(doc,"trans");

        for (XMLSize_t i = 0; i< elems->getLength(); i++)
        {
            String type=xfr->GetAttributeValue(elems->item(i)->getAttributes(),"type");
            int col=0;
            if(type=="CUT")
                col=0;
            if(type=="DIS")
                col=1;
            if(type=="OTH")
                col=2;
            if(type=="FOI")
                col=3;
            int start=atoi(xfr->GetAttributeValue(elems->item(i)->getAttributes(),"preFNum").c_str());
            int end=atoi(xfr->GetAttributeValue(elems->item(i)->getAttributes(),"postFNum").c_str());
            //If the gradual transition is less than mShortGrad, add it as a CUT
            if(start-end<mShortGrad){
                mGroundTruth->SetValue(1,(start+end)/2-1,0);
                mGroundTruth->SetValue(1,(start+end)/2,0);
            }
            for(int i=start;i<=end;i++)
            {
                mGroundTruth->SetValue(1,i,col);
            }
        }
        delete xfr;

        SetPart(mCUTs, mGroundTruth, 0,0, mGroundTruth->CW(), 1, 0, 0);
        SetPart(mDISs, mGroundTruth, 0,1, mGroundTruth->CW(), 1, 0, 0);
        SetPart(mOTHs, mGroundTruth, 0,2, mGroundTruth->CW(), 1, 0, 0);
        SetPart(mFOIs, mGroundTruth, 0,3, mGroundTruth->CW(), 1, 0, 0);

    } 

    int mVerbose;
    double mThreshold;
    double mProbThresh;
    int mRadius;
    double mSigma;
    double mBlurSigma;
    int mBinCount;
    bool mHistEq;
    bool mHistNorm;
    bool mNormW;
    bool mBlur;
    bool mCentered;
    int  mBlurDer;
    String mHistSim;
    String mSymmetry;
    int mWindowSize;
    int mShortGrad;
    
    String mSvmModelFile;
    String mDataDir;
    String mRefDir;
    String mRefPrefix;
    String mMode;
    String mRunName;
    bool   mGUI;
    bool   mThresholdOnly;
    bool   mPredictOnly;
    bool   mPredictOutput;
    bool   mDoProcessing;

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    Core::Training::Svm* mSvm;
#endif // REPOSITORY_USED
    
    Stream::RgbDataSrc* mSrc;
    
    Array2dScalarReal64* mGauss;
    Array2dScalarReal64* mWeights;
    Array2dScalarReal64* mScores;
    Array2dScalarReal64* mPredictions;
    
    Array2dScalarReal64* mGroundTruth;
    Array2dScalarReal64* mCUTs;
    Array2dScalarReal64* mDISs;
    Array2dScalarReal64* mOTHs;
    Array2dScalarReal64* mFOIs;
    
    Vector::VectorTem<Real64>* mFeatures;
    Vector::VectorTem<Real64>* mFeaturesReversed;
    
    String     mSim;
    HistType** mHistograms;

    Reporter*     mReporter;
    int           mShotNr;

    Geometry::RectanglePyramid mPyramid;
    int mHistPerFrame;
    int mPyrL,mPyrU;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(ShotSegmenter, Impala.Core.VideoSet);

} // namespace VideoSet
} // namespace Core
} // namespace Impala

#endif

---