|| || || || || || || ||

---

ShotSegmenterGUI.h

Go to the documentation of this file.

#ifndef Impala_Core_VideoSet_ShotSegmenterGUI_h
#define Impala_Core_VideoSet_ShotSegmenterGUI_h

#include "Basis/CmdOptions.h"
#include "OglGui/Strut.h"
#include "Core/Geometry/Rectangle.h"
#include "Core/Geometry/RectanglePyramid.h"
#include "Core/Stream/RgbDataSrc.h"
#include "Core/Stream/RgbDataSrcFactory.h"
#include "Visualization/VideoNav.h"
#include "Visualization/AppControlSrc.h"
// since we are not using libraries:
#include "Visualization/Plot/Plot.h"
#include "Visualization/Plot/Line.h"
#include "Core/Vector/VectorTem.h"
#include "Core/Array/MulVal.h"
#include "Core/Array/PrintData.h"
#include "Core/Array/MakeFrom3Images.h"
#include "Core/Array/GaussDerivative2d.h"
#include "Core/VideoSet/ShotSegmenter.h"
#undef Next



namespace Impala
{
namespace Core
{
namespace VideoSet
{

using namespace Util;
using namespace Visualization;
using namespace Core::Array;
using namespace Core::Vector;
using namespace Core::Stream;
using namespace Core::Histogram;
using namespace Core::Geometry;

class ShotSegmenterGUI : public Window, public AppControlSrc,
                   public VideoNavListener, public OglGui::ViewListener
{
public:

    ShotSegmenterGUI(int x, int y, bool doZoom, bool mainIs2d, bool vpIs2d,
               RgbDataSrc* src) :
        Window(x, y, SuggestWndWidth(1, 2*src->FrameWidth()+src->FrameHeight()) + 30,
               SuggestWndHeight(1, 2*src->FrameHeight())+30,  mainIs2d),
        AppControlSrc(1)
    {
        //Get The Command line options
        CmdOptions& options = CmdOptions::GetInstance();
        mBlurSigma = options.GetDouble("SSBlurSigma");
        mBlurDer   = options.GetInt("SSBlurDer");
        mShotSeg = new ShotSegmenter(0,options);
        mShotSeg->HandleNewFile(0,0,src);

        //Set the source
        SetSource(src);

        //standart video navigation stuff
        mVideoDecision = new VideoNav(this,mPredictSrc);
        mVideoDecision->SetVideoNavListener(this);

        new OglGui::Strut(this,src->FrameHeight()+5,1);

        mVideoProcess = new VideoNav(this, src);
        mVideoProcess->SetVideoNavListener(this);

        //Initialize plots and views
        int WindowWidth=W()-10;
        int HalfWindowWidth=WindowWidth/2;

        /*mHistogramPlot = new Plot::Plot(this,   5,  175,
                                                HalfWindowWidth,  150);
        mHistogramPlottable = new Plot::Line(0, true);
        mHistogramPlottable->SetColor(0,0.5,1);
        mHistogramPlot->Add(mHistogramPlottable);

        //Histogram from SrcWindow
        mHistogramPlotW = new Plot::Plot(this,  HalfWindowWidth+10,  175,
                                                HalfWindowWidth,     150);
        mHistogramPlottableW = new Plot::Line(0, true);
        mHistogramPlottableW->SetColor(1,0.5,0);
        mHistogramPlotW->Add(mHistogramPlottableW);
        */
        mScoresPlot = new Plot::Plot(this, 5,    30,
                                           W()-10,src->FrameHeight()-30);
        mScoresPlot->SetDisableGlobalKeyListener(false);
        mScoresPlot->SetDisableOGLViewKeys(false);

        mScoresPlottable = new Plot::Line(0, true);
        mProbsPlottable = new Plot::Line(0, true);
        mCUTPlottable = new Plot::Line(0, true);
        mDISPlottable = new Plot::Line(0, true);
        mOTHPlottable = new Plot::Line(0, true);
        mFOIPlottable = new Plot::Line(0, true);
        mProcessRange  = new Plot::Line(0, true);
        mDecisionRange = new Plot::Line(0, true);

        mPlotScale1 = new Plot::Line(0, true);
        mPlotScale2 = new Plot::Line(0, true);
        mPlotScale3 = new Plot::Line(0, true);
        mPlotScale4 = new Plot::Line(0, true);
        mPlotScale5 = new Plot::Line(0, true);

        mDecisionBoundaryPlottable = new Plot::Line(0, true);

        mScoresPlot->Add(mPlotScale1);
        mScoresPlot->Add(mPlotScale2);
        mScoresPlot->Add(mPlotScale3);
        mScoresPlot->Add(mPlotScale4);
        mScoresPlot->Add(mPlotScale5);
        mScoresPlot->Add(mProcessRange);
        mScoresPlot->Add(mDecisionRange);
        mScoresPlot->Add(mScoresPlottable);
        mScoresPlot->Add(mProbsPlottable);
        mScoresPlot->Add(mCUTPlottable);
        mScoresPlot->Add(mDISPlottable);
        mScoresPlot->Add(mOTHPlottable);
        mScoresPlot->Add(mFOIPlottable);

        mScoresPlot->Add(mDecisionBoundaryPlottable);
        mScoresPlot->SetBackground(0xff000000);

        mScoresPlottable->SetDomain(0,src->LastFrame());
        mProbsPlottable->SetDomain(0,src->LastFrame());
        mCUTPlottable->SetDomain(0,src->LastFrame());
        mDISPlottable->SetDomain(0,src->LastFrame());
        mOTHPlottable->SetDomain(0,src->LastFrame());
        mFOIPlottable->SetDomain(0,src->LastFrame());
        mProcessRange->SetDomain(0,src->LastFrame());
        mDecisionRange->SetDomain(0,src->LastFrame());
        mPlotScale1->SetDomain(0,src->LastFrame());
        mPlotScale2->SetDomain(0,src->LastFrame());
        mPlotScale3->SetDomain(0,src->LastFrame());
        mPlotScale4->SetDomain(0,src->LastFrame());
        mPlotScale5->SetDomain(0,src->LastFrame());
        mDecisionBoundaryPlottable->SetDomain(0,src->LastFrame());

        mScoresPlottable->SetColor(1,0,0);//Red
        mProbsPlottable->SetColor(1,1,0);//Yellow
        mCUTPlottable->SetColor(1,0.5,0);//Orange
        mDISPlottable->SetColor(0,1,0.5);//Green
        mOTHPlottable->SetColor(0,0.5,1);//Blue
        mFOIPlottable->SetColor(0.5,0,1);//Purple

        mPlotScale1->SetColor(0.4,0.4,0.4);
        mPlotScale2->SetColor(0.4,0.4,0.4);
        mPlotScale3->SetColor(0.4,0.4,0.4);
        mPlotScale4->SetColor(0.4,0.4,0.4);
        mPlotScale5->SetColor(0.4,0.4,0.4);

        mProcessRange->SetColor(1,1,1);
        mDecisionRange->SetColor(1,1,0);

        mDecisionBoundaryPlottable->SetColor(0,1,0);//green

        mDecisionBoundary = ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        SetVal(mDecisionBoundary,mShotSeg->GetProbThresh());

        mWeightsView=new View(mOglWnd,
                    src->FrameWidth()+10,
                    this->H()-src->FrameHeight()-3,
                    src->FrameHeight(),
                    src->FrameHeight(),1,true,true,true);

        mWeights = mShotSeg->GetWeights();

        mScoresPlottable->SetData(mShotSeg->GetScores());

        mProbsPlottable->SetData(mShotSeg->GetPredictions());
        mCUTPlottable->SetData(mShotSeg->GetCUTs());
        mDISPlottable->SetData(mShotSeg->GetDISs());
        mOTHPlottable->SetData(mShotSeg->GetOTHs());
        mFOIPlottable->SetData(mShotSeg->GetFOIs());

        mDecisionBoundaryPlottable->SetData(mDecisionBoundary);
        Array2dScalarReal64* s1=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        Array2dScalarReal64* s2=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        Array2dScalarReal64* s3=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        Array2dScalarReal64* s4=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        Array2dScalarReal64* s5=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        mPr=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        mDr=ArrayCreate<Array2dScalarReal64>(GetSrc()->LastFrame(),1,0,0);
        
        SetVal(s1,0.25);
        SetVal(s2,0.5);
        SetVal(s3,1.);
        SetVal(s4,2.);
        SetVal(s5,4.);
        SetVal(mPr,0);
        SetVal(mDr,0);

        mPlotScale1->SetData(s1);
        mPlotScale2->SetData(s2);
        mPlotScale3->SetData(s3);
        mPlotScale4->SetData(s4);
        mPlotScale5->SetData(s5);
        mProcessRange->SetData(mPr);
        mDecisionRange->SetData(mDr);



#ifdef OGL_USING_GLUT
        SetAlwaysDraw(); // UpdateScene() doesn't work.
#endif
        AppController::Instance().MainLoop();

    }

    void SetCUTs(Array2dScalarReal64* CUTS)
    {
        mCUTPlottable->SetData(CUTS);
    }
    void SetFOIs(Array2dScalarReal64* FOIS)
    {
        mFOIPlottable->SetData(FOIS);
    }
    void SetDISs(Array2dScalarReal64* DISS)
    {
        mDISPlottable->SetData(DISS);
    }
    void SetOTHs(Array2dScalarReal64* OTHS)
    {
        mOTHPlottable->SetData(OTHS);
    }
    void SetScores(Array2dScalarReal64* Scores)
    {
        mScoresPlottable->SetData(Scores);
    }
    void SetPredictions(Array2dScalarReal64* Predictions)
    {
        mProbsPlottable->SetData(Predictions);
    }
    void SetWeights(Array2dScalarReal64* Weights)
    {
        mWeights=Weights;
    }
    void SetSource(RgbDataSrc* src){
        SetSrc(src);
        AppController::Instance().AddControl((AppControlSrc*) this, true);

        RgbDataSrcFactory& factory = RgbDataSrcFactory::Instance();
        mPredictSrc = factory.Construct( src->GetName(),CmdOptions::GetInstance().GetString("src"));
    }

    void
    MainLoop(){
        AppController::Instance().MainLoop();
    }

    void
    HandleCycleSrc()
    {
        UpdateScene();
        SetStatusStr((char*) GetFpsString().c_str());
    }

    void
    HandleDestroyEvent()
    {
    //    mShotSeg->HandleDoneFile();
    }
    
    void
    BlurImage(Array2dVec3UInt8*& rgb,double sigma)
    {

        Array2dScalarUInt8* R=0;
        Array2dScalarUInt8* G=0;
        Array2dScalarUInt8* B=0;

        ProjectRange(R,rgb,1);
        ProjectRange(G,rgb,2);
        ProjectRange(B,rgb,3);
        
        Array2dScalarReal64* RReal=0;
        Array2dScalarReal64* GReal=0;
        Array2dScalarReal64* BReal=0;

        Array2dScalarReal64* RRealp=0;
        Array2dScalarReal64* GRealp=0;
        Array2dScalarReal64* BRealp=0;

        Set(RReal,R);
        Set(GReal,G);
        Set(BReal,B);
        
        RecGauss(RRealp,RReal,sigma,sigma,mBlurDer,mBlurDer,1);
        RecGauss(GRealp,GReal,sigma,sigma,mBlurDer,mBlurDer,1);
        RecGauss(BRealp,BReal,sigma,sigma,mBlurDer,mBlurDer,1);
        if(mBlurDer>0){
            Mul(RReal,RReal,RRealp);
            Mul(GReal,GReal,GRealp);
            Mul(BReal,BReal,BRealp);
            DivVal(RReal,RReal,PixMax(RRealp));
            DivVal(GReal,GReal,PixMax(GRealp));
            DivVal(BReal,BReal,PixMax(BRealp));
            RecGauss(RRealp,RReal,sigma,sigma,0,0,1);
            RecGauss(GRealp,GReal,sigma,sigma,0,0,1);
            RecGauss(BRealp,BReal,sigma,sigma,0,0,1);
        }

        delete rgb;
        delete R;
        delete G;
        delete B;

        R=G=B=0;
        rgb=0;

        Set(R,RRealp);
        Set(G,GRealp);
        Set(B,BRealp);

        MakeFrom3Images(rgb,R,G,B);

        delete R;delete G;delete B;
        delete RReal;delete GReal;delete BReal;
        delete RRealp;delete GRealp;delete BRealp;

    }

    void
    HandleNewFrame()
    {
        int ProcessFrame = GetSrc()->FrameNr();
        int PredictionFrame = mShotSeg->GetPredictionFrame();
        mPredictSrc->GotoFrame(PredictionFrame+1);
        mPr->SetValue(0,ProcessFrame-1,0);
        mPr->SetValue(0.5,ProcessFrame,0);
        mDr->SetValue(0,PredictionFrame-1,0);
        mDr->SetValue(0.5,PredictionFrame,0);

        mShotSeg->HandleNewFrame();
        //Update the frame
        ArraySystem::Instance().MarkMemoryUsage();
        Array2dVec3UInt8* rgb1 = Array::ArrayCreate<Array2dVec3UInt8>
                    (GetSrc()->FrameWidth(), GetSrc()->FrameHeight(), 
                        0, 0,mPredictSrc->DataPtr(),true);
        if(mShotSeg->Blur())
            BlurImage(rgb1,mBlurSigma);
        Array2dVec3UInt8* rgb2 = Array::ArrayCreate<Array2dVec3UInt8>
                    (GetSrc()->FrameWidth(), GetSrc()->FrameHeight(), 
                        0, 0,GetSrc()->DataPtr(),true);
        if(mShotSeg->Blur())
            BlurImage(rgb2,mBlurSigma);

        mVideoDecision->UpdateView(0,rgb1,"");
        mVideoProcess->UpdateView(0,rgb2,"");
        delete rgb1;
        delete rgb2;

        //Scale the weights for visualization
        MulVal(mWeights,mWeights,255/PixMax(mWeights));

        mScoresPlot->SetCenter(ProcessFrame-50,2,0);
        mScoresPlot->SetRange(100,5,1);

        mWeightsView->UpdateImage(mWeights,std::string(""));
        mWeightsView->SetZoom(mWeightsView->W()/mWeights->CW(),
                              mWeightsView->H()/mWeights->CH());

    }

    // specialization of base classes

    void
    NavFrameEvent(VideoNav* src, int requestedFrame, void* listenerData)
    {
        Jump(requestedFrame);
    }

    virtual void
    OnViewMouse(OglGui::View* view, int msg, int btn, int state,
                float x, float y)
    {
    }

    VideoNav*         mVideoDecision;
    VideoNav*         mVideoProcess;
    View*             mWeightsView;

    Plot::Plot*       mHistogramPlot;
    Plot::Plot*       mHistogramPlotW;
    Plot::Line*       mHistogramPlottable;
    Plot::Line*       mHistogramPlottableW;

    Plot::Plot*       mScoresPlot;
    Plot::Line*       mScoresPlottable;
    Plot::Line*       mProbsPlottable;
    Plot::Line*       mCUTPlottable;
    Plot::Line*       mDISPlottable;
    Plot::Line*       mOTHPlottable;
    Plot::Line*       mFOIPlottable;
    Plot::Line*       mDecisionBoundaryPlottable;
    Plot::Line*       mPlotScale1;
    Plot::Line*       mPlotScale2;
    Plot::Line*       mPlotScale3;
    Plot::Line*       mPlotScale4;
    Plot::Line*       mPlotScale5;
    Plot::Line*       mProcessRange;
    Plot::Line*       mDecisionRange;


    RgbDataSrc* mPredictSrc;

    Array2dScalarReal64* mWeights;
    Array2dScalarReal64* mPr;
    Array2dScalarReal64* mDr;
    Array2dScalarReal64* mDecisionBoundary;

    ShotSegmenter* mShotSeg;
    double mBlurSigma;
    int    mBlurDer;

};
} // namespace VideoSet
} // namespace Core
} // namespace Impala
#endif

---