|| || || || || || || ||

---

TrackerBackFore.h

Go to the documentation of this file.

#ifndef Impala_Core_Tracking_TrackerBackFore_h
#define Impala_Core_Tracking_TrackerBackFore_h

#include <iosfwd>

#include "Core/Array/Arrays.h"
#include "Core/Array/SetVal.h"
#include "Core/Array/AddVal.h"
#include "Core/Array/MulVal.h"
#include "Core/Array/MakeFromData.h"
#include "Core/Array/Pattern/FuncGenConv2d.h"
#include "Core/Array/Pattern/PatSet.h"
#include "Core/Array/Pattern/PatInOutOp.h"
#include "Core/Array/Trait/ExportMinMaxPos.h"

#include "Util/TimeStats.h"
#include "Core/Geometry/PointZ.h"
#include "Core/Geometry/Rectangle.h"

#include "Core/Tracking/Tracker.h"
#include "Core/Tracking/ConvolutionTraits.h"
#include "Core/Tracking/FrameRegionTable.h"
#include "Core/Tracking/GaborFilter.h"
#include "Core/Tracking/Classifier.h"

namespace Impala
{
namespace Core
{
namespace Tracking
{

const int cColorNum = 3;

class TrackerBackFore : public Tracker
{
public:
    TrackerBackFore(CmdOptions& options) :
        mSampleSpacing(options.GetInt("fore_back.sampleSpacing", 5)),
        mFilterSize(options.GetInt("fore_back.filterSize", 19)),
        mGabor(options.GetInt("fore_back.gabor.scale", 4),
               options.GetInt("fore_back.gabor.frequency", 2),
               options.GetInt("fore_back.gabor.directionCount", 2),
               mFilterSize)
    {
        mDimensionality = mGabor.GetFilterCount() * cColorNum; //the times 2 is because the gabor filters have a sine and cosine fliter
        mInput = 0;
        mSearchKernel = 0;
        mSearchResult = new Array::Array2dScalarReal64(21, 21, 0, 0);
        mReconstruction = 0;
        //mSimilarityPerFrame = 0;
        mClassifier = new Classifier(0, mDimensionality);

        mStats.AddGroup("features");
        mStats.AddGroup("statistics");
        mStats.AddGroup("backproject");
        mStats.AddGroup("search");

        mTrack = 0;
        mDebugFile.open(options.GetString("fore_back.tracker-debug-file").c_str());

    }

    virtual ~TrackerBackFore() // note: destructor not implemented correctly
    {
        if (mInput)
            delete mInput;
        if (mReconstruction)
            delete mReconstruction;
        if (mSearchKernel)
            delete mSearchKernel;
        delete mSearchResult;
        mDebugFile.close();
    }


    void Initialize(Array::Array2dVec3UInt8* startImage, Tracking::FrameRegion* startPos, FrameRegionTable* resultTrack)
    {
        mTrack = resultTrack;
        // TODO: configure the color model, the types of filters and the number of orientations
        if (mInput)
            delete mInput;
        mWidth = startImage->W();
        mHeight = startImage->H();

        mTrack->Add(*startPos);

        mInput = new Array::Array2dVec3Real64(mWidth, mHeight, 21, 21);
        Array::Pattern::PatSet(mInput, startImage, 2, 2);

        if (mReconstruction)
            delete mReconstruction;
        mReconstruction = new Array::Array2dVec3UInt8(mWidth, mHeight, 0, 0);
        Geometry::Rectangle r = startPos->GetRectangle();
        NewPosition(r.mLeft, r.mTop, r.mRight, r.mBottom);
        mInitialLikelyhood = 0;
    }

    void Process(Array::Array2dVec3UInt8* image)
    {
        Array::Pattern::PatSet(mInput, image, 2, 2);
        Array::Pattern::PatSet(mReconstruction, mInput);
        if (ObjectVisible())
        {
            SearchObject();
            UpdateObject();
        }
        mTrack->Add(Core::Geometry::Rectangle(mPosition.mLeft,mPosition.mTop,mPosition.mRight,mPosition.mBottom));
        //Array::Array2dScalarReal64* copy = 0;
        //Array::MakeFromData2(copy, mSearchResult);
        //mSimilarityPerFrame->push_back(copy);
    }

    Array::Array2dVec3UInt8* GetReconstruction()
    {
        return mReconstruction;
    }

    bool ObjectVisible()
    {
        if (mPosition.mLeft >= mInput->CW())
            return false;
        if (mPosition.mTop >= mInput->CH())
            return false;
        if (mPosition.mRight <= 0)
            return false;
        if (mPosition.mBottom <= 0)
            return false;
        return true;
    }

    void GetFilterFromPosition(int x, int y, int& filterX, int& filterY)
    {
        if(x >= mSearchKernel->CW() || y >= mSearchKernel->CH())
            return;

        int w, h;
        w = mPosition.Width()/mSampleSpacing;
        h = mPosition.Height()/mSampleSpacing;
        x -= mFilterSize/2;
        y -= mFilterSize/2;

        filterX = x/mSampleSpacing;
        if (filterX < 0)
            filterX = 0;
        if (filterX >= w)
            filterX = w-1;

        filterY = y/mSampleSpacing;
        if (filterY < 0)
            filterY = 0;
        if (filterY >= h)
            filterY = h-1;
    }

    //DataArray* GetSimilarityData(); //transfers possession to caller

    double mLikelyhood;
    double mInitialLikelyhood;

private:
    void SearchObject()
    {
        // find best match using the classifiers (eq.10)
        Array::Array2dScalarReal64 response(mDimensionality, 1, 0, 0);
        Array::SetVal<Array::Array2dVec3Real64>(mSearchKernel, mSearchKernel, 0);
        int w,h;
        w = mPosition.Width()/mSampleSpacing;
        h = mPosition.Height()/mSampleSpacing;
        int i,j;

        mStats.SelectGroup(2);
        mStats.StartTime();
        for(j=0 ; j<h ; j++)
        {
            for(i=0 ; i<w ; i++)
            {
                mClassifier->GetClassifiers(i+j*w, response);
                mGabor.Reconstruct(*mSearchKernel, response, i*mSampleSpacing, j*mSampleSpacing);
            }
        }
        mStats.StopTime();

        mStats.SelectGroup(3);
        mStats.StartTime();
        int bw = mSearchKernel->CW()/2 +1;
        int bh = mSearchKernel->CH()/2 +1;

        Array::Array2dVec3Real64* searchArea = new Array::Array2dVec3Real64(21, 21, bw, bh);
        FillSearchArea(searchArea, bw, bh);
        SetVal(mSearchResult, mSearchResult, 0); //weird that this is necessary, you'd expect that the convolution would initialise the result

        TraitBpoInproduct prod;
        TraitBpoAddAssignPtr bpoAdd;
        searchArea->FindNaN("searchArea");
        mSearchKernel->FindNaN("mSearchKernel");
        mSearchResult->FindNaN("mSearchResult");
        Array::Pattern::FuncGenConv2dDispatch(mSearchResult, searchArea, mSearchKernel, prod, bpoAdd); // why didn't i use the pattern? maybe because the pattern reallocates the destination...
        mSearchResult->FindNaN("mSearchResult2");

        Array::Trait::ExportMinMaxPos<double, Array::Array2dScalarReal64> minmax;
        Array::Pattern::PatInOutOp(mSearchResult, minmax);

        mPosition.mLeft += minmax.mMaxPos.mX - 10;
        mPosition.mTop += minmax.mMaxPos.mY - 10;
        mPosition.mRight += minmax.mMaxPos.mX - 10; 
        mPosition.mBottom += minmax.mMaxPos.mY - 10;

        FormBackground();
        mStats.StopTime();

        // warning apparently mSearchResult can contain 'not-a-number's which yields a faulty position
        if(minmax.mMaxVal < -10000)
            mLikelyhood = mInitialLikelyhood * 0.1;
        else
        {
            // to really get a likelyhood we should devide by the maximum possible score, but what's that?
            mLikelyhood = mSearchResult->Value(minmax.mMaxPos.mX, minmax.mMaxPos.mY);
        }
        if(mInitialLikelyhood == 0)
            mInitialLikelyhood = mLikelyhood;

        delete searchArea;
    }

    void UpdateObject()
    {
        // Get gabor response in the foreground (eq.4)
        // blend with the current foreground (eq.17)
        int w,h;
        w = mPosition.Width()/mSampleSpacing;
        h = mPosition.Height()/mSampleSpacing;

        mStats.SelectGroup(0);
        mStats.StartTime();
        Array::Array2dScalarReal64 response(mDimensionality, 1, 0, 0);
        int i,j;
        for(j=0 ; j<h ; j++)
        {
            for(i=0 ; i<w; i++)
            {
                int x = mPosition.mLeft+mSampleOffset.mX+(i*mSampleSpacing);
                int y = mPosition.mTop+mSampleOffset.mY+(j*mSampleSpacing);
                mGabor.Apply(response.CPB(0, 0), *mInput, x, y);
                mClassifier->UpdateVec(response,i+j*w);
            }
        }
        mStats.StopTime();

        mStats.SelectGroup(1);
        mStats.StartTime();
        Array::Array2dScalarReal64 background(mDimensionality, mBackMaxPoints, 0, 0);
        ComputeBackgroundResponse(background);
        mClassifier->Update(background);
        mClassifier->WriteSimpleDebug(mDebugFile);
    }

    void NewPosition(int left, int top, int right, int bottom)
    {
        mStats.Reset();

        // clip against the image
        if (left < 0)
            left = 0;
        if (top < 0)
            top = 0;
        if (right >= mInput->CW())
            right = mInput->CW() - 1;
        if (bottom >= mInput->CH())
            bottom = mInput->CH() - 1;

        mPosition.mLeft = left;
        mPosition.mTop = top;
        mPosition.mRight = right;
        mPosition.mBottom = bottom;
        mSampleOffset.mX = ((mPosition.Width() % mSampleSpacing) + mSampleSpacing) / 2;
        mSampleOffset.mY = ((mPosition.Height() % mSampleSpacing) + mSampleSpacing) / 2;

        FormBackground();
        if(mSearchKernel)
            delete mSearchKernel;

        int w = mPosition.Width() / mSampleSpacing;
        int h = mPosition.Height() / mSampleSpacing;
        mSearchKernel = new Array::Array2dVec3Real64(mFilterSize + (w-1)*mSampleSpacing, mFilterSize + (h-1)*mSampleSpacing, 0, 0);

        // this intialises the classifier
        SetForeground();
        Array::Array2dScalarReal64 background(mDimensionality, mBackMaxPoints, 0, 0);
        ComputeBackgroundResponse(background);
        mClassifier->SetBackground(background);
    }


    void SetForeground()
    {
        // this funtion implements eq.4 of the paper
        Array::Array2dScalarReal64 response(mDimensionality, 1, 0, 0);
        int w,h;
        w = mPosition.Width()/mSampleSpacing;
        h = mPosition.Height()/mSampleSpacing;

        delete mClassifier;
        mClassifier = new Classifier(w*h, mDimensionality);

        int left = mSampleOffset.mX + mPosition.mLeft;
        int top = mSampleOffset.mY + mPosition.mTop;
        for(int j=0 ; j<h ; j++)
            for(int i=0 ; i<w; i++)
            {
                mGabor.Apply(response.CPB(0,0), *mInput, left+(i*mSampleSpacing), top+(j*mSampleSpacing));
                mClassifier->SetVec(response, i+j*w);
            }
    }

    void FormBackground()
    {
        mBack[0].mLeft = mPosition.mLeft - 3*mSampleSpacing;
        mBack[0].mTop = mPosition.mTop - 3*mSampleSpacing;
        mBack[0].mRight = mPosition.mRight + 3*mSampleSpacing -1;
        mBack[0].mBottom = mPosition.mTop -1;

        mBack[1].mLeft = mPosition.mLeft - 3*mSampleSpacing;
        mBack[1].mTop = mPosition.mTop - mSampleSpacing;
        mBack[1].mRight = mPosition.mLeft -1;
        mBack[1].mBottom = mPosition.mBottom + mSampleSpacing -1;

        mBack[2].mLeft = mPosition.mRight;
        mBack[2].mTop = mPosition.mTop - mSampleSpacing;
        mBack[2].mRight = mPosition.mRight + 3*mSampleSpacing -1;
        mBack[2].mBottom = mPosition.mBottom + mSampleSpacing -1;

        mBack[3].mLeft = mPosition.mLeft - 3*mSampleSpacing;
        mBack[3].mTop = mPosition.mBottom;
        mBack[3].mRight = mPosition.mRight + 3*mSampleSpacing -1;
        mBack[3].mBottom = mPosition.mBottom + 3*mSampleSpacing -1;

        if(mPosition.mLeft < 3*mSampleSpacing)
        {
            mBack[0].mLeft = 0;
            mBack[1].mLeft = 0;
            mBack[3].mLeft = 0;
        }
        if(mPosition.mTop < 3*mSampleSpacing)
        {
            mBack[0].mTop = 0;
        }
        if(mPosition.mRight+3*mSampleSpacing >= mWidth)
        {
            mBack[0].mRight = mWidth;
            mBack[2].mRight = mWidth;
            mBack[3].mRight = mWidth;
        }
        if(mPosition.mBottom+3*mSampleSpacing >= mHeight)
        {
            mBack[3].mBottom = mHeight;
        }

        int w = mBack[0].Width()/mSampleSpacing;
        int h = mBack[1].Height()/mSampleSpacing;
        ILOG_DEBUG("rect[0].w=" << w << " rect[1].h=" << h);
        w--;
        h--;
        ILOG_DEBUG("sample space = " << mSampleSpacing);
        mBackMaxPoints = (w * 4) + (h * 4);
        ILOG_DEBUG("maxpoints = "<<mBackMaxPoints<<" (4*"<<w<<" + 4*"<<h<<")");
    }

    void ComputeBackgroundResponse(Array::Array2dScalarReal64& background)
    {
        // get response of the background (eq.5)
        int count = 0;
        for(int b=0 ; b<4 ; b++)
        {
            int w,h;
            w = mBack[b].Width()/mSampleSpacing;
            h = mBack[b].Height()/mSampleSpacing;
            ILOG_DEBUG("rect = "<<mBack[b]<<", w="<<w<<", h="<<h);
            int i,j;
            for(j=1 ; j<h ; j++)
            {
                for(i=1 ; i<w; i++)
                {
                    double* data = background.CPB(0, count);
                    count++;
                    mGabor.Apply(data, *mInput, 
                                 mBack[b].mLeft+(i*mSampleSpacing),
                                 mBack[b].mTop+(j*mSampleSpacing));
                }
            }
            ILOG_DEBUG("count = "<<count);
        }
        while(count < background.CH()) // this actually never occurs
        {
            double* data = background.CPB(0, count);
            for(int j=0 ; j<background.CW() ; j++)
                data[j] = 0;
            count++;
        }
    }

    void FillSearchArea(Array::Array2dVec3Real64* searchArea, int bw, int bh)
    {


        // first cheat with the sizes of the searchArea
        int w = searchArea->W();
        int h = searchArea->H();
        searchArea->mBW = 0;
        searchArea->mBH = 0;
        searchArea->mCW = w;
        searchArea->mCH = h;
        SetVal(searchArea, searchArea, 0); 
        int x = mPosition.mLeft + mPosition.Width()/2;
        int y = mPosition.mTop + mPosition.Height()/2;

        // see if the search area lies partially outside the image
        Core::Geometry::Rectangle overflow;
        if (x - bw - 10 < 0)
            overflow.mLeft = -(x - bw - 10);
        if (y - bh - 10 < 0)
            overflow.mTop  = -(y - bh - 10);
        if (x + bw + 10 >= mInput->CW())
            overflow.mRight = x + bw + 10 - mInput->CW() + 1;
        if (y + bh + 10 >= mInput->CH())
            overflow.mBottom = y + bh + 10 - mInput->CH() + 1;

        // copy the data
        Array::Pattern::PatSet
            (searchArea, mInput,
            x - bw - 10 + overflow.mLeft,
            y - bh - 10 + overflow.mTop,
            2*bw + 21 - (overflow.mLeft + overflow.mRight),
            2*bh + 21 - (overflow.mTop + overflow.mBottom),
            0, 0);

        // restore correct dimensions
        searchArea->mCW = 21;
        searchArea->mCH = 21;
        searchArea->mBW = bw;
        searchArea->mBH = bh;
    }

    FrameRegionTable* mTrack;

    Array::Array2dVec3Real64* mInput;
    Util::TimeStats mStats;

    // data for tracking
    int mHeight, mWidth;
    Core::Geometry::Rectangle mPosition;
    Core::Geometry::PointZ mSampleOffset;
    Core::Geometry::Rectangle mBack[4];
    int mBackMaxPoints;
    Array::Array2dVec3Real64* mSearchKernel;
    Array::Array2dVec3UInt8* mReconstruction;
    Array::Array2dScalarReal64* mSearchResult;
    //DataArray* mSimilarityPerFrame;
    Classifier* mClassifier;
    int mSampleSpacing, mFilterSize;
    int mDimensionality;
    int mDirectionCount;
    GaborFilterSet mGabor;
    std::ofstream mDebugFile;

    static log4cpp::Category& sLog;
};

ILOG_VAR_INIT(TrackerBackFore, Impala.Core.Tracking);

}//namespace Tracking
}//namespace Core
}//namespace Impala

#endif // !Impala_Core_Tracking_TrackerBackFore_h

---