|| || || || || || || ||

---

VisSem.h

Go to the documentation of this file.

#ifndef Impala_Core_Feature_VisSem_h
#define Impala_Core_Feature_VisSem_h

#include "Core/Feature/Computor.h"
#include "Core/Vector/HistogramAccumulation.h"
#include "Core/Vector/Concat.h"
#include "Core/Vector/HistogramIntersection.h"
#include "Core/Vector/Similarity.h"
#include "Core/Vector/HistogramNormalization.h"
#include "Core/Vector/Avg.h"
#include "Core/Vector/Max.h"
#include "Core/Vector/ElemAvg.h"
#include "Core/Vector/ElemMax.h"
#include "Core/Vector/AddAssign.h"
#include "Core/Vector/Clear.h"
#include "Core/Vector/VectorSet.h"
#include "Core/Histogram/MakeHistogram1dSet.h"
#include "Core/Vector/WeibullSim.h"
#include "Core/Histogram/ComputeWeibull.h"
#include "Core/Array/InvWiccest.h"
#include "Core/Array/VisSemGabor.h"
#include "Core/Array/PixStat.h"

namespace Impala
{
namespace Core
{
namespace Feature
{


class VisSem : public Computor
{
public:

    typedef Array::Array2dVec3UInt8 Array2dVec3UInt8;
    typedef Array::Array2dScalarReal64 Array2dScalarReal64;
    typedef Vector::VectorTem<Real64> VectorReal64;
    typedef Array::ArraySet<Array2dScalarReal64> InvSetType;
    typedef Histogram::Histogram1dSet<Array2dScalarReal64> HistSetType;

    VisSem(String name, CmdOptions& options) : Computor(name, options)
    {
        if (GetName() == "vissem")
        {
            for (int i=0 ; i<options.GetInt("nrScales") ; i++)
            {
                String s = "spatialSigma_s" + MakeString(i);
                AddScale(options.GetDouble(s));
            }
            mDoRot = options.GetBool("doRot");
            mDoC = options.GetBool("doC");
            mHistBinCount = options.GetInt("histBinCount");
            SetRegionDescriptorLength(2); // weibulls
        }
        if (GetName() == "vissemgabor")
        {
            mNrOrient = 4;
            AddScale(2.828);
            mFreq.push_back(0.720);
            AddScale(1.414);
            mFreq.push_back(2.094);
            mHistBinCount = 101;
            mCumulativeHist = false;
            SetRegionDescriptorLength(mHistBinCount);
        }

        mDoCodebook = options.GetBool("doCodebook");
        ILOG_INFO("mDoCodebook: " << mDoCodebook);

        // do codeword uncertainty (instead of kernel codebook)
        mDoUNC = options.GetBool("doUNC");
        ILOG_INFO("doUNC: " << mDoUNC);

        mKeyframesName = FileNameBase(options.GetString("keyframeMask"));
        if (mKeyframesName == "")
            mKeyframesName = FileNameBase(options.GetString("keyframeMaskAnno"));
        if (mKeyframesName == "")
            mKeyframesName = "all";

        InvSetType invSet;  // to figure out what is to be computed
        ComputePixelFeatureSet(invSet, 0, 0);
        SetPixelFeatureSetSize(invSet.Size());

        if (!IsClusterBased())
            for (int i=0 ; i<GetNrPixelFeatureSets() ; i++)
                mHistAccu.push_back(NewHistSet());
    }

    // add additional information to feature definitions before saving them
    virtual FeatureDefinition
    ExtendFeatureDefinition(FeatureDefinition def)
    {
        def = Computor::ExtendFeatureDefinition(def);
        def.AddParameter("UNC", mDoUNC);
        def.AddParameter("Codebook", mDoCodebook);
        return def;
    }

    virtual FeatureTable*
    MakeProtoFeatureTable(int pixelFeatureSet, int regionFeatureSet,
                          String clusterType, String clusterVal)
    {
        String pName = GetPixelFeatureDefName(pixelFeatureSet);
        String pVal = GetPixelFeatureDefVal(pixelFeatureSet);
        String rName = GetRegionFeatureDefName(regionFeatureSet);
        String rVal = GetRegionFeatureDefVal(regionFeatureSet);
        FeatureDefinition def(GetName());
        def.AddParameter("proto", FileNameBase(mProtoDataFile));
        def.AddParameter("keyframes", mKeyframesName);
        def.AddParameter(pName, pVal);
        def.AddParameter(rName, rVal);
        def.AddParameter(clusterType, clusterVal);
        AddProtoFeatureTable(def, 1, GetRegionFeatureLength());
        return GetProtoFeatureTableSet()->GetTable(def);
    }

    virtual void
    ReadPrototypes(String protoSet, String maskSet, int maxNr)
    {
        ReadProtoFeatureTableSet(protoSet, maskSet);
        if (maxNr > 0)
            LimitProtoFeatureTableSize(maxNr);
        MakeFeatureTableSet();
    }

    virtual void
    ComputeRegionFeatures(Array::Array2dVec3UInt8* im, Quid quid)
    {
        HistSetType* hSet = NewHistSet();
        for (int p=0 ; p<GetNrPixelFeatureSets() ; p++)
        {
            InvSetType invSet;
            ComputePixelFeatureSet(invSet, p, im);
            Array2dScalarReal64* a = invSet.Array(0);
            mPyramid.SetImageSize(a->CW(), a->CH());
            for (int l=0 ; l<mPyramid.NrLevels() ; l++)
            {
                FeatureTable* tab = GetRegionFeatureTable(p, l);
                for (int r=0 ; r<mPyramid.NrRects(l) ; r++)
                {
                    Geometry::Rectangle rect = mPyramid.Rect(l, r);
                    Histogram::MakeHistogram1dSet(hSet, invSet, rect, true);
                    VectorReal64 vec = ComputeRegionDescriptor(hSet);
                    if (CheckRegionDescriptor(vec))
                        tab->Add(quid, vec);
                }
            }
            invSet.Delete();
        }
        delete hSet;
    }

    virtual void
    ComputeFeatures(Array::Array2dVec3UInt8* im, Quid quid)
    {
        for (int p=0 ; p<GetNrPixelFeatureSets() ; p++)
        {
            InvSetType invSet;
            ComputePixelFeatureSet(invSet, p, im);
            Array2dScalarReal64* a = invSet.Array(0);
            mPyramid.SetImageSize(a->CW(), a->CH());
            mLazebnikPyramid.SetImageSize(a->CW(), a->CH());
            ComputeFeaturesPixelSet(invSet, p, quid);
            invSet.Delete();
        }
    }

    // stat part

    void
    InitStat()
    {
        mFeatMin.reserve(GetPixelFeatureSetSize());
        mFeatMax.reserve(GetPixelFeatureSetSize());
        mFeatMinAll.reserve(GetPixelFeatureSetSize());
        mFeatMaxAll.reserve(GetPixelFeatureSetSize());
        for (int i=0 ; i<GetPixelFeatureSetSize() ; i++)
        {
            mFeatMin[i] = 1000;
            mFeatMax[i] = -1000;
            mFeatMinAll[i] = 1000;
            mFeatMaxAll[i] = -1000;
        }
    }

    void
    ImageStat(Array::Array2dVec3UInt8* im)
    {
        for (int p=0 ; p<GetNrPixelFeatureSets() ; p++)
        {
            InvSetType invSet;
            ComputePixelFeatureSet(invSet, p, im);

            HistSetType* hSet = NewHistSet();
            Geometry::Rectangle roiRect(15, 15, 352-15, 240-15);
            Histogram::MakeHistogram1dSet(hSet, invSet, roiRect, true);
            for (int i=0 ; i<GetPixelFeatureSetSize() ; i++)
            {
                Array2dScalarReal64* roi = MakeRoi(invSet.Array(i), roiRect);
                Real64 minVal, maxVal, meanVal, varVal;
                PixStat(roi, &minVal, &maxVal, &meanVal, &varVal);
                mFeatMin[i] = Min(mFeatMin[i], minVal);
                mFeatMax[i] = Max(mFeatMax[i], maxVal);
                Real64 perc = hSet->Hist(i).OutlierPercentage();
                if (perc > 1)
                    std::cout << "maxVal = " << maxVal << ", perc = "
                              << perc << ", i = " << i << std::endl;
                delete roi;
            }
            delete hSet;
            invSet.Delete();
        }
    }

    void
    StatMinMax2All()
    {
        for (int i=0 ; i<GetPixelFeatureSetSize() ; i++)
        {
            mFeatMinAll[i] = Min(mFeatMin[i], mFeatMinAll[i]);
            mFeatMaxAll[i] = Max(mFeatMax[i], mFeatMaxAll[i]);
            std::cout << "feat = " << i << ", min = " << mFeatMin[i] << " ("
                      << mFeatMinAll[i] << "), max = " << mFeatMax[i]
                      << " (" << mFeatMaxAll[i] << ")" << std::endl;
            mFeatMin[i] = 1000;
            mFeatMax[i] = -1000;
        }
    }

    // annotation part

    void
    ClearAnnoAccu()
    {
        for (int i=0 ; i<mHistAccu.size() ; i++)
            Vector::Clear(mHistAccu[i]);
    }

    void
    AddImageToAccu(Array::Array2dVec3UInt8* im, Geometry::Rectangle rect)
    {
        // to match rect.Inside:
        //
        rect.mLeft += 1;
        rect.mRight -= 1;
        rect.mTop += 1;
        rect.mBottom -= 1;
        //

        for (int p=0 ; p<GetNrPixelFeatureSets() ; p++)
        {
            InvSetType invSet;
            ComputePixelFeatureSet(invSet, p, im);
            HistSetType* hSet = NewHistSet();
            Histogram::MakeHistogram1dSet(hSet, invSet, rect, true);
            Vector::AddAssign(mHistAccu[p], hSet, 0, 0, GetPixelFeatureSetSize());
            mHistAccu[p]->SetSize(GetPixelFeatureSetSize());
            delete hSet;
            invSet.Delete();
        }
    }

    void
    AddAccuToAnnoTable(int bookmarkNr, String anno)
    {
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        Quid quid = mProtoLabels.Add(anno);
#else // REPOSITORY_USED
        Quid quid = mProtoLabels->Add(anno);
#endif // REPOSITORY_USED
        for (int p=0 ; p<GetNrPixelFeatureSets() ; p++)
        {
            VectorReal64 v = ComputeRegionDescriptor(mHistAccu[p]);
            FeatureTable* tab = GetProtoFeatureTable(p, anno);
            tab->Add(quid, v);
        }
    }

private:

    HistSetType*
    NewHistSet()
    {
        // default range
        HistSetType* hSet = new HistSetType(-1.0, 1.0, mHistBinCount,
                                            GetPixelFeatureSetSize());
        if (GetName() == "vissemgabor")
        {
            for (int i=0 ; i<GetPixelFeatureSetSize()-2 ; i++)
            { // adjust range for texture
                if (i%3 == 0)
                    hSet->SetRange(i, -0.0001, 1.0);
                else
                    hSet->SetRange(i, -0.0001, 0.5);
            }
        }
        return hSet;
    }

    void
    ComputePixelFeatureSet(InvSetType& fSet, int idx, Array2dVec3UInt8* im)
    {
        if (GetName() == "vissem")
            Array::InvWiccest(fSet, im, mDoRot, mDoC, GetScale(idx), 3, true);
        if (GetName() == "vissemgabor")
            Array::VisSemGabor(fSet, im, GetScale(idx), mFreq[idx], mNrOrient);
    }

    VectorReal64
    ComputeRegionDescriptor(HistSetType* hSet)
    {
        if (GetName() == "vissemgabor")
        {
            if (mCumulativeHist)
                Vector::HistogramAccumulation(hSet, false, true, 0);
            else
                Vector::HistogramNormalization(hSet, true, 0);
            return Vector::Concat(hSet);
        }
        // GetName() == "vissem"
        VectorReal64 v(GetRegionFeatureLength());
        Histogram::ComputeWeibull(v, hSet, false, false);
        return v;
    }

    bool
    CheckRegionDescriptor(VectorReal64 vec)
    {
        if (GetName() == "vissem")
        {
            bool eliminateZeros = true;
            if (!eliminateZeros)
                return true;
            bool ok = true;
            for (int i=0 ; i<vec.Size() ; i++)
            {
                if (vec[i] == 0)
                {
                    ok = false;
                    break;
                }
            }
            return ok;
        }
        return true;
    }

    VectorReal64
    ComputeRegionSimilarity(VectorReal64 regionVec, FeatureTable* protoTab)
    {
        int nrProto = protoTab->Size();
        VectorReal64 res(nrProto);
        if (GetName() == "vissem")
        {
            Vector::Similarity(res.GetData(), regionVec, protoTab->GetColumn2(),
                               0, 1, Vector::WeibullSim<Real64>);
        }
        if (GetName() == "vissemgabor")
        {
            Vector::Similarity(res.GetData(), regionVec, protoTab->GetColumn2(),
                               0, 1, Vector::HistogramIntersection<Real64>);
            Vector::DivAssign(res, Real64(GetPixelFeatureSetSize()));
        }
        return res;
    }

    // do codeword uncertainty (instead of kernel codebook)
    void
    DoUNC(FeatureTableSet* regionSim, int regionFeature)
    {
        for (int r=0 ; r<mPyramid.NrRects(regionFeature) ; r++)
        {
            Real64 totWeight = 0;
            for (int t=0 ; t<regionSim->Size() ; t++)
            {
                VectorReal64 regionVec = regionSim->GetTable(t)->Get2(r);
                totWeight += Sum(regionVec);
            }
            for (int t=0 ; t<regionSim->Size() ; t++)
            {
                VectorReal64 regionVec = regionSim->GetTable(t)->Get2(r);
                DivAssign(regionVec, totWeight);
                regionSim->GetTable(t)->Set2(r, regionVec);
            }
        }
    }

    // do the traditional codebook approach (hard assignment)
    void
    DoCodebook(FeatureTableSet* regionSim, int regionFeature)
    {
        for (int r=0 ; r<mPyramid.NrRects(regionFeature) ; r++)
        {
            Real64 maxSim = -1;
            for (int t=0 ; t<regionSim->Size() ; t++)
            {
                VectorReal64 regionVec = regionSim->GetTable(t)->Get2(r);
                maxSim = Impala::Max(maxSim, ElemMax(regionVec));
            }
            for (int t=0 ; t<regionSim->Size() ; t++)
            {
                VectorReal64 regionVec = regionSim->GetTable(t)->Get2(r);
                for (int i=0 ; i<regionVec.Size() ; i++)
                    regionVec[i] = (regionVec[i] == maxSim) ? 1.0 : 0.0;
                regionSim->GetTable(t)->Set2(r, regionVec);
            }
        }
    }

    VectorReal64
    DoSpatialPyramid(Geometry::Rectangle rect, VectorReal64 vec)
    {
        VectorReal64 res(vec.Size() * mLazebnikPyramid.TotalNrRects());
        res = 0.0;

        int levelBase = 0;
        double normVal = 1.0; // normalization of the pyramid levels
        // construct pyramid backwards, to deal with normalization
        for (int level=mLazebnikPyramid.NrLevels()-1 ; level>=0 ; level--)
        {
            // set lazebnik spatial pyramid normalization proportial
            // to the number of cells (as Lazebnik does)
            if (level != mLazebnikPyramid.NrLevels() - 1)
            {
                normVal = normVal / ((double)mLazebnikPyramid.NrRects(level+1) /
                                     mLazebnikPyramid.NrRects(level));
            }

            int nrRect = mLazebnikPyramid.NrRects(level);
            for (int r=0 ; r<nrRect ; r++)
            {
                Geometry::Rectangle rectLazeb = mLazebnikPyramid.Rect(level, r);
                // if rect intersects with rectLazebnik, add it to the
                // lazebnikpyramid-bin. This allows putting overlapping
                // regions in multiple lazebnikpyramid-bins.
                if (rectLazeb.Intersects(rect))
                {
                    int rectBase = levelBase + r * vec.Size();
                    for (int i=0 ; i<vec.Size()  ; i++)
                    {
                        // do normVal squared, cause later the Vector::Avg is
                        // taken, with a lot of zeros which will counterbalance
                        // normVal once (leaving one)
                        res[rectBase + i] = vec[i] * normVal * normVal;
                    }
                }
            }
            levelBase += nrRect * vec.Size();
        }
        return res;
    }

    void
    DoSpatialPyramid(FeatureTableSet* regionSim, int regionFeature,
                     FeatureTableSet* regionSimSP)
    {
        for (int r=0 ; r<mPyramid.NrRects(regionFeature) ; r++)
        {
            Geometry::Rectangle rect = mPyramid.Rect(regionFeature, r);
            for (int t=0 ; t<regionSim->Size() ; t++)
            {
                VectorReal64 regionVec = regionSim->GetTable(t)->Get2(r);
                VectorReal64 final = DoSpatialPyramid(rect, regionVec);
                regionSimSP->GetTable(t)->Add(0, final);
            }
        }
    }

    void
    ComputeFeaturesPixelSet(InvSetType& invSet, int p, Quid quid)
    {
        HistSetType* hSet = NewHistSet();
        for (int l=0 ; l<mPyramid.NrLevels() ; l++)
        {
            std::vector<FeatureTable*> protoTabs = GetProtoFeatureTables(p, l);
            if (!IsClusterBased())
                protoTabs = GetProtoFeatureTables(p);
            FeatureTableSet regionSim;
            FeatureTableSet regionSimSP;
            for (int t=0 ; t<protoTabs.size() ; t++)
            {
                int nrCodeWords = (IsClusterBased()) ? protoTabs[t]->Size() : 1;
                FeatureDefinition def("sim");
                regionSim.Add(new FeatureTable(def, 200, nrCodeWords));
                nrCodeWords *= mLazebnikPyramid.TotalNrRects();
                regionSimSP.Add(new FeatureTable(def, 200, nrCodeWords));
            }

            for (int r=0 ; r<mPyramid.NrRects(l) ; r++)
            {
                Geometry::Rectangle rect = mPyramid.Rect(l, r);
                Histogram::MakeHistogram1dSet(hSet, invSet, rect, true);

                VectorReal64 exVec = ComputeRegionDescriptor(hSet);

                for (int t=0 ; t<protoTabs.size() ; t++)
                {
                    VectorReal64 sVec = ComputeRegionSimilarity(exVec,
                                                                protoTabs[t]);
                    VectorReal64 regionVec;
                    if (IsClusterBased())
                    {
                        regionVec = sVec;
                    }
                    else
                    {   // average all annos (kernel codebook)
                        regionVec = VectorReal64(1);
                        regionVec[0] = Vector::ElemAvg(sVec, 0, 0, -1);
                    }
                    regionSim.GetTable(t)->Add(0, regionVec);
                }
            }

            if (mDoUNC)
                DoUNC(&regionSim, l);
            if (mDoCodebook)
                DoCodebook(&regionSim, l);
            DoSpatialPyramid(&regionSim, l, &regionSimSP);

            for (int t=0 ; t<protoTabs.size() ; t++)
            {
                FeatureTable* simTab = regionSimSP.GetTable(t);
                VectorReal64 vMax;
                Vector::Max(&vMax, simTab->GetColumn2(), simTab->Size(), 0);

                VectorReal64 vAvg;
                Vector::Avg(&vAvg, simTab->GetColumn2(), simTab->Size(), 0);
                FeatureDefinition def = protoTabs[t]->GetFeatureDefinition();
                FeatureTable* tab = GetFeatureTable(def, l);
                tab->Add(quid, Vector::Concat(vMax, vAvg));
            }
            regionSim.Delete();
            regionSimSP.Delete();
        }
        delete hSet;
    }

    bool                mDoRot;
    bool                mDoC;
    int                 mHistBinCount;
    bool                mCumulativeHist;
    int                 mNrOrient;
    std::vector<double> mFreq;

    std::vector<Real64> mFeatMin;
    std::vector<Real64> mFeatMax;
    std::vector<Real64> mFeatMinAll;
    std::vector<Real64> mFeatMaxAll;

    std::vector<HistSetType*> mHistAccu;

    String mKeyframesName;
    bool   mDoCodebook;
    bool   mDoUNC;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(VisSem, Impala.Core.Feature);

} // namespace Feature
} // namespace Core
} // namespace Impala

#endif

---