|| || || || || ||

---

Color64.h

Go to the documentation of this file.

#ifndef Impala_Core_Feature_Color64_h
#define Impala_Core_Feature_Color64_h

// extract color 64 bin features, provided by xirong.
// color64: 44-d color correlgoram, 14-d color texture moment, and 6-d RGB color moment

// Color correlgoram:
// J. Huang, S. Kumar, M. Mitra, W. Zhu, and R. Zabih, “Image indexing using color correlograms,” in Proc. CVPR, 1997, pp. 762–768.
// Color texture:
// H. Yu, M. Li, H. Zhang, and J. Feng, “Color texture moment for contentbased image retrieval,” in Proc. ICIP, 2002, pp. 929–932.
//
// We have used this simple color64 feature a few times in the past, with a moderate success for large-scale web image retrieval, e.g.,
// http://staff.science.uva.nl/~xirong/pub/li-tag-double.pdf.
// For each image, we extract 44-d color correlgoram [44], 14-d color texture moment [45], and 6-d RGB color moment. 
// We separately normalize the three features into unit length and concatenate them into a single vector. 
// We use the Euclidean distance as a dissimilarity measurement.

#include "Basis/String.h"
#include "Core/Database/RawDataSet.h"
#include "Core/Feature/FeatureDefinition.h"
#include "Core/Vector/VectorTem.h"
#include "Core/Feature/FeatureTable.h"
#include "Core/Feature/Grid.h"

#ifndef max
#define max(a,b)            (((a) > (b)) ? (a) : (b))
#endif

#ifndef min
#define min(a,b)            (((a) < (b)) ? (a) : (b))
#endif

namespace Impala
{
namespace Core
{
namespace Feature
{


class Color64
{
public:

    Color64(String featureName="color64")
    {
        mFeatureName = featureName;

        // set the feature mode based on mFeatureName
        // as: 0, 1, 2, 3, 4, 5, 6
        //
        // mode=0: color64
        // mode=1: 44-d color correlogram
        // mode=2: 14-d color texture moment
        // mode=3: 6-d RGB color moment
        // mode=4: color64 grid 2x2
        // mode=5: color64 grid 3x3
        // mode=6: color64 grid 4x4
        // mode=7: color64 grid 5x5
        // mode=8: color64 grid 6x6
        //
        SetFeatureMode();

        mFeature = new float [64];

        //mGrid = new Core::Feature::Grid(2,2);

    }

    ~Color64()
    {
        delete mFeature;

        //delete mGrid;
    }

    String
    GetName() const
    {
        return mFeatureName;
    }

    void
    SetGridDimension(int nRow, int nCol)
    {
        mRow = nRow;
        mCol = nCol;
    }

    int
    Extract(Core::Array::Array2dVec3UInt8* im, Core::Vector::VectorReal64& histogram, Core::VideoSet::VideoSet* vs = NULL)
    {
        switch (mode)
        {
         case 0: // "color64", 64dim = 44+14+6

             ExtractCorrelogramTextureMoment(im->CPB(), im->CW(), im->CH(), mFeature);

             for (int d=0; d<64; d++)
             {
                 histogram[d] = mFeature[d];
             }
             //VerifyFeatureSum(mFeature, 64, 3);
             VerifyFeatureRange(mFeature, 64);

         case 1: // 44-d color correlogram

             ExtractCorrelogramTextureMoment(im->CPB(), im->CW(), im->CH(), mFeature, 1);

             for (int d=0; d<44; d++)
             {
                 histogram[d] = mFeature[d];
             }
             //VerifyFeatureSum(mFeature, 44, 1);
             VerifyFeatureRange(mFeature, 44);

             break;

         case 2: // 14-d color texture moment

             ExtractCorrelogramTextureMoment(im->CPB(), im->CW(), im->CH(), mFeature, 2);
             for (int d=0; d<14; d++)
             {
                 histogram[d] = mFeature[d+44];
             }
             //VerifyFeatureSum(mFeature+44, 14, 1);
             VerifyFeatureRange(mFeature+44,14);
             break;

         case 3: // 6-d RGB color moment
            
             ExtractCorrelogramTextureMoment(im->CPB(), im->CW(), im->CH(), mFeature, 3);
             for (int d=0; d<6; d++)
             {
                 histogram[d] = mFeature[d+44+14];
             }
             //VerifyFeatureSum(mFeature+44+14, 6, 1);
             VerifyFeatureRange(mFeature+44+14, 6);
             break;

         case 4: // 256-d color64 grid 2x2

             ComputeGrid(im, histogram, 2, 2, vs);             

             break;

         case 5: // 576-d color64 grid 3x3
             
             ComputeGrid(im, histogram, 3, 3, vs);

             break;

         case 6: // 1024-d color64 grid 4x4
             
             ComputeGrid(im, histogram, 4, 4, vs);

             break;

         case 7: // 1600-d color64 grid 5x5
             
             ComputeGrid(im, histogram, 5, 5, vs);

             break;

         case 8: // 2304-d color64 grid 6x6
             
             ComputeGrid(im, histogram, 6, 6, vs);

             break;

         default: 

             std::cout << "Feature Extraction is skipped." << std::endl;
             break;
        }

        return 0;
    }

    void
    ComputeGrid(Core::Array::Array2dVec3UInt8* im, Core::Vector::VectorReal64& histogram, int nRow, int nCol, Core::VideoSet::VideoSet* vs = NULL)
    {
        SetGridDimension(nRow, nCol);

        Core::Feature::Grid* pGrid = new Core::Feature::Grid(mRow, mCol);

        BYTE** grid_buffers= new BYTE* [mRow*mCol];
        int nGridWidth = (im->CW()/mCol);
        int nGridHeight = (im->CH()/mRow);
        for (int k=0; k< mRow*mCol; k++)
        {
         grid_buffers[k] = new BYTE[nGridWidth*nGridHeight*3];
        }

        pGrid->RetrievalGridBuffers( (BYTE*)im->CPB(), grid_buffers, im->CW(), im->CH(), vs);


        for (int k=0; k< mRow*mCol; k++)
        {
         ExtractCorrelogramTextureMoment(grid_buffers[k], nGridWidth, nGridHeight, mFeature);
         for (int d=0; d<64; d++)
         {
             histogram[k*64+d] = mFeature[d];
         }
         //VerifyFeatureSum(mFeature, 64, 3);
         VerifyFeatureRange(mFeature, 64);
         
        }

        for (int k=0; k< mRow*mCol; k++)
        {
         delete [](grid_buffers[k]);
        }
        delete []grid_buffers;
        delete pGrid;

    }

    void
    SetFeatureMode()
    {
        if (mFeatureName == "color64")
        {
            mode=0;
            mBinCount = 64;
        }
        else if (mFeatureName == "correlogram44")
        {
            mode=1;
            mBinCount = 44;
        }
        else if (mFeatureName == "texturemoment14")
        {
            mode=2;
            mBinCount = 14;
        }
        else if (mFeatureName == "rgbmoment6")
        {
            mode=3;
            mBinCount = 6;
        }
        else if (mFeatureName == "color64grid2x2")
        {
            mode=4;
            mBinCount = 64*2*2;
        }
        else if (mFeatureName == "color64grid3x3")
        {
            mode=5;
            mBinCount = 64*3*3;
        }
        else if (mFeatureName == "color64grid4x4")
        {
            mode=6;
            mBinCount = 64*4*4;
        }
        else if (mFeatureName == "color64grid5x5")
        {
            mode=7;
            mBinCount = 64*5*5;
        }
        else if (mFeatureName == "color64grid6x6")
        {
            mode=8;
            mBinCount = 64*6*6;
        }
        else if (mFeatureName == "")
        {
            mode=-1;
            mBinCount = 0;
            std::cout << "FeatureName is empty." << std::endl;
        }
        else // "color64"
        {
            mode=-1;
            mBinCount = 0;
            std::cout << "FeatureName is un-defined: " << mFeatureName << std::endl;
        }
    }

    // Main Function for the "color64" feature extraction
    //
    // mode: see SetFeatureMode()
    int ExtractCorrelogramTextureMoment(BYTE *pImage, int nWidth, int nHeight,  void *pFeature, int mode=0)
    {
            if (pFeature == 0)
            {
                    return -1;
            }
        memset(pFeature, 0, sizeof(float) * 64);

        if (nWidth < 3 || nHeight < 3)
        {
            return -1;
        }

        float* pfCorrelogram44 = (float*)pFeature;
        float* pfTextureMoment = pfCorrelogram44 + 44;
        float* pfColorMoment = pfTextureMoment + 14;

        float* pfImage = new float[nWidth * nHeight];
        BYTE* pbImage = new BYTE[nWidth * nHeight];

        if (pbImage != NULL && pfImage != NULL)
        {
            for (int y = 0; y < nHeight; y ++)
            {
                float* pf = pfImage + y * nWidth;
                BYTE* pb = pbImage + y * nWidth;

                for (int x = 0; x < nWidth; x ++)
                {
                    // Note: the order of the buffer is RGBRGB...
                    BYTE r = pImage[(y*nWidth+x)*3+0];
                    BYTE g = pImage[(y*nWidth+x)*3+1];
                    BYTE b = pImage[(y*nWidth+x)*3+2];

                    pf[x] =(float) r + g + b;
                    pb[x] = QuantizeColorHSV44(r, g, b);

                    pfColorMoment[0] += r;
                    pfColorMoment[1] += g;
                    pfColorMoment[2] += b;

                    pfColorMoment[3] += r * r;
                    pfColorMoment[4] += g * g;
                    pfColorMoment[5] += b * b;
                }
            }

            float fArea =(float) nWidth * nHeight;

            switch (mode)
            {
             case 1: // 44-d color correlogram

                ExtractCorrelogram50(pbImage, nWidth, nHeight, pfCorrelogram44);
                break;

             case 2: // 14-d color texture moment

                ExtractTextureMoment(pfImage, nWidth, nHeight, pfTextureMoment);
                break;

             case 3: // 6-d RGB color moment
                
                for (int i = 0; i < 3; i ++)
                {
                    pfColorMoment[i] /= fArea;
                    pfColorMoment[i + 3] = pfColorMoment[i + 3] / fArea - pfColorMoment[i] * pfColorMoment[i];
                    pfColorMoment[i + 3] = (float)sqrt(fabs(pfColorMoment[i + 3]));
                }
                NormalizeVector(pfColorMoment, 6);
                break;

             default: // color64 = 44+14+6

                ExtractCorrelogram50(pbImage, nWidth, nHeight, pfCorrelogram44);
                ExtractTextureMoment(pfImage, nWidth, nHeight, pfTextureMoment);

                for (int i = 0; i < 3; i ++)
                {
                    pfColorMoment[i] /= fArea;
                    pfColorMoment[i + 3] = pfColorMoment[i + 3] / fArea - pfColorMoment[i] * pfColorMoment[i];
                    pfColorMoment[i + 3] = (float)sqrt(fabs(pfColorMoment[i + 3]));
                }
                NormalizeVector(pfColorMoment, 6);

                break;
            }

        }
        if (pbImage != NULL)
        {
            delete []pbImage;
        }
        if (pfImage != NULL)
        {
            delete []pfImage;
        }

            return 0;
    }

    void ExtractCorrelogram50(BYTE* pbImage, int nWidth, int nHeight, float* pfFeature)
    {
        float afHistogram[50], afCorrelogram[50];
        memset(afHistogram, 0, sizeof(afHistogram));
        memset(afCorrelogram, 0, sizeof(afCorrelogram));

        for (int y = 1; y < nHeight - 1; y ++)
        {
            BYTE* pbCurr = pbImage + y * nWidth;
            BYTE* pbPrev = pbCurr - nWidth;
            BYTE* pbNext = pbCurr + nWidth;

            for (int x = 1; x < nWidth - 1; x ++)
            {
                // Note: nIndex = [0,43]
                int nIndex = pbCurr[x];
                int nCount = (pbPrev[x - 1] == nIndex) + (pbPrev[x] == nIndex) + (pbPrev[x + 1] == nIndex)
                           + (pbCurr[x - 1] == nIndex) + (pbCurr[x + 1] == nIndex)
                           + (pbNext[x - 1] == nIndex) + (pbNext[x] == nIndex) + (pbNext[x + 1] == nIndex);
                afHistogram[nIndex] ++;
                afCorrelogram[nIndex] += nCount;
            }
        }

        for (int i = 0; i < 50; i ++)
        {
            if (afHistogram[i] > 0)
            {
                pfFeature[i] = afCorrelogram[i] / afHistogram[i];
            }
        }
        NormalizeVector(pfFeature, 50);
    }

    void ExtractTextureMoment(float* pfImage, int nWidth, int nHeight, float* pfFeature)
    {
        const float fW2 = (float)sqrt(2.0);
        for (int y = 1; y < nHeight - 1; y ++)
        {
            float* pfCurr = pfImage + y * nWidth;
            float* pfPrev = pfCurr - nWidth;
            float* pfNext = pfCurr + nWidth;

            for (int x = 1; x < nWidth - 1; x ++)
            {
                float af[8];

                af[0] = pfPrev[x - 1] + pfPrev[x] + pfPrev[x + 1]
                        + pfCurr[x - 1] + pfCurr[x + 1]
                        + pfNext[x - 1] + pfNext[x] + pfNext[x + 1];

                af[1] = - pfPrev[x - 1] + pfPrev[x] - pfPrev[x + 1]
                        + pfCurr[x - 1] + pfCurr[x + 1]
                        - pfNext[x - 1] + pfNext[x] - pfNext[x + 1];

                af[2] = - pfPrev[x - 1] + pfPrev[x + 1] - pfNext[x - 1] + pfNext[x + 1]
                        + fW2 * (- pfCurr[x - 1] + pfCurr[x + 1]);

                af[3] = - pfPrev[x - 1] - pfPrev[x + 1] + pfNext[x - 1] + pfNext[x + 1]
                        + fW2 * (- pfPrev[x] + pfNext[x]);

                af[4] = fW2 * (- pfPrev[x] + pfCurr[x - 1] + pfCurr[x + 1] - pfNext[x]);

                af[5] = fW2 * (pfPrev[x - 1] - pfPrev[x + 1] - pfNext[x - 1] + pfNext[x + 1]);

                af[6] = pfPrev[x - 1] - pfPrev[x + 1] + pfNext[x - 1] - pfNext[x + 1]
                        + fW2 * (- pfCurr[x - 1] + pfCurr[x + 1]);

                af[7] = - pfPrev[x - 1] - pfPrev[x + 1] + pfNext[x - 1] + pfNext[x + 1]
                        + fW2 * (pfPrev[x] - pfNext[x]);

                /*float fValueNorm = af[0];
                if (fValueNorm < 1e-4f)
                {
                    fValueNorm = 1e-4f;
                }
                fValueNorm = 1.0f / fValueNorm;
                */
                for (int i = 1; i < 8; i ++)
                {
                    //af[i] *= fValueNorm;
                    pfFeature[i - 1] += fabs(af[i]);
                    pfFeature[i + 6] += af[i] * af[i];
                }
            }
        }

        float fSizeNorm = 1.0f / ((nWidth - 2) * (nHeight - 2));
        for (int i = 0; i < 7; i ++)
        {
            pfFeature[i] *= fSizeNorm;
            pfFeature[i + 7] = (float)sqrt(fabs(pfFeature[i + 7] * fSizeNorm - pfFeature[i] * pfFeature[i]));
        }
        NormalizeVector(pfFeature, 14);
    }

    void NormalizeVector(float* pf, int nLen)
    {
        if (pf != NULL && nLen > 0)
        {
            float f = 0;
                    for (int i = 0; i < nLen; i ++)
            {
                f += pf[i] * pf[i];
            }
            f = sqrt(f);
            if (f < 1e-4f)
            {
                f = 1e-4f;
            }
            for (int i = 0; i < nLen; i ++)
            {
                pf[i] /= (float)f;
            }
        }
    }

    int QuantizeColorHSV44(BYTE bRed, BYTE bGreen, BYTE bBlue)
    {
        static BYTE RGB2HSV[16][16][16];
        static bool bInit = false;
        if (!bInit)
        {
            for (int r = 0; r < 16; r ++)
            {
                for (int g = 0; g < 16; g ++)
                {
                    for (int b = 0; b < 16; b ++)
                    {
                        int nMax = max(r, max(g, b));
                        int nMin = min(r, min(g, b));

                        float h, s, v;
                        if (nMax == nMin)
                        {
                            h = 0;
                        }
                        else if (r == nMin)
                        {
                            h = 3.0f + (float)(b - g) / (nMax - nMin);
                        }
                        else if (g == nMin)
                        {
                            h = 5.0f + (float)(r - b) / (nMax - nMin);
                        }
                        else //if (b == nMin)
                        {
                            h = 1.0f + (float)(g - r) / (nMax - nMin);
                        }
                        if (nMax > 0)
                        {
                            s = (float)(nMax - nMin) / nMax;
                        }
                        else
                        {
                            s = 0;
                        }
                        v = (float)nMax / 15.0f;

                        int H = (int)(h * 60);
                        int S = (int)(s * 10);
                        int V = (int)(v * 10);
                        int nIndex;

                        if (V < 2)     // black area
                        {
                            nIndex = 0;
                        }
                        else if (S < 2) // Gray area
                        {
                            if (V < 8)
                            {
                                nIndex = V - 1; // 1~6, gray area
                            }
                            else
                            {
                                nIndex = 7 + (V >= 9);   // 7~8, white area
                            }
                        }
                        else // Color area
                        {
                            int nColor;
                            if (H < 22)
                            {
                                nColor = 0; // red
                            }
                            else if (H < 45)
                            {
                                nColor = 1; // orance
                            }
                            else if (H < 70)
                            {
                                nColor = 2; // yellow
                            }
                            else if (H < 155)
                            {
                                nColor = 3; // green
                            }
                            else if (H < 186)
                            {
                                nColor = 4; // cyan
                            }
                            else if (H < 278)
                            {
                                nColor = 5; // blue
                            }
                            else if (H < 330)
                            {
                                nColor = 6; // purple
                            }
                            else
                            {
                                nColor = 0; // red
                            }
                            
                            if (V >= 7) // SV plane is segmented into 5 regions
                            {
                                nIndex = 2 + (S >= 5) + (S >= 8);
                            }
                            else
                            {
                                nIndex = (S >= 7);
                            }
                            nIndex += 9 + 5 * nColor; // 9~43
                        }
                        RGB2HSV[r][g][b] = nIndex;
                    }
                }
            }
            bInit = true;
        }
        return RGB2HSV[bRed / 16][bGreen / 16][bBlue / 16];
    }

    bool VerifyFeatureSum(float* pFeature, int nDim, float ref=1.0)
    {
        float sum = 0;
        for (int i=0; i<nDim; i++)
        {
            sum += pFeature[i];
        }

        float diff = sum-ref;
        if ( fabs(diff) < 1e-10 )
            return true;
        else
        {
            std::cout << "ERROR: Feature Sum Verifiction failed." << std::endl;
            return false;
        }
        
    }

    bool VerifyFeatureRange(float* pFeature, int nDim, float min=0, float max=1.0)
    {
        float sum = 0;
        for (int i=0; i<nDim; i++)
        {
            float value = pFeature[i];
            if (value< min || value > max )
            {
                std::cout << "ERROR: Feature Range Verifiction failed: " << value << std::endl;
                return false;
            }
        }

        return true;
    }

private:

    Database::RawDataSet* mDataSet;
    String                mFeatureName;
    int                   mode;
    float*                mFeature;

    int                   mBinCount;
    //Core::Feature::Grid*  mGrid;
    int                   mRow;
    int                   mCol;

};//class

} // namespace Feature
} // namespace Core
} // namespace Impala

#endif

---