|| || || || || ||

---

ConfusionMatrix.h

Go to the documentation of this file.

#ifndef Impala_Core_Table_ConfusionMatrix_h
#define Impala_Core_Table_ConfusionMatrix_h

#include "Core/Table/Bayes.h"

namespace Impala
{
namespace Core
{
namespace Table
{


class ConfusionMatrix : public Bayes
{
public:

    ConfusionMatrix(String setName, String conceptSet, String modelName, String featureName,
                                       String conceptPath, String quid, int nStart,int nEnd): 
         Bayes("", setName, conceptSet, conceptSet, quid, nStart, nEnd)
    {

        mSetName = setName;
        mModelName = modelName;
        mFeatureName = featureName;

        mConceptPath = conceptPath;

        // load annotation table set
        if (mIsImageSet)
        {
            mAnnoImgSet = Core::ImageSet::MakeImageSet(setName);
            mAnnoTabSet = Core::Table::AnnotationTableSet::MakeFromFile(mAnnoImgSet, mConceptSet, true,
                                                                           StringToQuidClass(mQuid));
        }
        else
        {
            mAnnoVidSet = Core::VideoSet::MakeVideoSet(setName);
            mAnnoTabSet = Core::Table::AnnotationTableSet::MakeFromFile(mAnnoVidSet, mConceptSet, true,
                                                                           StringToQuidClass(mQuid));            
        }
            
        if (mAnnoTabSet == NULL)
        {
            printf("ERROR: The Concept Set does NOT exist.\n");
            return;
        }

        mConceptNum = mAnnoTabSet->Size();
        if (mConceptNum<0)
        {
            printf("ERROR: The number of conceptis Zero.\n");
            return;
        }

        tabConcept = new AnnotationTable* [mConceptNum];
    
        // Note: mVidSet/mImgSet has been initialized in Bayes class   
        if (mIsImageSet)
        {
            //mGenreSimSet = Core::Table::SimilarityTableSet::MakeFromFile(genrePath, mImgSet->GetDatabase());
            mConceptSimSet = Core::Table::SimilarityTableSet::MakeFromFile(mConceptPath, mImgSet->GetDatabase());
        }
        else
        {
            mKeyframes = new Core::VideoSet::Keyframes(mVidSet, "keyframes"); 

            //mGenreSimSet = Core::Table::SimilarityTableSet::MakeFromFile(genrePath, mVidSet->GetDatabase());
            mConceptSimSet = Core::Table::SimilarityTableSet::MakeFromFile(mConceptPath, mVidSet->GetDatabase());
        }
            

        conceptSimTable = new SimilarityTableSet::SimTableType* [mConceptNum];

        mQuidTable = mConceptSimSet->GetQuidTable();
        mTableSize = mQuidTable->Size();

        /*Core::Table::QuidTable* conceptQuidTable = mConceptSimSet->GetQuidTable();
        int conceptTableSize = conceptQuidTable->Size();

        //_ASSERT(mTableSize == conceptTableSize);
        if (mTableSize != conceptTableSize)
        {
            printf("ERROR: Quid Table is NOT the same for concept and genre.\n");
            return;
        }*/


        if ((mEnd == -1) || (mEnd > mQuidTable->Size()))
            mEnd = mQuidTable->Size();

        //mWriteTable = bWriteTable;

        //genreSimTable = NULL;
        //conceptSimTable = NULL;

        AllocateMatrix(mConfusionMatrix, mConceptNum, mConceptNum);
        ResetMatrix(mConfusionMatrix, mConceptNum, mConceptNum);
    }

    virtual ~ConfusionMatrix()
    {
        //if (NULL != genreSimTable)
        //    delete []genreSimTable;
        if (NULL != conceptSimTable)
            delete []conceptSimTable;

        //delete mGenreSimSet;
        delete mConceptSimSet;

        delete mKeyframes;

        ReleaseMatrix(mConfusionMatrix, mConceptNum);
    }

    void
    DoConfusionMatrix()
    {
        std::cout << std::endl;
        std::cout << "DumpConfusionMatrix() - my own tool" << std::endl;
        std::cout << std::endl;

        // load annotation and similarity tables
        for (int c=0 ; c< mConceptNum ; c++)
        {
            tabConcept[c] = mAnnoTabSet->GetTable(c);

            std::cout << "table " << c << ", name = " << mConceptSimSet->GetName(c) << std::endl;
            conceptSimTable[c] = mConceptSimSet->GetSimTable(c);
        }

        // get the mean of each concept list
        Real64* pMean = new Real64[mConceptNum];
        memset(pMean,0,sizeof(Real64)*mConceptNum);
        for (int c=0 ; c<mConceptNum ; c++)
        {
            Real64 fSum = 0;
            for (int i=0 ; i<mTableSize ; i++)
            {
                Quid quid = mQuidTable->Get1(i);                    
                Real64 sim = conceptSimTable[c]->Get1(i);
                fSum += sim;
            }
            pMean[c] = fSum/mTableSize;
        }

        // get the variance of each concept list
        Real64* pStd = new Real64[mConceptNum];
        memset(pStd,0,sizeof(Real64)*mConceptNum);
        for (int c=0 ; c<mConceptNum ; c++)
        {
            Real64 fSum = 0;
            for (int i=0 ; i<mTableSize ; i++)
            {
                Quid quid = mQuidTable->Get1(i);                    
                Real64 sim = conceptSimTable[c]->Get1(i);
                fSum += pow(sim - pMean[c],2);
            }
            pStd[c] = sqrt(fSum/(mTableSize-1));
        }

        // dump the old data distribution
        std::cout << std::endl;
        std::cout << "Before gaussian normalization:" << std::endl;
        for (int c=0; c<mConceptNum; c++ )
        {
            printf("  (mean,std) = (%.10f, %.10f)\n", pMean[c], pStd[c]);
        }

        // do the gaussian normalization of the similarity table
        for (int c=0 ; c<mConceptNum ; c++)
        {
            Real64 fSum = 0;
            for (int i=0 ; i<mTableSize ; i++)
            {
                Quid quid = mQuidTable->Get1(i);                    
                Real64 old_score = conceptSimTable[c]->Get1(i);
                Real64 new_score = (old_score - pMean[c])/pStd[c];

                // set the normalized value as the new score
                conceptSimTable[c]->Set1(i,new_score);
            }   
        }

        // check the updated data distribution
        memset(pMean,0,sizeof(Real64)*mConceptNum);
        memset(pStd,0,sizeof(Real64)*mConceptNum);
        for (int c=0 ; c<mConceptNum ; c++)
        {
            Real64 fSum = 0;
            for (int i=0 ; i<mTableSize ; i++)
            {
                Quid quid = mQuidTable->Get1(i);                    
                Real64 sim = conceptSimTable[c]->Get1(i);
                fSum += sim;
            }
            pMean[c] = fSum/mTableSize;
        }

        // get the variance of each concept list
        for (int c=0 ; c<mConceptNum ; c++)
        {
            Real64 fSum = 0;
            for (int i=0 ; i<mTableSize ; i++)
            {
                Quid quid = mQuidTable->Get1(i);                    
                Real64 sim = conceptSimTable[c]->Get1(i);
                fSum += pow(sim - pMean[c],2);
            }
            pStd[c] = sqrt(fSum/(mTableSize-1));
        }

        // dump the normalized data distribution
        std::cout << "After gaussian normalization:" << std::endl;
        for (int c=0; c<mConceptNum; c++ )
        {
            printf("  (mean,std) = (%.10f, %.10f)\n", pMean[c], pStd[c]);
        }

        delete []pMean;
        delete []pStd;

        // loop for all the key-frames in the table
        int nDupLabels = 0;
        for (int i=0 ; i<mTableSize ; i++)
        {
            Quid quid = mQuidTable->Get1(i);

            // for keyframe i: find the highest score among all concepts
            Real64 fMaxValue = -1;
            int nDetect = -1;

            for (int c=0 ; c<mConceptNum ; c++)
            {
                //String concept = mConceptSimSet->GetName(c);
                //std::cout << concept << ", ";
                Real64 sim = conceptSimTable[c]->Get1(i);
                if (sim>fMaxValue)
                {
                    fMaxValue = sim;
                    nDetect = c;
                }
            }

            // for keyframe i: find the ground truth that concept keyframe i belongs to
            int nTruth = -1;
            int count = 0;
            for (int c=0 ; c<mConceptNum ; c++)
            {
                //String concept = mConceptSimSet->GetName(c);
                //std::cout << concept << ", ";
                if ( tabConcept[c]->IsPositive(i) )
                {
                    nTruth = c;
                    count ++;
                }
            }
            // be sure that only one unique label

            if (count == 0) // nTruth == -1
            {
                // no ground truth for current key-frame
                continue;
            }
            else if (count > 1)
            {
                //std::cout << "Warning: there are " << count << " labels for shot " << i << std::endl;
                nDupLabels++;
            }

            int nCorrect = 0;
            int nError = 0;

            // mConfusionMatrix[i=nTruth][j=nDetect]:
            // 1) A row (fixed i) stores all the predicted results for concept i
            // 2) A column (fixed j) stores all the ground truth of concept j
            // 3) the sum of a column j should be the total number of positive samples for concept j (before normalization)
            if (nDetect == nTruth)
            {
                mConfusionMatrix[nTruth][nTruth] += 1.0;
                nCorrect ++;
                                    
            }
            else
            {
                // re-corrected order for row/column
                mConfusionMatrix[nTruth][nDetect] += 1.0;
                nError ++;
            }
        }

        std::cout << "Duplicated Labels: " << nDupLabels << std::endl;

        DumpMatrix("Confusion Matrix", mConfusionMatrix, mConceptNum, mConceptNum, false);

        String outfileInt = "./ConfusionMatrix/" + mSetName + "_" + mConceptSet + "_" + mFeatureName + "_int.txt";
        SaveConfigurationMatrix(outfileInt.c_str(), false);

        // normalizations: sum of a column y (fixed x) is 1
        // mConfusionMatrix[x=nTruth][y=nDetect]:
        for (int x=0 ; x<mConceptNum; x++)
        {
            Real64 sum = 0;
            for (int y=0 ; y<mConceptNum ; y++)
                sum += mConfusionMatrix[x][y];

            if (sum < 0) continue;

            for (int y=0 ; y<mConceptNum ; y++)
                mConfusionMatrix[x][y] /= sum;
        }

        printf("The sum of each column (actual class)\n");
        for (int x=0 ; x<mConceptNum; x++)
        {
            Real64 sum = 0;
            for (int y=0 ; y<mConceptNum ; y++)
                sum += mConfusionMatrix[x][y];

            printf("%.5f ", sum);
        }
        printf("\n");

        DumpMatrix("Confusion Matrix", mConfusionMatrix, mConceptNum, mConceptNum);

        String outfile = "./ConfusionMatrix/" + mSetName + "_" + mConceptSet + "_" + mFeatureName + "_3f.txt";
        SaveConfigurationMatrix(outfile.c_str());

        int x=0;
    }

    void
    // dump simiarity table and annotation table into two matrix for CAMP plot (matlab tool by Jasper)
    DumpConfusionMatrixCAMP(bool bSorted = false)
    {
        std::cout << "DumpConfusionMatrixCAMP() - for CAMP tool by Jasper" << std::endl;

        // load annotation and similarity tables
        for (int c=0 ; c< mConceptNum ; c++)
        {
            tabConcept[c] = mAnnoTabSet->GetTable(c);

            std::cout << "sim table " << c << ", name = " << mConceptSimSet->GetName(c) << std::endl;
            conceptSimTable[c] = mConceptSimSet->GetSimTable(c);
        }

        // we assume skipping is concept-independent.
        Impala::Core::Table::AnnotationTable* truth = tabConcept[0];
        int nTotalShots = truth->GetNrPositive() + truth->GetNrNegative();


        // dump similarity table for all the concepts, into a matrix
        String fileSimTable = "./ConfusionMatrix/" + mSetName + "_" + mConceptSet + "_" + mFeatureName + "_sim_table.txt";
        FILE* fpSim = fopen(fileSimTable.c_str(), "wt");
        if (NULL == fpSim)
        {
            std::cout << "File path deos NOT exist: " << fileSimTable << std::endl;
            return;
        } 
        fprintf(fpSim,"%d\t%d\n",nTotalShots,mConceptNum); 

        // dump annotation table for all the concepts, into a matrix
        String fileAnnoTable = "./ConfusionMatrix/" + mSetName + "_" + mConceptSet + "_" + mFeatureName + "_anno_table.txt";
        FILE* fpAnno = fopen(fileAnnoTable.c_str(), "wt");
        if (NULL == fpAnno)
        {
            std::cout << "File path deos NOT exist: " << fileAnnoTable << std::endl;
            return;
        } 
        fprintf(fpAnno,"%d\t%d\n",nTotalShots,mConceptNum); 

        std::cout << "start to merge sim and anno tables into two single files ..." << std::endl;
        for (int i=0 ; i<mTableSize ; i++)
        {
            // only for un-sorted table
            Quid quid = mQuidTable->Get1(i);

            bool bIsSkip = false;
            for (int c=0 ; c<mConceptNum ; c++)
            {
                Impala::Core::Table::SimilarityTableSet::RankTableType* rank = mConceptSimSet->GetRankTable(c);
                Impala::Core::Table::AnnotationTable* groundTruth = tabConcept[c];

                Quid q;
                int curPos = -1;
                if (bSorted)
                {
                    // only for similarity-based sorted table
                    q = rank->Get1(i);
                    curPos = Column::Find(mQuidTable->GetColumn1(), q);
                    if (curPos == mTableSize)
                    {
                        // if returned index is equal to table size, it mean that quid is not found.
                        std::cout << "Can NOT find the Quid: " << QuidObj(q) << std::endl;
                        continue;
                    }
                }
                else
                {
                    //for un-sorted table
                    q = quid;
                    curPos = i;
                }

                // 1) dump similarity table for the concept c, into a matrix
                if (!groundTruth->IsSkip(q))
                {
                    // skipped shots are all ignored.
                    // only saving positive or negative samples
                    Real64 prob = conceptSimTable[c]->Get1(curPos);
                    fprintf(fpSim,"%f\t", prob);
                }
                else
                {
                    // we assume skipping is concept-independent.
                    bIsSkip = true;
                    
                    break;
                }

                // 2) dump annotation table for the concept c, into a matrix
                if ( groundTruth->IsPositive(q) )
                    fprintf(fpAnno,"%d ", 1);
                else // ( groundTruth->IsNegative(q) )
                    fprintf(fpAnno,"%d ", 0);
                //else //( groundTruth->IsSkip(q) )
                    //fprintf(fpAnno,"%d ", -1);
                
            }
             
            if ( !bIsSkip)
            {
                // no blank line for skipped shots.
                fprintf(fpSim,"\n");
                fprintf(fpAnno,"\n");
            }

            if (i%20000 == 0 && i>0)
            {
                std::cout << " both saved: " << i << " lines." << std::endl;
            }
            
        }
        fclose(fpSim);
        fclose(fpAnno);

        std::cout << "Done. [*_sim_table.txt;*_anno_table.txt]" << std::endl;

        int x=0;
    }

    void
    DumpConceptSimilarityTableSet()
    {
        std::cout << "SimilarityTableSet " << mConceptSimSet->GetDescription() << std::endl << std::endl;

        //for (int k=0 ; k<mConceptSimSet->NrTables() ; k++)
        //_ASSERT(mConceptNum == mConceptSimSet->NrTables());
        for (int k=0 ; k<mConceptNum ; k++)
        {
            //SimilarityTableSet::SimTableType* conceptSimTable = mConceptSimSet->GetSimTable(k);
            conceptSimTable[k] = mConceptSimSet->GetSimTable(k);

            std::cout << "table " << k << ", name = " << mConceptSimSet->GetName(k)
                      << std::endl;
            if (conceptSimTable[k]->Size() != mQuidTable->Size())
            {
                //ILOG_ERROR("DumpRanking: simtable size doesn't match");
                printf("DumpRanking: simtable size doesn't match\n");
                continue;
            }
            String concept = mConceptSimSet->GetName(k);
            for (int i=mStart ; i<mEnd ; i++)
            {
                Quid quid = mQuidTable->Get1(i);
                //std::cout << i << ", ";
                std::cout << concept << ", ";

                if (!mIsImageSet)
                {
                    int keyId = mKeyframes->GetFrameId(quid);
                    if (keyId != -1)
                        std::cout << FileNameBase(mKeyframes->GetName(keyId)) << ", ";
                    else
                        std::cout << mVidSet->QuidToString(quid, true) << ", ";
                }
                std::cout << conceptSimTable[k]->Get1(i) << std::endl;
            }
            std::cout << std::endl;
        }
    }

    void
    SaveConfigurationMatrix(const char* file, bool bDumpFloat = true)
    {
        if (bDumpFloat)
            DumpMatrix("mConfusionMatrix", mConfusionMatrix, mConceptNum, mConceptNum, file);
        else
            DumpMatrix("mConfusionMatrix", mConfusionMatrix, mConceptNum, mConceptNum, file, false);
    }

protected:

    //SimilarityTableSet* mGenreSimSet;
    SimilarityTableSet* mConceptSimSet;

    //SimilarityTableSet::SimTableType** genreSimTable;
    SimilarityTableSet::SimTableType** conceptSimTable;

    Core::Table::QuidTable* mQuidTable;  // only for similiarty table set

    String mConceptPath;

    Core::VideoSet::Keyframes* mKeyframes;

    AnnotationTableSet* mAnnoTabSet;

    //bool mWriteTable;

    double** mConfusionMatrix;      // conditional probability P(c_{k}|g_{j})

    String mModelName;
    String mFeatureName;

};

} // namespace Table
} // namespace Core
} // namespace Impala

#endif

---