|| || || || || ||

---

SimilarityTableSet.h

Go to the documentation of this file.

#ifndef Impala_Core_Table_SimilarityTableSet_h
#define Impala_Core_Table_SimilarityTableSet_h

#include <vector>
#include "Util/DatabaseReadString.h"
#include "Util/DatabaseWriteString.h"
#include "Basis/NativeTypes.h"
#include "Core/Column/GenerateId.h"
#include "Core/Column/Sort.h"
#include "Core/Column/Find.h"
#include "Core/Table/TableTem.h"
#include "Core/Table/Copy.h"
#include "Core/Table/Read.h"
#include "Core/Table/Write.h"
#include "Core/Table/Append.h"
#include "Core/Table/QuidTable.h"
#include "Core/Table/ScoreTable.h"
#include "Core/Table/Equals.h"
#include "Core/ImageSet/ImageSet.h"
#include "Core/VideoSet/VideoSet.h"

namespace Impala
{
namespace Core
{
namespace Table
{


class SimilarityTableSet
{
public:

    typedef TableTem<Column::ColumnTem<Real64> > SimTableType;
    typedef TableTem<Column::ColumnTem<Quid> > RankTableType;

    SimilarityTableSet(std::vector<String> names, int tableSize)
    {
        mQuids = new QuidTable(tableSize);
        mQuids->SetColName(1, "quid");
        for (size_t i=0 ; i<names.size() ; i++)
        {
            AddTable(names[i]);
        }
    }

    virtual
    ~SimilarityTableSet()
    {
        delete mQuids;
        for (size_t i=0 ; i<mSims.size() ; i++)
            delete mSims[i];
        for (size_t i=0 ; i<mRanks.size() ; i++)
            delete mRanks[i];
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    static SimilarityTableSet*
    MakeFromFile(VideoSet::VideoSet* videoSet, String conceptSet, String model,
                 String feature)
    {
        String path = videoSet->GetFilePathSimilarityIndex(conceptSet, model,
                                                           feature, "names.txt",
                                                           false, false);
        if (path.empty())
            return 0;
        return MakeFromFile(path, videoSet->GetDatabase());
    }

    static SimilarityTableSet*
    MakeFromFile(Database::RawDataSet* dataSet, String walkType, String conceptSet,
                 String model, String featureString, int fileOrDirId)
    {
        String path =
            dataSet->GetFilePathSimilarityData(walkType, conceptSet, model,
                                                featureString, fileOrDirId,
                                                "names.txt", false, false);
        if (path.empty())
            return 0;
        return MakeFromFile(path, dataSet->GetDatabase());
    }

// RvB: Added in support of VOC image sets
static SimilarityTableSet*
MakeFromFile(ImageSet::ImageSet* is, String conceptSet, String model,
             String featureString)
{
    String path =
        is->GetFilePathSimilarityIndex(conceptSet, model, featureString,
                                      "names.txt", false, false);
    if (path.empty())
        return 0;
    return MakeFromFile(path, is->GetDatabase());
}

    static SimilarityTableSet*
    MakeFromFile(String path, Util::Database* db) // path to "names.txt"
    {
        std::vector<String> names;
        Util::DatabaseReadString(std::back_inserter(names), path, db, true);
        SimilarityTableSet* res = new SimilarityTableSet(names, 0);
        path.resize(path.size() - 9); // remove "names.txt"
        if (path.size() == 0)
            path = "./";
        res->Load(path, db);
        path.resize(path.size() - 1); // remove "/"
        String dir = FileNameTail(path);
        res->SetDescription(dir);
        return res;
    }

    static bool
    Exists(Database::RawDataSet* dataSet, String walkType, String conceptSet,
           String model, String featureString, int fileId)
    {
        String path =
            dataSet->GetFilePathSimilarityData(walkType, conceptSet, model,
                                               featureString, fileId,
                                               "names.txt", false, true);
        return !path.empty();
    }

#endif // REPOSITORY_USED

    String
    GetDescription()
    {
        return mDescription;
    }

    void
    SetDescription(String description)
    {
        mDescription = description;
    }

    int
    NrTables() const
    {
        return mNames.size();
    }

    int
    TableSize() const
    {
        return mQuids->Size();
    }

    std::vector<String>
    GetNames()
    {
        return mNames;
    }

    String
    GetName(int id) const
    {
        return mNames[id];
    }

    int
    GetIndex(String name)
    {
        for (int i=0 ; i<mNames.size() ; i++)
            if (mNames[i] == name)
                return i;
        return -1;
    }

    void
    Rename(int id, String name)
    {
        mNames[id] = name;
    }

    int
    GetQuidClass() const
    {
        if (mQuids->Size() < 1) // cannot tell
            return QUID_CLASS_UNKNOWN;
        return QuidClass(mQuids->Get1(0));
    }

    QuidTable*
    GetQuidTable() const
    {
        return mQuids;
    }

    int
    FindQuid(Quid quid)
    {
        return Column::Find(mQuids->GetColumn1(), quid);
    }

    void
    ChangeQuidObject(int newObject)
    {
        int nrChanged = 0;
        for (int i=0 ; i<mQuids->Size() ; i++)
        {
            Quid q = mQuids->Get1(i);
            if (QuidObject(q) != newObject)
            {
                q = QuidObjectChange(q, newObject);
                mQuids->Set1(i, q);
                nrChanged++;
            }
        }
        if (nrChanged != 0)
            ILOG_INFO("Changed " << nrChanged << " quid objects");
    }

    SimTableType*
    GetSimTable(int id) const
    {
        return mSims[id];
    }

    void
    SetSimTable(int id, ScoreTable* scores)
    {
        ILOG_INFO("score table size = " << scores->Size());
        if(scores->Size() != mQuids->Size())
        {
            ILOG_ERROR("SetSimTable: size of quids and ScoreTable do not match");
            scores->GetQuidTable()->Dump(0, 0, -1);
            mQuids->Dump(0, 0, -1);
            exit(1);
        }
        SimTableType* dst = mSims[id];
        dst->SetSize(scores->Size());
        Copy(dst->GetColumn1(), scores->GetColumn2(), scores->Size());
    }

    SimTableType*
    GetSimTable(String name) const
    {
        for (size_t i=0 ; i<mNames.size() ; i++)
        {
            if (mNames[i] == name)
                return mSims[i];
        }
        return 0;
    }

    RankTableType*
    GetRankTable(int id) const
    {
        return mRanks[id];
    }

    RankTableType*
    GetRankTable(String name) const
    {
        for (size_t i=0 ; i<mNames.size() ; i++)
        {
            if (mNames[i] == name)
                return mRanks[i];
        }
        return 0;
    }

    void
    ComputeRank(int tableIdx, bool descending)
    {
        SimTableType* simTable = mSims[tableIdx];
        int size = simTable->Size();
        if (size != mQuids->Size())
            ILOG_ERROR("ComputeRank: tables sizes do not match: " << size << 
                       " " << mQuids->Size());
        SimTableType* tmp = new SimTableType(size);
        Core::Table::Copy(tmp, simTable);
        Core::Table::Copy(mRanks[tableIdx], mQuids);
        if (descending)
            Column::SortDescending(mRanks[tableIdx]->GetColumn1(),
                                   tmp->GetColumn1(), tmp->Size());
        else
            Column::SortAscending(mRanks[tableIdx]->GetColumn1(),
                                  tmp->GetColumn1(), tmp->Size());
        delete tmp;
    }

    void
    ComputeRanks(bool descending)
    {
        SimTableType* tmp = new SimTableType(mQuids->Size());
        for (int i=0 ; i<NrTables() ; i++)
        {
            ILOG_DEBUG("Sorting table " << i << " of " << NrTables());
            if (mSims[i]->Size() != mQuids->Size())
            {
                ILOG_ERROR("ComputeRanks: tables sizes do not match: " <<
                           mSims[i]->Size() << " " << mQuids->Size());
                continue;
            }
            Core::Table::Copy(mRanks[i], mQuids);
            Core::Table::Copy(tmp, mSims[i]);
            if (descending)
                Column::SortDescending(mRanks[i]->GetColumn1(),
                                       tmp->GetColumn1(), tmp->Size());
            else
                Column::SortAscending(mRanks[i]->GetColumn1(),
                                      tmp->GetColumn1(), tmp->Size());
        }
        delete tmp;
    }

    void
    ComputeRanksTopN(int topN, bool descending)
    {
        if (mQuids->Size() < topN)
        {
            ILOG_ERROR("ComputeRanksTopN: not enough quids for top " << topN);
            return;
        }
        if (mRanks[0]->Capacity() != topN)
        {
            for (int i=0 ; i<NrTables() ; i++)
            {
                mRanks[i]->Reserve(topN, false);
                mRanks[i]->SetSize(topN); // will be filled below
            }
        }
        Column::ColumnTem<Real64>* tmp = new Column::ColumnTem<Real64>(topN);
        for (int i=0 ; i<NrTables() ; i++)
        {
            if (descending)
                Column::SortDescendingTopN(mQuids->GetColumn1(),
                                           mSims[i]->GetColumn1(),
                                           mRanks[i]->GetColumn1(), tmp);
            else
                Column::SortAscendingTopN(mQuids->GetColumn1(),
                                          mSims[i]->GetColumn1(),
                                          mRanks[i]->GetColumn1(), tmp);
        }
        delete tmp;
    }

    void
    Append(SimilarityTableSet* simSet)
    {
        if (NrTables() != simSet->NrTables())
        {
            ILOG_ERROR("Append: different number of tables");
            return;
        }
        Core::Table::Append(GetQuidTable(), simSet->GetQuidTable());
        for (int i=0 ; i<NrTables() ; i++)
        {
            Core::Table::Append(GetSimTable(i), simSet->GetSimTable(i));
        }
    }

    void
    Merge(SimilarityTableSet* arg)
    {
        if (mQuids->Diff(arg->GetQuidTable()) != 0)
        {
            ILOG_ERROR("Merge: different quids so cannot merge");
            return;
        }
        for (int i=0 ; i<arg->NrTables() ; i++)
        {
            String name = arg->GetName(i);
            int idx = GetIndex(name);
            while (idx != -1)
            {
                name = name + "_2";
                idx = GetIndex(name);
            }
            AddTable(name);
            idx = GetIndex(name);
            Core::Table::Copy(GetSimTable(idx), arg->GetSimTable(i));
            Core::Table::Copy(GetRankTable(idx), arg->GetRankTable(i));
        }
    }

    // Differences in rank may occur due to non-deterministic sorting...
    int
    Diff(SimilarityTableSet* arg, bool doDiffRank = false) const
    {
        if (NrTables() != arg->NrTables())
        {
            ILOG_ERROR("Diff: NrTables differs: " << NrTables() << " vs "
                       << arg->NrTables());
            return 1;
        }
        if (TableSize() != arg->TableSize())
        {
            ILOG_ERROR("Diff: TableSize differs: " << TableSize() << " vs "
                       << arg->TableSize());
            return 1;
        }
        QuidTable* qTable1 = GetQuidTable();
        QuidTable* qTable2 = arg->GetQuidTable();
        int qDiff = 0;
        for (int i=0 ; i<qTable1->Size() ; i++)
        {
            if (qTable1->Get1(i) != qTable2->Get1(i))
            {
                qDiff++;
                ILOG_DEBUG("Diff: quid elem " << i << ": "
                           << qTable1->Get1(i) << " vs " << qTable2->Get1(i));
            }
        }
        if (qDiff > 0)
        {
            ILOG_ERROR("Diff: " << qDiff << " differences in quid table");
            return qDiff;
        }

        int totalDiff = 0;
        for (int t=0 ; t<NrTables() ; t++)
        {
            int nDiff = 0;
            SimTableType* sTable1 = GetSimTable(t);
            SimTableType* sTable2 = arg->GetSimTable(t);
            for (int i=0 ; i<sTable1->Size() ; i++)
            {
                if (::fabs(sTable1->Get1(i) - sTable2->Get1(i)) > 0.00001)
                {
                    nDiff++;
                    ILOG_DEBUG("Diff: sim table " << t << ", elem " << i <<
                               ": " << sTable1->Get1(i) << " vs " <<
                               sTable2->Get1(i));
                }
            }
            if (nDiff > 0)
                ILOG_ERROR("Diff: " << nDiff << " differences in sim table " <<
                           GetName(t));
            totalDiff += nDiff;

            if (!doDiffRank)
                continue;

            nDiff = 0;
            RankTableType* rTable1 = GetRankTable(t);
            RankTableType* rTable2 = arg->GetRankTable(t);
            for (int i=0 ; i<rTable1->Size() ; i++)
            {
                if (rTable1->Get1(i) != rTable2->Get1(i))
                {
                    nDiff++;
                    ILOG_DEBUG("Diff: rank table " << t << ", elem " << i <<
                               ": " << rTable1->Get1(i) << " vs " <<
                               rTable2->Get1(i));
                }
            }
            if (nDiff > 0)
                ILOG_ERROR("Diff: " << nDiff << " differences in rank table " <<
                           GetName(t));
            totalDiff += nDiff;
        }

        if (totalDiff > 0)
            ILOG_ERROR("Diff: total of " << totalDiff << " differences");
        return totalDiff;
    }

    // I/O part

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    void
    Load(CString fileBase, Util::Database* db)
    {
        LoadQuids(fileBase, db);
        LoadSims(fileBase, db);
        LoadRanks(fileBase, db);
    }

    void
    LoadQuids(CString fileBase, Util::Database* db)
    {
        String fName = fileBase + "all_quids.tab";
        Core::Table::Read(mQuids, fName, db);
    }

    void
    LoadSim(size_t tableIdx, CString fileBase, Util::Database* db)
    {
        String fName = fileBase + mNames[tableIdx] + "_sim.tab";
        Core::Table::Read(mSims[tableIdx], fName, db);
    }

    void
    LoadSims(CString fileBase, Util::Database* db)
    {
        for (size_t i=0 ; i<mNames.size() ; i++)
            LoadSim(i, fileBase, db);
    }

    void
    LoadRank(size_t tableIdx, CString fileBase, Util::Database* db)
    {
        String fName = fileBase + mNames[tableIdx] + "_rank.tab";
        Core::Table::Read(mRanks[tableIdx], fName, db);
    }

    void
    LoadRanks(CString fileBase, Util::Database* db)
    {
        for (size_t i=0 ; i<mNames.size() ; i++)
            LoadRank(i, fileBase, db);
    }

    void
    Save(int tableIdx, CString fileBase, Util::Database* db, bool binary)
    {
        SaveQuids(fileBase, db, binary);
        SaveSim(tableIdx, fileBase, db, binary);
        SaveRank(tableIdx, fileBase, db, binary);
    }

    void
    Save(Database::RawDataSet* dataSet, CString conceptSet, CString model,
         CString feature, bool binary)
    {
        String path = dataSet->GetFilePathSimilarityIndex(conceptSet, model,
                                                          feature, "names.txt",
                                                          true, false);
        if (path.empty())
            return;
        path.resize(path.size() - 9); // remove "names.txt"
        Save(path, dataSet->GetDatabase(), binary);
    }

    void
    Save(Database::RawDataSet* dataSet, String walkType, String conceptSet,
         CString model, CString feature, int fileOrDirId, bool binary)
    {
        String path = dataSet->GetFilePathSimilarityData
            (walkType, conceptSet, model, feature, fileOrDirId, "names.txt",
             true, false);
        if (path.empty())
            return;
        path.resize(path.size() - 9); // remove "names.txt"
        Save(path, dataSet->GetDatabase(), binary);
    }

    void
    Save(CString fileBase, Util::Database* db, bool binary)
    {
        SaveNames(fileBase, db);
        SaveQuids(fileBase, db, binary);
        SaveSims(fileBase, db, binary);
        SaveRanks(fileBase, db, binary);
    }

    void
    SaveNames(CString fileBase, Util::Database* db)
    {
        String fName = fileBase + "names.txt";
        Util::DatabaseWriteString(fName, db, mNames.begin(), mNames.end());
    }

    void
    SaveQuids(CString fileBase, Util::Database* db, bool binary)
    {
        String fName = fileBase + "all_quids.tab";
        Core::Table::Write(mQuids, fName, db, binary);
    }

    void
    SaveSim(size_t tableIdx, CString fileBase, Util::Database* db, bool binary)
    {
        String fName = fileBase + mNames[tableIdx] + "_sim.tab";
        Core::Table::Write(mSims[tableIdx], fName, db, binary);
    }

    void
    SaveSims(CString fileBase, Util::Database* db, bool binary)
    {
        for (size_t i=0 ; i<mNames.size() ; i++)
            SaveSim(i, fileBase, db, binary);
    }

    void
    SaveRank(size_t tableIdx, CString fileBase, Util::Database* db, bool binary)
    {
        String fName = fileBase + mNames[tableIdx] + "_rank.tab";
        Core::Table::Write(mRanks[tableIdx], fName, db, binary);
    }

    void
    SaveRanks(CString fileBase, Util::Database* db, bool binary)
    {
        for (size_t i=0 ; i<mNames.size() ; i++)
            SaveRank(i, fileBase, db, binary);
    }
#endif // REPOSITORY_USED

    // misc

    void
    Dump(bool doTable)
    {
        std::cout << "SimilarityTableSet " << GetDescription() << std::endl;
        for (int i=0 ; i<NrTables() ; i++)
        {
            std::cout << "table " << i << ", name = " << GetName(i) << std::endl;
            if (doTable)
                GetSimTable(i)->Dump();
        }
    }

    void
    DumpRanking(int from = 0, int to = -1)
    {
        DumpRanking(0, from, to);
    }

    virtual void
    DumpRanking(Database::RawDataSet* set, int from = 0, int to = -1)
    {
        if (to == -1 || to > TableSize())
            to = TableSize();
        if (to < from)
            to = from;
        std::cout << "SimilarityTableSet " << GetDescription() << std::endl;
        for (int t=0 ; t<NrTables() ; t++)
        {
            SimTableType* simTable = GetSimTable(t);
            RankTableType* rankTable = GetRankTable(t);
            std::cout << "table " << t << ", name = " << GetName(t) << std::endl;
            if (simTable->Size() != mQuids->Size())
            {
                ILOG_ERROR("DumpRanking: simtable size doesn't match");
                continue;
            }
            if (rankTable->Size() != mQuids->Size())
            {
                ILOG_ERROR("DumpRanking: ranktable size doesn't match");
                continue;
            }
            std::cout << "i, Quid, similarity, rank, sim_at_rank" << std::endl;
            for (int i=from ; i<to ; i++)
            {
                Quid quid = mQuids->Get1(i);
                std::cout << i << ", ";
                if (set && (quid != 0))
                    std::cout << set->QuidToString(quid, true) << ", ";
                else
                    std::cout << QuidObj(quid) << ", ";
                std::cout << simTable->Get1(i) << ", ";
                int rank = Find(rankTable->GetColumn1(), quid);
                std::cout << rank << ", ";
                rank = FindQuid(rankTable->Get1(i));
                if ((rank >= 0) && (rank < simTable->Size()))
                    std::cout << simTable->Get1(rank) << std::endl;
                else
                    std::cout << "<no sim>" << std::endl;
            }
            std::cout << std::endl;
        }
    }

    //dump top N rank list, sorted based on similarity values
    void
    DumpRankingSorted(Database::RawDataSet* set, Impala::String OutPath, int from = 0, int to = -1)
    {
        if (to == -1 || to > TableSize())
            to = TableSize();
        if (to < from)
            to = from;
        std::cout << "SimilarityTableSet " << GetDescription() << std::endl;
        for (int t=0 ; t<NrTables() ; t++)
        {
            SimTableType* simTable = GetSimTable(t);
            RankTableType* rankTable = GetRankTable(t);
            //std::cout << "table " << t << ", name = " << GetName(t) << std::endl;
            if (simTable->Size() != mQuids->Size())
            {
                ILOG_ERROR("DumpRanking: simtable size doesn't match");
                continue;
            }
            if (rankTable->Size() != mQuids->Size())
            {
                ILOG_ERROR("DumpRanking: ranktable size doesn't match");
                continue;
            }

            String ListName = OutPath + GetName(t) + ".txt";
            FILE* fp = fopen(ListName.c_str(), "wt");
            //std::cout << "rank, i(position), Quid, similarity" << std::endl;
            fprintf(fp,"1=rank, 2=position(i), 3=Quid, 4=similarity\n");
            for (int i=from ; i<to ; i++)
            {
                //Quid quid = mQuids->Get1(i);
                Quid quid = rankTable->Get1(i);
                int rank = FindQuid(quid);
                
                //std::cout << rank << ", ";
                String QuidStr;
                if (set && (quid != 0))
                    QuidStr = set->QuidToString(quid, true);
                else
                    QuidStr = QuidObj(quid).ToString();
                //std::cout << QuidStr << ", ";

                Real64 sim = simTable->Get1(rank);
                fprintf(fp,"%d\t%d\t%s\t%f\n",i, rank, QuidStr.c_str(), sim);

                //std::cout << simTable->Get1(rank) << ", ";
                //std::cout << i << ", ";
                //if ((rank >= 0) && (rank < simTable->Size()))
                //    std::cout << simTable->Get1(rank) << std::endl;
                //else
                //    std::cout << "<no sim>" << std::endl;
            }
            fclose(fp);
            //std::cout << std::endl;
        }
    }

    void
    DumpMatrix(Database::RawDataSet* set, bool dumpQuid)
    {
        for (int i=0 ; i<TableSize() ; i++)
        {
            Quid quid = mQuids->Get1(i);
            if (dumpQuid)
            {
                if (set)
                    std::cout << set->QuidToString(quid, true) << ", ";
                else
                    std::cout << QuidObj(quid) << ", ";
            }
            for (int t=0 ; t<NrTables() ; t++)
            {
                SimTableType* simTable = GetSimTable(t);
                if ((i == 0) && (simTable->Size() != mQuids->Size()))
                {
                    ILOG_ERROR("DumpMatrix: simtable size doesn't match");
                    return;
                }
                std::cout << simTable->Get1(i);
                if (t < NrTables() - 1)
                    std::cout << ", ";
            }
            std::cout << std::endl;
        }
    }

private:

    void
    AddTable(CString name)
    {
        mNames.push_back(name);
        SimTableType* sim = new SimTableType(mQuids->Capacity());
        sim->SetColName(1, "similarity");
        mSims.push_back(sim);
        RankTableType* rank = new RankTableType(mQuids->Capacity());
        rank->SetColName(1, "id");
        mRanks.push_back(rank);
    }

    String                      mDescription;
    QuidTable*                  mQuids;
    std::vector<String>         mNames;
    std::vector<SimTableType*>  mSims;
    std::vector<RankTableType*> mRanks;

    ILOG_VAR_DEC;

};

ILOG_VAR_INIT(SimilarityTableSet, Impala.Core.Table);

} // namespace Table
} // namespace Core
} // namespace Impala

#endif

---