|| || || || || ||

---

FeatureTable.h

Go to the documentation of this file.

#ifndef Impala_Core_Feature_FeatureTable_h
#define Impala_Core_Feature_FeatureTable_h

#include "Util/StringParser.h"
#include "Core/Array/Arrays.h"
#include "Core/Vector/VectorSet.h"
#include "Core/Vector/Norm1Dist.h"
#include "Core/Column/Find.h"
#include "Core/Column/Copy.h"
#include "Core/Table/Read.h"
#include "Core/Feature/FeatureDefinition.h"
#include "Core/Table/AnnotationTable.h"
#include "Core/Table/QuidTable.h"
#include "Core/Table/Read.h"
#include "Core/Table/Append.h"

namespace Impala
{
namespace Core
{
namespace Feature
{


typedef Table::TableTem<Column::ColumnTem<Quid>,
                        Vector::VectorSet<Array::Array2dScalarReal64> >
        FeatureTableBaseType;


class FeatureTable : public FeatureTableBaseType
{
public:

    typedef Column::ColumnTem<Quid> ColumnQuid;
    typedef Vector::VectorSet<Array::Array2dScalarReal64> ColumnVectorSet;
    typedef Vector::VectorTem<Real64> VectorReal64;

    FeatureTable(FeatureDefinition def) :
        FeatureTableBaseType(ColumnQuid(0), ColumnVectorSet(true, 1, 0))
    {
        mFeatureDef = def;
        SetColName(1, "object");
        SetColName(2, "values");
    }

    FeatureTable(FeatureDefinition def, int tableSize, int vecLen) :
        FeatureTableBaseType(ColumnQuid(tableSize),
                             ColumnVectorSet(true, vecLen, tableSize))
    {
        mFeatureDef = def;
        SetColName(1, "object");
        SetColName(2, mFeatureDef.GetName());
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    static FeatureTable*
    MakeFromFile(FeatureDefinition def, String fileName, Util::Database* db)
    {
        FeatureTable* res = new FeatureTable(def);
        Read(res, fileName, db);
        return res;
    }

    static FeatureTable*
    MakeFromDataSet(Database::RawDataSet* is, FeatureDefinition mDef,
                    String walkType)
    {
        Int64 totalSize = 0;
        int dimensionality = 0;
        std::vector<String> filenames;
        if (is->GetQuidClass() == QUID_CLASS_IMAGE)
        {
            for (int dirId = 0; dirId < is->NrDirs(); dirId++)
            {
                String fName = is->GetFilePathFeatureData(mDef, dirId, false,
                                                          -1, false, false);
                filenames.push_back(fName);
            }
        }
        else if (is->GetQuidClass() == QUID_CLASS_FRAME)
        {
            for (int i = 0; i < is->NrFiles(); i++)
            {
                String fName = is->GetFilePathFeatureData(walkType, mDef, i, 
                                                          false, -1, 
                                                          false, false);
                filenames.push_back(fName);
            }
        }
        else
        {
            ILOG_ERROR("MakeFromDataSet: Unknown quid class");
            return 0;
        }
        for (int dirId = 0; dirId < filenames.size(); dirId++)
        {
            Feature::FeatureTable tmp(mDef);
            Core::Table::Read(&tmp, filenames[dirId], is->GetDatabase());
            totalSize += tmp.Size();
            dimensionality = tmp.GetFeatureVectorLength();
        }
        FeatureTable* table = new FeatureTable(mDef, totalSize, dimensionality);
        for(int dirId = 0; dirId < filenames.size(); dirId++)
        {
            Feature::FeatureTable tmp(mDef);
            Core::Table::Read(&tmp, filenames[dirId], is->GetDatabase());
            Core::Table::Append(table, &tmp);
        }
        return table;
    }
#endif // REPOSITORY_USED

    FeatureDefinition
    GetFeatureDefinition()
    {
        return mFeatureDef;
    }

    void
    SetFeatureDefinition(FeatureDefinition def)
    {
        mFeatureDef = def;
    }

    int
    GetFeatureVectorLength() const
    {
        return GetColumn2()->GetVectorLength(0);
    }

    int
    FindQuid(Quid quid)
    {
        return Column::Find(GetColumn1(), quid);
    }

    VectorReal64
    FindFeature(Quid quid)
    {
        int rank = FindQuid(quid);
        if (rank >= Size())
        {
            ILOG_ERROR("FindFeature: no quid << " << QuidObj(quid));
            return VectorReal64();
        }
        return Get2(rank);
    }

    Core::Table::QuidTable*
    GetQuidTable()
    {
        Core::Table::QuidTable* t = new Core::Table::QuidTable(Size());
        Copy(t->GetColumn1(), GetColumn1(), Size(), 0, 0);
        t->SetSize(Size());
        return t;
    }

    int
    Diff(FeatureTable* arg) const
    {
        if (Size() != arg->Size())
        {
            ILOG_ERROR("Diff: Size differs: " << Size() << " vs " <<
                       arg->Size());
            return 1;
        }
        if (GetFeatureVectorLength() != arg->GetFeatureVectorLength())
        {
            ILOG_ERROR("Diff: vec length differs: " << GetFeatureVectorLength()
                       << " vs " << arg->GetFeatureVectorLength());
            return 1;
        }
        int nDiff = 0;
        for (int i=0 ; i<Size() ; i++)
        {
            if (Get1(i) != arg->Get1(i))
            {
                ILOG_DEBUG("Quid " << i << " differs " << Get1(i) <<
                           " vs " << arg->Get1(i));
                nDiff++;
            }
            else
            {
                Real64 dist = Vector::Norm1Dist(Get2(i), arg->Get2(i));
                if (IsNan(dist) || (dist > 0.00001))
                {
                    //ILOG_DEBUG("vector " << i << " differs by " << dist << ": "
                    //           << Get2(i) << " vs " << arg->Get2(i));
                    ILOG_DEBUG("vector " << i << " differs by " << dist << ": "
                               << Get2(i).PrintE(5) << " vs "
                               << arg->Get2(i).PrintE(5));
                    nDiff++;
                }
            }
        }
        if (nDiff > 0)
            ILOG_ERROR("Found " << nDiff << " differences");
        return nDiff;
    }

    int
    DiffMatrix(FeatureTable* arg) const
    {
        if (Size() != arg->Size())
        {
            ILOG_ERROR("Diff: Size differs: " << Size() << " vs " <<
                       arg->Size());
            return 1;
        }
        if (GetFeatureVectorLength() != arg->GetFeatureVectorLength())
        {
            ILOG_ERROR("Diff: vec length differs: " << GetFeatureVectorLength()
                       << " vs " << arg->GetFeatureVectorLength());
            return 1;
        }
        int nDiff = 0;
        for (int i=0 ; i<Size() ; i++)
        {
            if (Get1(i) != arg->Get1(i))
            {
                ILOG_DEBUG("Quid " << i << " differs " << Get1(i) <<
                           " vs " << arg->Get1(i));
                nDiff++;
            }
            else
            {
                VectorReal64 v1 = Get2(i);
                VectorReal64 v2 = arg->Get2(i);
                for (int j=0 ; j<v1.Size() ; j++)
                {
                    if (fabs(v1[j] - v2[j]) > 0.00001)
                    {
                        ILOG_DEBUG("row=" << i << ", col=" << j << " differs: "
                                   << v1[j] << " vs " << v2[j]);
                        nDiff++;
                    }
                }
            }
        }
        if (nDiff > 0)
            ILOG_ERROR("Found " << nDiff << " differences");
        return nDiff;
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    void
    WriteSvmFile(String fileName, Util::Database* db)
    {
        WriteSvmFile(fileName, db, 0);
    }

    void
    WriteSvmFile(String fileName, Util::Database* db,
                 Core::Table::AnnotationTable* truth)
    {
        Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp");
        if (buf)
        {
            ExportAscii(buf, truth, true, true);
            delete buf;
        }
    }

    void
    ReadPlainAsciiFile(String fileName, Util::Database* db)
    {
        Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "");
        if (buf)
        {
            ImportAscii(buf);
            delete buf;
        }
    }

    void
    WritePlainAsciiFile(String fileName, Util::Database* db)
    {
        Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp");
        if (buf)
        {
            ExportAscii(buf, 0, false, false);
            delete buf;
        }
    }
#endif // REPOSITORY_USED

    // import feature vectors (not quids!) from plain ascii
    void
    ImportAscii(Util::IOBuffer* buf)
    {
        if (! (buf && buf->Valid()))
            return;

        Quid quid = 0; // invalid
        int nrElem = -1;
        while (buf->Available())
        {
            String line = buf->ReadLine();
            Util::StringParser p(line);
            if (nrElem == -1)
            {
                Real64* data = new Real64[line.size() / 2];
                int i = 0;
                while (!p.TheEnd())
                {
                    Real64 a = p.GetDouble(' ', false, true);
                    data[i++] = a;
                }
                nrElem = i;
                int nrLines = buf->Size() / line.size() + 1; // estimate
                Array::Array2dScalarReal64* storage =
                    new Array::Array2dScalarReal64(nrElem, nrLines, 0, 0);
                GetColumn2()->SetStorage(storage);
                VectorReal64 vec(nrElem, data, false);
                Add(quid, vec);
                delete data;
            }
            else
            {
                VectorReal64 vec(nrElem);
                int i = 0;
                while (!p.TheEnd())
                {
                    Real64 a = p.GetDouble(' ', false, true);
                    vec[i++] = a;
                    if (i > nrElem)
                    {
                        ILOG_ERROR("ImportAscii: too many elements in vector");
                        break;
                    }
                }
                Add(quid, vec);
            }
        }
    }

    void
    ExportAscii(Util::IOBuffer* buf, Core::Table::AnnotationTable* truth,
                bool printQuid, bool printIndex)
    {
        if (! (buf && buf->Valid()))
            return;

        for (int i=0 ; i<Size() ; i++)
        {
            String s("");
            if (printQuid)
            {
                Quid quid = Get1(i);
                if (truth == 0)
                {
                    s = QuidObj(quid).ToString() + " ";
                }
                else
                {
                    s = (truth->IsPositive(quid)) ? "+1 " : "-1 ";
                }
            }
            ColumnVectorSet::ColElemType v = Get2(i);
            for (int j=0 ; j<v.Size() ; j++)
            {
                if (printIndex)
                    s += MakeString(j+1) + ":";
                s += MakeString(v[j]) + " ";
            }
            buf->Puts(s);
        }
    }

    virtual void
    Dump(int from = 0, int to = -1)
    {
        if (to == -1 || to > Size())
            to = Size();
        if (to < from)
            to = from;
        std::cout << "Dumping table from " << from << " to " << to
                  << " (table size=" << Size() << ", capacity=" << Capacity()
                  << ", featureLen=" << GetFeatureVectorLength() << ")"
                  << std::endl;
        for (int i=0 ; i<2 ; i++)
            std::cout << i+1 << "=" << GetColName(i+1) << ", ";
        std::cout << std::endl;
        for (int i=from ; i<to ; i++)
        {
            std::cout << QuidObj(Get1(i)) << ", " << Get2(i) << std::endl;
        }
        std::cout << std::endl;
    }

    virtual void
    Dump(Database::RawDataSet* dataSet, int from = 0, int to = -1)
    {
        if (dataSet == 0)
           return Dump(from, to);

        if (to == -1 || to > Size())
            to = Size();
        if (to < from)
            to = from;
        std::cout << "Dumping table from " << from << " to " << to
                  << " (table size=" << Size() << ", capacity=" << Capacity()
                  << ")" << std::endl;
        for (int i=0 ; i<2 ; i++)
            std::cout << i+1 << "=" << GetColName(i+1) << ", ";
        std::cout << std::endl;
        for (int i=from ; i<to ; i++)
        {
            Quid quid = Get1(i);
            std::cout << dataSet->QuidToString(quid, true) << ", " << Get2(i)
                      << std::endl;
        }
        std::cout << std::endl;
    }

private:

    FeatureDefinition mFeatureDef;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(FeatureTable, Impala.Core.Feature);

} // namespace Feature
} // namespace Core
} // namespace Impala

#endif

---