|| || || || || || || ||

---

Tester.h

Go to the documentation of this file.

#ifndef Impala_Core_Training_Test_h
#define Impala_Core_Training_Test_h

#include "Core/Equals.h"
#include "Core/Table/Equals.h"
#include "Core/Training/AveragePrecision.h"
#include "Core/Training/Equals.h"
#include "Link/Svm/LinkSvm.h"
#include "Core/Training/TesterIoHelper.h"
#include "Core/Array/Equals.h"

namespace Impala
{
namespace Core
{
namespace Training
{

using Core::Equals;
    
class Tester
{
public:
    Tester(DataFactory* input,
           VideoSet::VideoSet* reference,
           VideoSet::VideoSet* output)
        : mIO(input, reference, output)
    {
        mAnnotation = 0;
        mDataSrc = 0;
        mFails = 0;
    }

    virtual ~Tester()
    {
        ClearData();
    }

    void
    Test()
    {
        mFails = 0;
        mIO.ReadAnnotation(mAnnotation);
        TestTrainAndApply("precomputed");
        TestTrainAndApply("rbf");
        TestFolds();
        TestMakeSvmProblem();
        TestAveragePrecision();
        ClearData();
        Report();
    }

    void
    TestMpi()
    {
        mFails = 0;
        mIO.ReadAnnotation(mAnnotation);
        TestTrainAndApply("dist-precomputed");
        TestDistVsNonDist();
        ClearData();
        Report();
    }

    void
    Report()
    {
        if(mFails == 0)
        {
            ILOG_INFO("\n\n\nall tests passed.\n");
        }
        else
        {
            if(mFails == 1)
            {
                ILOG_WARNING("\n\n\n*** 1 test failed! ***\n");
            }
            else
                ILOG_WARNING("\n\n\n*** " << mFails << " tests failed! ***\n");
        }
    }

private:
    void
    TestTrainAndApply(String kernel)
    {
        ClearData(mDataSrc);
        ILOG_INFO("annotation: "<< mAnnotation);
        mIO.MakeTrainData(mDataSrc, mAnnotation, kernel);
        TestTrainModel(kernel);
        TestApply(kernel);
    }

    void
    TestFolds()
    {
        std::vector<Table::QuidTable*> folds =
            mAnnotation->MakeEpisodeFolds(3, 0);
        for(int i=0 ; i<folds.size() ; ++i)
        {
            Table::QuidTable* fold = folds[i];
            mIO.WriteFold(i, fold);
            Table::QuidTable *refFold = mIO.ReadReferenceFold(i);
            Test(fold, refFold, "fold test #" + MakeString(i));
            delete refFold;
            delete fold;
        }
    }

    void
    TestMakeSvmProblem()
    {
        if(mDataSrc == 0)
            mIO.MakeTrainData(mDataSrc, mAnnotation, "rbf");
        FirstFold();
        svm_problem* problem = mDataSrc->MakeSvmProblem();
        mIO.WriteProblem(problem);
        svm_problem* refProblem = mIO.ReadReferenceProblem();
        Test(problem, refProblem, "make svm problem test");
        // ILOG_INFO("dump problem:");
        // Dump(problem, 10, 8, std::cout);
        // ILOG_INFO("dump ref problem:");
        // Dump(refProblem, 10, 8, std::cout);
        DestroySvmProblem(refProblem);
        mDataSrc->FreeProblem(problem);
    }

    void
    TestTrainModel(String kernel)
    {
        FirstFold();
        Util::PropertySet props;
        props.Add("kernel", kernel);
        ILOG_DEBUG("data size = "<< mDataSrc->Size());
        Svm svm;
        svm.Train(&props, mDataSrc);
        mIO.WriteModel(&svm, kernel);
        Svm refSvm;
        mIO.ReadModel(&refSvm, kernel);
        Test(&svm, &refSvm, "train " + kernel + " model");
    }

    void
    TestApply(String kernel)
    {
        FirstFoldInv();
        Svm svm;
        mIO.ReadModel(&svm, kernel);
        Table::ScoreTable* ranking = svm.Predict(mDataSrc);

        mIO.WriteRanking(ranking, kernel);
        Table::ScoreTable* refRanking = mIO.ReadRanking(kernel);
        double tolerance = 1e-14;
        Test(ranking, refRanking, "apply concept " + kernel, tolerance);
        delete ranking;
        delete refRanking;
    }
     
     
     

    void
    TestAveragePrecision()
    {
        Table::ScoreTable* refRanking = mIO.ReadRanking("rbf");
        AveragePrecision ap(mAnnotation);
        double score = ap.Compute(refRanking);
        mIO.WriteAveragePrecision(score);
        double refScore = mIO.ReadAveragePrecision();
        Test(score, refScore, "average precision");
        delete refRanking;
    }

    void
    ClearData()
    {
        ClearData(mAnnotation);
        ClearData(mDataSrc);
    }

    template<class type>
    void
    ClearData(type *& data)
    {
        if(data)
            delete data;
        data = 0;
    }

    void
    FirstFold()
    {
        // getting the test fold is actually wrong, but backwards compatible
        mDataSrc->FilterTestFold(0, 3, 1, false); 
        ILOG_DEBUG("first fold in train data size: " << mDataSrc->Size() );
    }

    void
    FirstFoldInv()
    {
        // getting the train fold is actually wrong, but backwards compatible
        mDataSrc->FilterTrainFold(0, 3, 1, false);
        ILOG_DEBUG("fold 2+3 in train data size: " << mDataSrc->Size() );
    }

    void
    TestDistVsNonDist()
    {
        TrainDataSrc* dist=0;
        mIO.MakeTrainData(dist, mAnnotation, "dist-precomputed");
        Matrix::Mat* matDist = dist->MakeDataCopy(-1, -1);
        //matDist->WriteTo(std::cout, -1, -1);

        TrainDataSrc* nonDist=0;
        mIO.MakeTrainData(nonDist, mAnnotation, "precomputed");
        Matrix::Mat* matNonDist = nonDist->MakeDataCopy(-1, -1);
        //matNonDist->WriteTo(std::cout, -1, -1);

        Test(matDist, matNonDist, "dist == non-dist");
        delete dist;
        delete nonDist;
    }


    template<class type>
    void
    Test(const type item, const type reference, String testname)
    {
        Test(Equals(item, reference), testname);
    }

    template<class type>
    void
    Test(const type item, const type reference, 
              String testname, double tolerance)
    {
        Test(Equals(item, reference, tolerance), testname);
    }

    void
    Test(bool passed, String testname)
    {
        if(passed)
        {
            ILOG_INFO("passed " << testname);
        }
        else
        {
            ++mFails;
            ILOG_WARNING("*** FAILED *** " << testname);
        }
    }

    int mFails;
    TesterIOHelper mIO;
    Table::AnnotationTable* mAnnotation;
    TrainDataSrc* mDataSrc;

    ILOG_VAR_DECL;
};

ILOG_VAR_INIT(Tester, Impala.Core.Training);

}//namespace Core
}//namespace Training
}//Impala

#endif

---