00001 #ifndef Impala_Core_Training_ApplyConcepts_h
00002 #define Impala_Core_Training_ApplyConcepts_h
00003
00004 #include "Core/Database/MakeRawDataSet.h"
00005 #include "Core/Feature/FeatureTable.h"
00006 #include "Core/Training/Classifier.h"
00007 #include "Process/MemoryInfo.h"
00008 #include "Persistency/KeywordListRepository.h"
00009 #include "Persistency/SvmRepository.h"
00010 #include "Persistency/FeatureTableRepository.h"
00011 #include "Persistency/SimilarityTableSetRepository.h"
00012 #include "Core/Training/ApplyConceptsHelperFeatures.h"
00013 #include "Core/Training/ApplyConceptsHelperKernels.h"
00014 #include "Core/Training/ApplyConceptsHelperFik.h"
00015
00016 namespace Impala
00017 {
00018 namespace Core
00019 {
00020 namespace Training
00021 {
00022
00023
00031 class ApplyConcepts
00032 {
00033 public:
00034 typedef Table::ScoreTable ScoreTable;
00035 typedef Feature::FeatureDefinition FeatureDefinition;
00036 typedef Feature::FeatureTable FeatureTable;
00037 typedef Table::KeywordList KeywordList;
00038 typedef Persistency::ModelLocator ModelLocator;
00039 typedef Persistency::FeatureLocator FeatureLocator;
00040 typedef Persistency::SimilarityTableSetLocator SimilarityTableSetLocator;
00041
00042 public:
00043
00051 ApplyConcepts(CmdOptions& options, Database::RawDataSet* dataSet,
00052 bool onIndex)
00053 {
00054 if (options.GetNrArg() < 6)
00055 {
00056 ILOG_ERROR("Missing argument");
00057 return;
00058 }
00059 mDataSet = dataSet;
00060 mAnnoSet = Database::MakeRawDataSet(options.GetArg(2));
00061 String conceptSet = options.GetArg(3);
00062 String modelType = options.GetArg(4);
00063 String featureName = options.GetArg(5);
00064 Persistency::KeywordListLocator keyLoc(mAnnoSet->GetLocator(),
00065 conceptSet);
00066 mConcepts = *(Persistency::KeywordListRepository().Get(keyLoc));
00067
00068 String walkType =
00069 (onIndex) ? options.GetString("classifyIndexCat") : "dummy";
00070 String container = (onIndex) ? "" : "dummy";
00071 mFeatLoc = FeatureLocator(mDataSet->GetLocator(), false, onIndex,
00072 walkType, featureName, container);
00073 mSimSetLoc = SimilarityTableSetLocator(mDataSet->GetLocator(), onIndex,
00074 walkType, conceptSet, modelType,
00075 featureName, container);
00076 mModelLoc = ModelLocator(mAnnoSet->GetLocator(), conceptSet, modelType,
00077 featureName, mConcepts[0]);
00078 mHelper = MakeHelper(options.GetBool("precomputed"),
00079 options.GetBool("fik"));
00080 mHelper->Initialise(mAnnoSet, mModelLoc);
00081 mModelLoc.SetConcept("dummy");
00082 mKernelDataOnly = options.GetBool("kernelDataOnly");
00083 }
00084
00085 virtual
00086 ~ApplyConcepts()
00087 {
00088 delete mHelper;
00089 delete mAnnoSet;
00090 }
00091
00092 virtual void
00093 SetWalkType(CString walkType)
00094 {
00095 mFeatLoc.SetWalkType(walkType);
00096 mSimSetLoc.SetWalkType(walkType);
00097 }
00098
00099 virtual void
00100 NextContainer(CString container)
00101 {
00102 mFeatLoc.SetContainer(container);
00103 mSimSetLoc.SetContainer(container);
00104 if ((!CmdOptions::GetInstance().GetBool("override")) &&
00105 Persistency::SimilarityTableSetRepository().Exists(mSimSetLoc))
00106 {
00107 ILOG_INFO("SimilarityTableSet already exists, skipping");
00108 return;
00109 }
00110 ILOG_DEBUG("loading features: "<<Process::MemoryInfo::GetUsageString());
00111 FeatureTable* features = mHelper->LoadFeatures(mFeatLoc);
00112 if (mKernelDataOnly)
00113 {
00114 ILOG_INFO("KernelDataOnly so skipping real apply");
00115 return;
00116 }
00117
00118 ILOG_DEBUG("done features: "<< Process::MemoryInfo::GetUsageString());
00119 Table::SimilarityTableSet simSet(mConcepts, features->Size());
00120 Core::Table::Copy(simSet.GetQuidTable(), features);
00121
00122 Timer timerTotal;
00123 for (int i=0 ; i<mConcepts.size() ; i++)
00124 {
00125 ILOG_INFO("concept " << i << " of " << mConcepts.size());
00126 Timer timerConcept;
00127 ModelLocator modelLoc = mModelLoc;
00128 modelLoc.SetConcept(mConcepts[i]);
00129 ILOG_INFO("model " << modelLoc);
00130 ILOG_DEBUG("pre-predict: "<< Process::MemoryInfo::GetUsageString());
00131 ScoreTable* scores = mHelper->Predict(modelLoc);
00132 simSet.SetSimTable(i, scores);
00133 delete scores;
00134 ILOG_DEBUG("post-predict: "<< Process::MemoryInfo::GetUsageString());
00135 simSet.ComputeRank(i, true);
00136 ILOG_INFO("time this concept = " << timerConcept.SplitTimeStr()
00137 << ", total = " << timerTotal.SplitTimeStr());
00138 }
00139 ILOG_DEBUG("pre-save: "<< Process::MemoryInfo::GetUsageString());
00140 Persistency::SimilarityTableSetRepository().Add(mSimSetLoc, &simSet);
00141 ILOG_DEBUG("pre-unload: "<< Process::MemoryInfo::GetUsageString());
00142 mHelper->UnloadFeatures();
00143 ILOG_DEBUG("post-unload: "<< Process::MemoryInfo::GetUsageString());
00144 }
00145
00146 private:
00147
00148 ApplyConceptsHelper*
00149 MakeHelper(bool isPrecomputed, bool fik)
00150 {
00151 if (fik)
00152 return new ApplyConceptsHelperFik;
00153 if (isPrecomputed)
00154 return new ApplyConceptsHelperKernels;
00155 return new ApplyConceptsHelperFeatures;
00156 }
00157
00158 ApplyConceptsHelper* mHelper;
00159 KeywordList mConcepts;
00160 ModelLocator mModelLoc;
00161 FeatureLocator mFeatLoc;
00162 SimilarityTableSetLocator mSimSetLoc;
00163 Database::RawDataSet* mDataSet;
00164 Database::RawDataSet* mAnnoSet;
00165 bool mKernelDataOnly;
00166
00167 ILOG_VAR_DEC;
00168
00169 };
00170
00171 ILOG_VAR_INIT(ApplyConcepts, Impala.Core.Training);
00172
00173 }
00174 }
00175 }
00176
00177 #endif