| FeatureTable* Impala::Core::Training::ApplyConceptsHelperKernels::LoadFeatures | ( | Database::PathCreator * | pathCreator | ) | [inline, virtual] |
Implements Impala::Core::Training::ApplyConceptsHelper.
Definition at line 58 of file ApplyConceptsHelperKernels.h.
References Impala::Core::Array::Add(), Impala::Core::Vector::Apply(), Impala::Core::Training::Chi2Distance(), Impala::Core::Table::Copy(), Impala::Core::Array::DivVal(), Impala::Core::Array::Exp(), Impala::FileNameTmp(), Impala::CmdOptions::GetBool(), Impala::Core::Database::RawDataSet::GetDatabase(), Impala::Core::Database::PathCreator::GetDatabase(), Impala::Util::Database::GetDataChannel(), Impala::Core::Database::PathCreator::GetDataSet(), Impala::Core::Database::PathCreator::GetFilePathFeatureData(), Impala::Core::Database::RawDataSet::GetFilePathFeatureIndex(), Impala::Core::Database::PathCreator::GetFolderOrVideoId(), Impala::CmdOptions::GetInstance(), Impala::Util::Database::GetIOBuffer(), Impala::Core::Database::RawDataSet::GetSetName(), Impala::Core::Database::PathCreator::GetWalkType(), ILOG_DEBUG, ILOG_ERROR, ILOG_INFO, ILOG_WARN, mAnnoSet, Impala::Core::Matrix::MatNrCol(), mAverages, mFeatureDefs, mFeatureExternal, mInitialized, mKernelMatrix, mKernelMatrixName, mTotalWeight, mTrainFeatureTempNames, Impala::Core::Array::MulVal(), mUseFeatureCache, mWeights, Impala::Core::Table::Read(), Impala::Core::Array::Set(), Impala::Core::Database::PathCreator::SetFeature(), Impala::Core::Vector::VectorSet< ArrayT >::SetSize(), Impala::Core::Table::Table::Size(), and Impala::Core::Table::Write().
00059 { 00060 if(!mInitialized) 00061 ILOG_ERROR("not initialized"); 00062 if(mFeatureExternal) 00063 ILOG_ERROR("features not empty"); 00064 00065 String kernelDataCacheFilename = 00066 pathCreator->GetDataSet()->GetFilePathKernelData( 00067 pathCreator->GetWalkType(), 00068 mAnnoSet->GetSetName(), 00069 mKernelMatrixName, 00070 pathCreator->GetFolderOrVideoId(), false, true); 00071 if(!kernelDataCacheFilename.empty()) 00072 { 00073 ILOG_INFO("Using cache from KernelData: " << 00074 kernelDataCacheFilename); 00075 // we have an on-disk cache in the KernelData folder 00076 // mFeatureExternal is used elsewhere for the quids and the size 00077 // so this featuretable can perform that role as well 00078 mFeatureExternal = FeatureTable::MakeFromFile( 00079 Feature::FeatureDefinition("cache"), 00080 kernelDataCacheFilename, 00081 pathCreator->GetDatabase()); 00082 00083 // wrap the storage of this feature table in a matrix/array 00084 size_t w = mFeatureExternal->GetColumn2()->GetStorage()->CW(); 00085 size_t h = mFeatureExternal->GetColumn2()->GetStorage()->CH(); 00086 ILOG_DEBUG("size of kernel matrix is ("<< w <<", "<< h <<")"); 00087 mKernelMatrix = new Array::Array2dScalarReal64(w, h, 0, 0, 00088 mFeatureExternal->GetColumn2()->GetStorage()->mData, true); 00089 return mFeatureExternal; 00090 } 00091 00092 mUseFeatureCache = (mAnnoSet->GetDatabase()->GetDataChannel() && 00093 !CmdOptions::GetInstance().GetBool("noFeatureCache")); 00094 if(mTrainFeatureTempNames.size() == 0) 00095 { 00096 for(int i=0 ; i<mFeatureDefs.size() ; ++i) 00097 { 00098 String path = mAnnoSet->GetFilePathFeatureIndex 00099 (mFeatureDefs[i], "", false, false); 00100 if (mUseFeatureCache) 00101 { 00102 /* open the IOBuffers for the training set features 00103 (e.g. copy to local node temp if we are using a 00104 datachannel) */ 00105 ILOG_INFO("buffering from train set FeatureIndex: " << 00106 mFeatureDefs[i].AsString()); 00107 // TODO: there might be a large gain possible here (?) 00108 String tmpName = FileNameTmp(); 00109 Util::IOBuffer* buffer = mAnnoSet->GetDatabase()->GetIOBuffer 00110 (path, true, false, tmpName); 00111 mTrainFeatureTempNames.push_back(tmpName); 00112 delete buffer; 00113 } 00114 else 00115 { 00116 mTrainFeatureTempNames.push_back(path); 00117 } 00118 } 00119 } 00120 00121 int sizeCheck = -1; 00122 Matrix::Mat* accumulator = 0; 00123 for(int i=0 ; i<mWeights.size() ; ++i) 00124 { 00125 // load apply to feature 00126 pathCreator->SetFeature(mFeatureDefs[i].AsString()); 00127 String path = pathCreator->GetFilePathFeatureData(false, false); 00128 if(path.empty()) 00129 { 00130 ILOG_WARN("FeatureData path is empty for the set to apply to:" << 00131 " did you forget --keyframes?"); 00132 } 00133 FeatureTable* feature = FeatureTable::MakeFromFile 00134 (mFeatureDefs[i], path, pathCreator->GetDatabase()); 00135 if(sizeCheck == -1) 00136 sizeCheck = feature->Size(); 00137 //TODO: what should happen is a comparison of the quids of the 00138 //different tables, not a check on just the size 00139 if(feature->Size() != sizeCheck) 00140 { 00141 ILOG_ERROR("Size mismatch in the FeatureData for the dataset to" 00142 << " apply to: have " << sizeCheck << " and " << 00143 feature->Size()); 00144 throw "bye"; 00145 } 00146 ILOG_INFO("size of apply set " << mFeatureDefs[i].AsString() << " is " << 00147 feature->Size()); 00148 00149 // load train feature 00150 FeatureTable* trainFeature = new FeatureTable(mFeatureDefs[i]); 00151 Util::IOBuffer* buffer = 0; 00152 if(mUseFeatureCache) 00153 { 00154 buffer = new Util::IOBufferFile 00155 (mTrainFeatureTempNames[i], true, false); 00156 } 00157 else 00158 { 00159 buffer = mAnnoSet->GetDatabase()->GetIOBuffer 00160 (mTrainFeatureTempNames[i], true, false, ""); 00161 } 00162 00163 Read(trainFeature, buffer); 00164 ILOG_INFO("size of train " << mFeatureDefs[i].AsString() << " is " << 00165 trainFeature->Size()); 00166 delete buffer; 00167 00168 // compute kernel distances between features 00169 typedef Feature::FeatureTable::ColumnVectorSet VectorSet; 00170 VectorSet* vectorsV = feature->GetColumn2(); 00171 VectorSet* vectorsH = trainFeature->GetColumn2(); 00172 vectorsV->SetSize(feature->Size()); 00173 vectorsH->SetSize(trainFeature->Size()); 00174 ILOG_INFO("Compute distance matrix for feature " << i << "..."); 00175 Matrix::Mat* distanceMatrix = Apply(&Training::Chi2Distance, 00176 vectorsH, vectorsV); 00177 00178 delete trainFeature; 00179 00180 // accumulate; this is the part inside the 'exp' in the kernel function 00181 MulVal(distanceMatrix, distanceMatrix, mWeights[i]); 00182 DivVal(distanceMatrix, distanceMatrix, -mAverages[i]); 00183 if(accumulator == 0) 00184 Set(accumulator, distanceMatrix); 00185 else 00186 Add(accumulator, accumulator, distanceMatrix); 00187 delete distanceMatrix; 00188 if(i == 0) 00189 { 00190 mFeatureExternal = feature; 00191 } 00192 else 00193 delete feature; 00194 } 00195 // only thing to do is weigh and exp 00196 DivVal(accumulator, accumulator, mTotalWeight); 00197 Exp(accumulator, accumulator); 00198 mKernelMatrix = accumulator; 00199 if(CmdOptions::GetInstance().GetBool("storeKernelData")) 00200 { 00201 FeatureTable* storage = new FeatureTable(mKernelMatrixName, sizeCheck, Matrix::MatNrCol(mKernelMatrix)); 00202 Column::Copy(storage->GetColumn1(), mFeatureExternal->GetColumn1(), sizeCheck); 00203 // the VectorSet likes owning the storage after setting it, 00204 // so we must create a wrapper to prevent it from deleting the 00205 // kernel matrix when the storage is destroyed 00206 Array::Array2dScalarReal64* wrapper = 00207 new Array::Array2dScalarReal64(mKernelMatrix->CW(), mKernelMatrix->CH(), 0, 0, mKernelMatrix->mData, true); 00208 storage->GetColumn2()->SetStorage(wrapper); 00209 storage->GetColumn2()->SetSize(sizeCheck); 00210 storage->SetSize(sizeCheck); 00211 String storageFilename = 00212 pathCreator->GetDataSet()->GetFilePathKernelData( 00213 pathCreator->GetWalkType(), 00214 mAnnoSet->GetSetName(), 00215 mKernelMatrixName, 00216 pathCreator->GetFolderOrVideoId(), true, false); 00217 Write(storage, storageFilename, pathCreator->GetDatabase(), true); 00218 delete storage; 00219 } 00220 return mFeatureExternal; 00221 }
Here is the call graph for this function:
