| Feature::FeatureTable* Impala::Application::OpenFeatureTable | ( | Feature::FeatureDefinition & | featureDef, | |
| RawDataSet * | dataset | |||
| ) |
opens feature table in MPI mode only node0 reads the table, then it is broadcasted
Definition at line 122 of file mainPrecomputeKernelMatrix.cpp.
References Impala::Core::Feature::FeatureDefinition::AsString(), Impala::Util::Broadcast(), Impala::Util::Database::GetIOBuffer(), ILOG_ERROR, ILOG_VAR, and Impala::Core::Table::Read().
Referenced by mainPrecomputeKernelMatrix().
00124 { 00125 /* if MPI 00126 node 0 opens IOBuffer and Broadcast 00127 every node loads table from IOBuffer 00128 every node only selects it's own part of the table 00129 */ 00130 ILOG_VAR(Application.mainPrecomputeKernelMatrix.OpenFeatureTable); 00131 Feature::FeatureTable* f = 0; 00132 std::string filename = 00133 dataset->GetFilePathFeatureIndex(featureDef, "", false, false); 00134 ILOG_INFO_ONCE("opening " << featureDef.AsString()); 00135 #ifndef MPI_USED 00136 f = Feature::FeatureTable::MakeFromFile(featureDef, filename, 00137 dataset->GetDatabase()); 00138 //f->SetSize(20); 00139 //ILOG_WARNING("***debug statment should be removed!***"); 00140 #else 00141 00146 Util::IOBuffer* buf; 00147 int myId = Link::Mpi::MyId(); 00148 if(myId == 0) 00149 { 00150 // use the dataset-relative database (!) 00151 Util::Database* db = dataset->GetDatabase(); 00152 buf = db->GetIOBuffer(filename, true, true, ""); 00153 if(buf == 0 || !buf->Valid()) 00154 { 00155 ILOG_ERROR("node0 couldn't open file"); 00156 } 00157 } 00158 else 00159 { 00160 buf = new Util::IOBuffer(); 00161 } 00162 Broadcast(buf); 00163 if (buf && buf->Valid()) 00164 { 00165 f = new Feature::FeatureTable(featureDef); 00166 Read(f, buf); 00167 delete buf; 00168 } 00169 else 00170 ILOG_ERROR(myId << ": could not read file from buffer"); 00171 Link::Mpi::Barrier(); 00172 #endif 00173 ILOG_INFO_ONCE("size = " << f->Size() << "; length = " << f->GetFeatureVectorLength()); 00174 00175 return f; 00176 }
Here is the call graph for this function:
