|| || || || || || || ||

---

mainPrecomputeKernelMatrix

int Impala::Application::mainPrecomputeKernelMatrix ( CmdOptions &  options  )  Definition at line 426 of file mainPrecomputeKernelMatrix.cpp. References Impala::Core::Array::Add(), Impala::atof(), CheckParameteres(), ComputeMatrix(), Impala::Core::Array::DivVal(), Impala::Core::Array::Exp(), Impala::CmdOptions::GetArg(), GetAverage(), Impala::CmdOptions::GetNrArg(), GetPartialTask(), Impala::Core::Feature::FeatureTable::GetQuidTable(), ILOG_ERROR, ILOG_INFO, ILOG_VAR, LoadAverages(), Impala::Core::Database::MakeRawDataSet(), Impala::Core::Array::MulVal(), OpenFeatureTable(), Impala::Core::Array::Set(), Impala::Core::Table::Table::Size(), Impala::Core::Table::Write(), WriteAverages(), WriteInfoFile(), and WriteResult(). Referenced by main(). { // this name is 26 characters. if you are tracing unexplainable C++ string // exceptions, check your log4cpp.properties to see if you are limiting // this name to just 25 characters... ILOG_VAR(Application.mainPrecomputeKernelMatrix); RawDataSet* dataset = Core::Database::MakeRawDataSet(options.GetArg(0)); if(!dataset) { ILOG_INFO_ONCE("failed to open dataset " << options.GetArg(0)); return 1; } String name2 = options.GetArg(1); RawDataSet* dataset2 = 0; if((name2 != "0") && (atof(name2) == 0.0)) // it is not a number, or 0 { dataset2 = Core::Database::MakeRawDataSet(name2, true); } if(dataset2) ILOG_INFO_ONCE("2 sets loaded: " << dataset2->GetSetName()); // for distribution we compute what part of the matrix we must compute // in single process case partcount == 1 // in multi process case it is sqrt(NrProcs()) (or 0 if this node doesn't compute anything) int partcount, row, column; GetPartialTask(partcount, row, column); ILOG_INFO_ONCE("Using a " << partcount << "x" << partcount << "grid"); int cpuCount = Link::Mpi::NrProcs(); ILOG_INFO_ONCE("total nodes = " << cpuCount << " unused nodes = " << cpuCount - partcount*partcount); if(partcount == 0) { ILOG_ERROR("unsupported number of nodes, number of nodes MUST be" << " sqare of a natural number"); exit(0); } else { ILOG_DEBUG_NODE("col = " << column << " row = " << row); } std::vector<Feature::FeatureDefinition> featureDefs; std::vector<double> weights; String resultname = options.GetArg(options.GetNrArg()-1); if(!CheckParameteres(options, dataset, dataset2, featureDefs, weights, resultname)) return 0; std::vector<double> averages; averages.resize(weights.size()); if(dataset2) { String filename = resultname + ".averages.raw"; filename = dataset->GetFilePathPrecomputedKernels(filename, "", false, false); if(filename == "") { ILOG_ERROR("couldn't read averages"); exit(0); } LoadAverages(dataset, filename, averages); for (int i = 0; i < averages.size(); i++) { ILOG_INFO_ONCE("average " << i << " " << averages[i]); } } if(!dataset2) { String tmp = dataset->GetFilePathPrecomputedKernels(resultname+".info", "", true, true); String tmp2 = dataset->GetFilePathPrecomputedKernels(resultname+".averages.raw", "", true, true); if(tmp.empty() && tmp2.empty()) { ILOG_INFO("A complete kernel already exists, nothing to do."); return 0; } if(tmp.empty()) { ILOG_ERROR("Incomplete kernel exists! Cleanup files first."); return 1; } } Util::Database* db = (dataset2) ? dataset2->GetDatabase() : dataset->GetDatabase(); Matrix::Mat* accumulator = 0; double totalweight = 0; for(int i=0 ; i<weights.size() ; ++i) { // open feature tables double weight = weights[i]; Feature::FeatureTable* f1 = OpenFeatureTable(featureDefs[i], dataset); Feature::FeatureTable* f2 = f1; if(dataset2) f2 = OpenFeatureTable(featureDefs[i], dataset2); if(Link::Mpi::MyId() == 0 && i == 0) { // create a .info file once String filename = resultname+".info"; if(dataset2) filename = dataset2->GetFilePathPrecomputedKernels(filename, dataset->GetSetName(), true, false); else filename = dataset->GetFilePathPrecomputedKernels(filename, "", true, false); if(filename == "") { ILOG_ERROR("could not save .info file"); exit(0); } ILOG_INFO("Saving info in " << filename); WriteInfoFile(f1->Size(), f2->Size(), partcount, filename, db); // write the total list of devel-quids once if (dataset2) filename = dataset2->GetFilePathPrecomputedKernels( resultname + ".columns.tab", dataset->GetSetName(), true, false); else filename = dataset->GetFilePathPrecomputedKernels (resultname + ".columns.tab", "", true, false); ILOG_INFO("Saving columns in " << filename); if (!filename.empty()) Write(f1->GetQuidTable(), filename, dataset->GetDatabase(), true); else ILOG_ERROR("Unable to write columns.tab"); } // compute kernel distances between features Matrix::Mat *distanceMatrix = ComputeMatrix(f1, f2, resultname, dataset, dataset2); // f1 & f2 are deleted by ComputeMatrix // if this is the 'learn' kernel matrix we compute the averages and // communicate between nodes to make sure everyone has access to them // otherwise, the averages are already loaded into the vector if(dataset2 == 0) { double average = GetAverage(distanceMatrix); averages[i] = average; } // accumulate; this is the part inside the 'exp' in the kernel function MulVal(distanceMatrix, distanceMatrix, weight); DivVal(distanceMatrix, distanceMatrix, -averages[i]); if(accumulator == 0) Set(accumulator, distanceMatrix); else Add(accumulator, accumulator, distanceMatrix); delete distanceMatrix; totalweight += weight; } ILOG_INFO_ONCE("finalising..."); // only thing to do is weigh and exp DivVal(accumulator, accumulator, totalweight); Exp(accumulator, accumulator); if(dataset2 == 0) { if(Link::Mpi::MyId() == 0) { String filename = resultname + ".averages.raw"; filename = dataset->GetFilePathPrecomputedKernels(filename, "", true, false); WriteAverages(filename, db, averages); } } // write the matrix if (dataset2) { resultname = dataset2->GetFilePathPrecomputedKernels (resultname, dataset->GetSetName(), true, false); } else { resultname = dataset->GetFilePathPrecomputedKernels (resultname, "", true, false); } WriteResult(resultname, db, accumulator); return 0; } Here is the call graph for this function:

---