|| || || || || ||

---

Precompute

int Impala::Application::Precompute::Precompute ( CmdOptions &  options  )  Definition at line 218 of file mainPrecomputeKernelMatrix.cpp. References Impala::Core::Array::Add(), Impala::Util::TimeStats::AddGroup(), Impala::Util::TimeStats::AddGroupsFromSub(), Impala::Util::TimeStats::AsString(), CheckFeatures(), ComputeMatrix(), Impala::Core::Matrix::VirtualMatrixFactory::ConstructMemory(), Impala::Core::Table::Copy(), Impala::Core::Array::DivVal(), Impala::Core::Array::Exp(), Impala::CmdOptions::GetArg(), GetAverage(), Impala::CmdOptions::GetBool(), Impala::Core::Matrix::VirtualMatrixFactory::GetInstance(), Impala::CmdOptions::GetInt(), Impala::CmdOptions::GetString(), Impala::Process::MemoryInfo::GetUsageString(), ILOG_INFO, ILOG_VAR, Impala::Core::Database::MakeRawDataSet(), Impala::Util::TimeStats::MeasureFirst(), Impala::Util::TimeStats::MeasureFromSub(), Impala::Util::TimeStats::MeasureLast(), Impala::Util::TimeStats::MeasureNext(), Impala::Core::Array::MulVal(), OpenFeatureTable(), Impala::Core::Array::Set(), and Impala::Core::Table::Table::Size(). Referenced by MainPrecompute(). { ILOG_VAR(Impala.Application.Precompute.Precompute); RawDataSet* thisSet = Core::Database::MakeRawDataSet(options.GetArg(0)); if (!thisSet) { ILOG_INFO_HEADNODE("Failed to open thisSet " << options.GetArg(0)); return 1; } String develSetName = options.GetArg(1); ILOG_INFO_HEADNODE("DevelSetName = [" << develSetName << "]"); RawDataSet* develSet = 0; if (!develSetName.empty()) { develSet = Core::Database::MakeRawDataSet(develSetName); if (!develSet) { ILOG_INFO_HEADNODE("Failed to open develSet " << develSetName); return 1; } } String model = options.GetArg(2); String kernel = options.GetArg(3); String indexCat = options.GetString("featureIndexCat"); KernelMatrixLocator daLoc(thisSet->GetLocator(), true, "", develSetName, model, kernel, ""); daLoc.SetFeatureIndexCat(indexCat); if (!options.GetBool("override") && DistributedAccessRepository().Exists(daLoc)) { ILOG_INFO("Kernel already exists."); return 0; } int startFeature = options.GetInt("startFeature"); DistributedAccess* da = 0; if (startFeature == 0) { da = new DistributedAccess(); if (!da->Valid(true, false)) return 1; ILOG_INFO_HEADNODE("Using a " << da->GetRowPartCount() << "x" << da->GetColumnPartCount() << "grid"); if (!CheckFeatures(options, thisSet, da)) return 1; } else { daLoc.SetIncrement(startFeature); daLoc.SetStartNode(0); daLoc.SetNodeCount(Link::Mpi::NrProcs()); da = DistributedAccessRepository().Get(daLoc); } if (develSet) { KernelMatrixLocator devLoc(develSet->GetLocator(), true, "", "", model, kernel, ""); devLoc.SetFeatureIndexCat(indexCat); devLoc.SetDoParts(0); devLoc.SetStartNode(0); devLoc.SetNodeCount(Link::Mpi::NrProcs()); DistributedAccess* devDA = DistributedAccessRepository().Get(devLoc); for (int i = 0; i < devDA->GetNrFeatures(); i++) { double average = devDA->GetAverage(i); ILOG_INFO_HEADNODE("average " << i << " " << average); da->SetAverage(i, average); } // Could do some more checking here... delete devDA; } else { if (model == "chi2") da->SetHasOwnAverages(true); } String distanceFunction = options.GetString("distanceFunction"); int numberFeatures = options.GetInt("numberFeatures"); if (numberFeatures == -1) numberFeatures = da->GetNrFeatures() - startFeature; if (startFeature + numberFeatures >= da->GetNrFeatures()) numberFeatures = da->GetNrFeatures() - startFeature; Matrix::Mat* accumulator = 0; if (startFeature != 0) accumulator = da->StealPart(); double totalweight = 0; for (int i=0 ; i<startFeature ; i++) totalweight += da->GetWeight(i); Util::TimeStats statsCompute; statsCompute.AddGroup("read features"); statsCompute.AddGroup("compute"); Util::TimeStats statsOverall; statsOverall.AddGroupsFromSub(&statsCompute); statsOverall.AddGroup("final exp"); statsOverall.AddGroup("write"); statsOverall.MeasureFirst(); for (int i=startFeature ; i<startFeature+numberFeatures ; i++) { statsCompute.MeasureFirst(); // open feature tables double weight = da->GetWeight(i); Feature::FeatureDefinition fDef(da->GetFeature(i)); Feature::FeatureTable* f2 = OpenFeatureTable(fDef, indexCat, thisSet); Feature::FeatureTable* f1 = f2; if (develSet) f1 = OpenFeatureTable(fDef, indexCat, develSet); if ((Link::Mpi::MyId() == 0) && (i == 0)) { Table::Copy(da->GetColumnQuids(), f1); da->SetColumns(f1->Size()); Table::Copy(da->GetRowQuids(), f2); da->SetRows(f2->Size()); } statsCompute.MeasureNext(); // compute kernel distances between features Matrix::Mat *distanceMatrix = ComputeMatrix(da, f1, f2, distanceFunction); // f1 & f2 are deleted by ComputeMatrix // if this is the 'learn' kernel matrix we compute the averages and // communicate between nodes to make sure everyone has access to them // otherwise, the averages are already loaded into the vector // Note : hik kernel does not use averages if (da->GetHasOwnAverages()) { if (distanceFunction == "dot") { da->SetAverage(i, 0.0); } else { double average = GetAverage(distanceMatrix); da->SetAverage(i, average); } } // accumulate; this is the part inside the 'exp' in the kernel function MulVal(distanceMatrix, distanceMatrix, weight); if (distanceFunction == "chi2") { DivVal(distanceMatrix, distanceMatrix, -da->GetAverage(i)); } if (accumulator == 0) Set(accumulator, distanceMatrix); else Add(accumulator, accumulator, distanceMatrix); delete distanceMatrix; totalweight += weight; statsCompute.MeasureLast(); ILOG_INFO_HEADNODE("Compute " << statsCompute.AsString()); ILOG_INFO_HEADNODE("Memory " << Process::MemoryInfo::GetUsageString()); } statsOverall.MeasureFromSub(&statsCompute); ILOG_INFO_HEADNODE("finalising..."); if (startFeature + numberFeatures == da->GetNrFeatures()) { // only thing to do is weigh and exp DivVal(accumulator, accumulator, totalweight); if (distanceFunction == "chi2") { Exp(accumulator, accumulator); } daLoc.SetIncrement(0); daLoc.SetWriteReal32(true); } else { daLoc.SetIncrement(startFeature + numberFeatures); } statsOverall.MeasureNext(); typedef Matrix::VirtualMatrixFactory VirtualMatrixFactory; VirtualMatrixFactory& vmf = VirtualMatrixFactory::GetInstance(); Matrix::VirtualMatrix* vm = vmf.ConstructMemory(accumulator); da->AddPart(vm); DistributedAccessRepository().Add(daLoc, da); statsOverall.MeasureLast(); ILOG_INFO_HEADNODE("Overall " << statsOverall.AsString()); delete da; return 0; } Here is the call graph for this function:

---