00001 #ifndef Impala_Core_Database_RawDataSet_h
00002 #define Impala_Core_Database_RawDataSet_h
00003
00004 #include "Basis/FileName.h"
00005 #include "Util/IOBufferFile.h"
00006 #include "Util/Database.h"
00007 #include "Util/StringParser.h"
00008 #include "Util/SimpleMap.h"
00009 #include "Basis/Timer.h"
00010 #include "Basis/QuidObj.h"
00011 #include "Basis/Quids.h"
00012 #include "Core/Feature/FeatureDefinition.h"
00013 #include "Link/DiskImage/DiskImageFuncs.h"
00014 #include "Persistency/Locator.h"
00015
00016 namespace Impala
00017 {
00018 namespace Core
00019 {
00020 namespace Database
00021 {
00022
00023
00026 class RawDataSet
00027 {
00028 public:
00029
00030 RawDataSet(String setName)
00031 {
00032 Init(setName);
00033 }
00034
00035 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00036 RawDataSet(Util::Database* db, String setName, String setFileName)
00037 {
00038 Init(db, setName, setFileName);
00039 }
00040
00041 RawDataSet(Util::Database* db, String setName, String setFileName,
00042 String nameExt)
00043 {
00044 Init(db, setName, setFileName);
00045 GenerateFileNames(nameExt);
00046 }
00047 #endif // REPOSITORY_USED
00048
00049 String
00050 GetSetName() const
00051 {
00052 return mSetName;
00053 }
00054
00055 String
00056 GetSetNameBase() const
00057 {
00058 return mSetNameBase;
00059 }
00060
00061 void
00062 SetProtocolAndHost(CString protocolAndHost)
00063 {
00064 mProtocolAndHost = protocolAndHost;
00065 }
00066
00067 Persistency::Locator
00068 GetLocator() const
00069 {
00070 if (mProtocolAndHost == "not_set")
00071 {
00072 CmdOptions& options = CmdOptions::GetInstance();
00073 ILOG_WARN("[GetLocator] protocol may not be what you expect" <<
00074 ", set = " << mSetName);
00075 return Persistency::Locator(mSetName, options);
00076 }
00077 return Persistency::Locator(mProtocolAndHost, mSetName);
00078 }
00079
00080 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00081 Util::Database*
00082 GetDatabase()
00083 {
00084 return mDb;
00085 }
00086
00087 void
00088 SetDatabase(Util::Database* db)
00089 {
00090 mDb = db;
00091 }
00092 #endif // REPOSITORY_USED
00093
00094 virtual int
00095 GetSetId()
00096 {
00097 return -1;
00098 }
00099
00100
00101 int
00102 GetQuidClass() const
00103 {
00104 return mQuidClass;
00105 }
00106
00107 int
00108 NrDirs() const
00109 {
00110 return mContainer.size();
00111 }
00112
00113
00114
00115 String
00116 GetSection(int elemId) const
00117 {
00118 String s = FileNamePath2(mContainer[elemId]);
00119 if (s.empty())
00120 return ".";
00121 return s;
00122 }
00123
00124 String
00125 GetSectionOfFile(int fileId) const
00126 {
00127 return GetSection(mFileKey[fileId]);
00128 }
00129
00130
00131
00132 String
00133 GetDir(int elemId) const
00134 {
00135 String s = FileNameTail(mContainer[elemId]);
00136 if (s.empty())
00137 return ".";
00138 return s;
00139 }
00140
00141 int
00142 GetDirId(String dir) const
00143 {
00144 for (int i=0 ; i<NrDirs() ; i++)
00145 if (GetDir(i) == dir)
00146 return i;
00147 return -1;
00148 }
00149
00150 int
00151 GetDirId(String section, String dir) const
00152 {
00153 for (int i=0 ; i<NrDirs() ; i++)
00154 if ((GetSection(i) == section) && (GetDir(i) == dir))
00155 return i;
00156 return -1;
00157 }
00158
00159 int
00160 GetDirIdOfFile(int fileId) const
00161 {
00162 return mFileKey[fileId];
00163 }
00164
00165 String
00166 GetDirOfFile(int fileId) const
00167 {
00168 return GetDir(mFileKey[fileId]);
00169 }
00170
00171
00172
00173 int
00174 GetFirstFileId(int elemId) const
00175 {
00176 if ((elemId < 0) || (elemId >= mFirstFileId.size()))
00177 {
00178 ILOG_ERROR("GetFirstFileId: id " << elemId << " is out of range");
00179 return -1;
00180 }
00181 return mFirstFileId[elemId];
00182 }
00183
00184 int
00185 GetNrFiles(int elemId) const
00186 {
00187 if (!HasFiles())
00188 return 0;
00189 return mNrFiles[elemId];
00190 }
00191
00192 bool
00193 HasFiles() const
00194 {
00195 return mHasFiles;
00196 }
00197
00198 int
00199 NrFiles() const
00200 {
00201 if (!HasFiles())
00202 return NrDirs();
00203 return mFile.size();
00204 }
00205
00206
00207
00208 String
00209 GetFile(int i) const
00210 {
00211 return mFile[i];
00212 }
00213
00214
00215
00216 String
00217 GetFileOrDir(int i, String extension) const
00218 {
00219 if (mFile[i] != String(""))
00220 return mFile[i];
00221 return GetDirOfFile(i) + extension;
00222 }
00223
00224
00225 String
00226 GetFileBase(int i) const
00227 {
00228 if (mFile[i] != String(""))
00229 return FileNameBase(mFile[i]);
00230 return GetDirOfFile(i);
00231 }
00232
00233
00234 int
00235 GetFileId(int idx) const
00236 {
00237 if ((idx >= 0) && (idx < mFileId.size()))
00238 return mFileId[idx];
00239 ILOG_ERROR("No fileId with index " << idx);
00240 return -1;
00241 }
00242
00243 int
00244 GetFileId(String file) const
00245 {
00246 for (int i=0 ; i<mFile.size() ; i++)
00247 if (mFile[i] == file)
00248 return i;
00249 return -1;
00250 }
00251
00252 int
00253 GetFileId(int dirId, String file) const
00254 {
00255 for (int i=0 ; i<mFile.size() ; i++)
00256 if ((mFileKey[i] == dirId) && (mFile[i] == file))
00257 return i;
00258 return -1;
00259 }
00260
00261 int
00262 GetFileId(String section, String dir, String file) const
00263 {
00264 if ((section == String("")) && (dir == String("")))
00265 return GetFileId(file);
00266 int dirId = GetDirId(section, dir);
00267 return GetFileId(dirId, file);
00268 }
00269
00270 int
00271 GetFileIdOfQuid(Quid q)
00272 {
00273 return (IsImageSet()) ? QuidId(q) : QuidObject(q);
00274 }
00275
00276
00277 void
00278 GenerateFileNames(String extension)
00279 {
00280 for (int i=0 ; i<mFile.size() ; i++)
00281 if (mFile[i] == String(""))
00282 mFile[i] = GetDirOfFile(i) + extension;
00283 mHasFiles = true;
00284 }
00285
00286 String
00287 GetAsPath(int fileId)
00288 {
00289 const int dirId = GetDirIdOfFile(fileId);
00290 String dir = GetDir(dirId);
00291
00292 if (Link::DiskImage::DiskImageUsed() && dir.find("//#") != String::npos)
00293 {
00294
00295 dir = StringReplaceAll(dir, "0x", "0x300x78", false);
00296
00297 dir = StringReplaceAll(dir, "//#", "/0x2f#", false);
00298 if (dir.find("/") == 0)
00299 dir = dir.substr(1);
00300 }
00301
00302 return GetSection(dirId) + "/" + dir + "/" + GetFile(fileId);
00303 }
00304
00305
00306
00307 bool
00308 IsImageSet() const
00309 {
00310 return GetQuidClass() == QUID_CLASS_IMAGE;
00311 }
00312
00313 int
00314 GetNrContainers() const
00315 {
00316 if (IsImageSet())
00317 return NrDirs();
00318 return NrFiles();
00319 }
00320
00321 String
00322 GetContainer(int nr)
00323 {
00324 if (IsImageSet())
00325 return GetContainerDir(nr);
00326 return GetContainerFile(nr);
00327 }
00328
00329 int
00330 GetContainerNrOfQuid(Quid q)
00331 {
00332 int fileId = (IsImageSet()) ? QuidId(q) : QuidObject(q);
00333 return (IsImageSet()) ? GetDirIdOfFile(fileId) : fileId;
00334 }
00335
00336 String
00337 GetContainerDir(int elemId)
00338 {
00339 return GetSection(elemId) + "/" + GetDir(elemId);
00340 }
00341
00342 String
00343 GetContainerDirOfFile(int fileId)
00344 {
00345 return GetSectionOfFile(fileId) + "/" + GetDirOfFile(fileId);
00346 }
00347
00348 String
00349 GetContainerFile(int fileId)
00350 {
00351 return GetAsPath(fileId);
00352 }
00353
00354 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00355
00356
00357 String
00358 MakeDir(String dir)
00359 {
00360 return mDb->MakeDir(dir);
00361 }
00362
00363 String
00364 MakeDir(String dir1, String dir2)
00365 {
00366 return mDb->MakeDir(dir1, dir2);
00367 }
00368
00369 String
00370 MakeDir(String dir1, String dir2, String dir3)
00371 {
00372 return mDb->MakeDir(dir1, dir2, dir3);
00373 }
00374
00375 String
00376 MakeDir(String dir1, String dir2, String dir3, String dir4)
00377 {
00378 return mDb->MakeDir(dir1, dir2, dir3, dir4);
00379 }
00380
00381 String
00382 MakeDir(String dir1, String dir2, String dir3, String dir4, String dir5)
00383 {
00384 return mDb->MakeDir(dir1, dir2, dir3, dir4, dir5);
00385 }
00386
00387
00388
00389 String
00390 GetFilePath(String dir, String file, bool toWrite, bool silent)
00391 {
00392 return mDb->GetFilePath(dir, file, toWrite, silent);
00393 }
00394
00395 String
00396 GetFilePath(String dir1, String dir2, String file, bool toWrite, bool silent)
00397 {
00398 return mDb->GetFilePath(dir1, dir2, file, toWrite, silent);
00399 }
00400
00401 String
00402 GetFilePath(String dir1, String dir2, String dir3, String file,
00403 bool toWrite, bool silent)
00404 {
00405 return mDb->GetFilePath(dir1, dir2, dir3, file, toWrite, silent);
00406 }
00407
00408 String
00409 GetFilePath(String dir1, String dir2, String dir3, String dir4, String file,
00410 bool toWrite, bool silent)
00411 {
00412 return mDb->GetFilePath(dir1, dir2, dir3, dir4, file, toWrite, silent);
00413 }
00414
00415 String
00416 GetFilePath(String dir1, String dir2, String dir3, String dir4, String dir5,
00417 String file, bool toWrite, bool silent)
00418 {
00419 return mDb->GetFilePath(dir1, dir2, dir3, dir4, dir5, file, toWrite,
00420 silent);
00421 }
00422
00423
00424
00425 String
00426 GetFilePathVideoData(String setName, bool toWrite, bool silent)
00427 {
00428 String dir = "VideoData";
00429 if (toWrite)
00430 MakeDir(dir);
00431 return GetFilePath(dir, setName, toWrite, silent);
00432 }
00433
00434 String
00435 GetFilePathVideoData(int nameId, bool toWrite, bool silent)
00436 {
00437 String dir = "VideoData/" + GetSectionOfFile(nameId) + "/"
00438 + GetDirOfFile(nameId);
00439 if (toWrite)
00440 MakeDir(dir);
00441 return GetFilePath(dir, GetFile(nameId), toWrite, silent);
00442 }
00443
00444 String
00445 GetFilePathImageData(String setName, bool toWrite, bool silent)
00446 {
00447 String dir = "ImageData";
00448 if (toWrite)
00449 MakeDir(dir);
00450 return GetFilePath(dir, setName, toWrite, silent);
00451 }
00452
00453 String
00454 GetFilePathImageData(int nameId, bool toWrite, bool silent)
00455 {
00456 String dir = "ImageData/" + GetSectionOfFile(nameId) + "/"
00457 + GetDirOfFile(nameId);
00458 if (toWrite)
00459 MakeDir(dir);
00460 return GetFilePath(dir, GetFile(nameId), toWrite, silent);
00461 }
00462
00463 String
00464 GetFilePathImageArchive(String archiveName, bool toWrite, bool silent)
00465 {
00466 String dir = "ImageArchive";
00467 if (toWrite)
00468 MakeDir(dir);
00469 return GetFilePath(dir, archiveName, toWrite, silent);
00470 }
00471
00472 String
00473 GetFilePathImageArchive(int fileId, String archiveName, bool toWrite,
00474 bool silent)
00475 {
00476 String dir = "ImageArchive/" + GetSectionOfFile(fileId) + "/"
00477 + GetDirOfFile(fileId);
00478 if (toWrite)
00479 MakeDir(dir);
00480 return GetFilePath(dir, archiveName, toWrite, silent);
00481 }
00482
00483 String
00484 GetFilePathFrames(int fileId, String name, bool toWrite, bool silent)
00485 {
00486 String dir = "Frames/" + GetAsPath(fileId);
00487 if (toWrite)
00488 MakeDir(dir);
00489 return GetFilePath(dir, name, toWrite, silent);
00490 }
00491
00492 String
00493 GetFilePathFrames(int fileId, int frameNr, String suffix, bool toWrite,
00494 bool silent)
00495 {
00496 String dir = "Frames/" + GetAsPath(fileId);
00497 if (toWrite)
00498 MakeDir(dir);
00499 char frameName[200];
00500 sprintf(frameName, "frame_%06d", frameNr);
00501 return GetFilePath(dir, String(frameName)+suffix, toWrite, silent);
00502 }
00503
00504 String
00505 GetFilePathVideoIndex(String subDir, String file, bool toWrite, bool silent)
00506 {
00507 String dir("VideoIndex");
00508 if (!subDir.empty())
00509 dir += "/" + subDir;
00510 if (toWrite)
00511 MakeDir(dir);
00512 return GetFilePath(dir, file, toWrite, silent);
00513 }
00514
00515
00516
00517
00518
00519
00520
00521
00522
00523
00524
00525
00526
00531 String
00532 GetFilePathMetaData(int fileId, String metaName, bool useDbFile,
00533 String suffix, bool toWrite, bool silent)
00534 {
00535 String dir = "MetaData/" + GetAsPath(fileId) + "/" + metaName;
00536 if (toWrite)
00537 MakeDir(dir);
00538 String fName = ((useDbFile) ? GetFileBase(fileId) : "") + suffix;
00539 return mDb->GetFilePath(dir, fName, toWrite, silent);
00540 }
00541
00542 String
00543 GetFilePathMpeg7Shot(int fileId, bool toWrite, bool silent)
00544 {
00545 String dir = "MetaData/shots/" + GetAsPath(fileId);
00546 if (toWrite)
00547 MakeDir(dir);
00548 return GetFilePath(dir, GetFileBase(fileId) + ".xml", toWrite, silent);
00549 }
00550
00551 String
00552 GetFilePathMpeg7Anno(int fileId, String conceptSet, String xmlName,
00553 bool toWrite, bool silent)
00554 {
00555 String dir = "MetaData/annotations/" + conceptSet + "/"
00556 + GetAsPath(fileId);
00557 if (toWrite)
00558 MakeDir(dir);
00559 return GetFilePath(dir, xmlName, toWrite, silent);
00560 }
00561
00562 String
00563 GetFilePathMpeg7Feature(int fileId, String xmlName, bool toWrite,
00564 bool silent)
00565 {
00566 String dir = "MetaData/features/" + GetAsPath(fileId);
00567 if (toWrite)
00568 MakeDir(dir);
00569 return GetFilePath(dir, xmlName, toWrite, silent);
00570 }
00571
00572 String
00573 GetFilePathMpeg7Concept(int fileId, String conceptSet, String xmlName,
00574 bool toWrite, bool silent)
00575 {
00576 String dir = "MetaData/similarities/" + conceptSet + "/"
00577 + GetAsPath(fileId);
00578 if (toWrite)
00579 MakeDir(dir);
00580 return GetFilePath(dir, xmlName, toWrite, silent);
00581 }
00582
00583 typedef Feature::FeatureDefinition FeatureDefinition;
00584
00585 String
00586 GetFilePathPrototype(String protoDataFile, String featureName,
00587 String fileName, bool toWrite, bool silent)
00588 {
00589 String dir = "Codebooks/" + protoDataFile + "/" + featureName;
00590 if (toWrite)
00591 MakeDir(dir);
00592 return GetFilePath(dir, fileName, toWrite, silent);
00593 }
00594
00595
00596 String
00597 GetFilePathFeatureData(String walkType, FeatureDefinition def,
00598 int fileOrDirId, bool partial, int startFrameOrFile,
00599 bool toWrite, bool silent)
00600 {
00601
00602 if (walkType == "Image")
00603 walkType = ".";
00604
00605 String dir = PathJoin("FeatureData", walkType, def.GetName());
00606 if (mQuidClass == QUID_CLASS_IMAGE)
00607 {
00608 dir = PathJoin(dir, GetSection(fileOrDirId), GetDir(fileOrDirId));
00609 }
00610 else
00611 {
00612 dir = PathJoin(dir, GetAsPath(fileOrDirId));
00613 }
00614 if (partial)
00615 dir = PathJoin(dir, "part_" + MakeString(startFrameOrFile));
00616 if (toWrite)
00617 MakeDir(dir);
00618 return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
00619 }
00620
00621
00622 String
00623 GetFilePathFeatureData(FeatureDefinition def, int dirId, bool partial,
00624 int startFile, bool toWrite, bool silent)
00625 {
00626 String dir = "FeatureData/" + def.GetName() + "/" + GetSection(dirId)
00627 + "/" + GetDir(dirId) + "/";
00628 if (partial)
00629 dir += "/part_" + MakeString(startFile);
00630 if (toWrite)
00631 MakeDir(dir);
00632 return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
00633 }
00634
00635 String
00636 GetFilePathFeatureIndex(FeatureDefinition def, String subSet, bool toWrite,
00637 bool silent)
00638 {
00639 String dir = "FeatureIndex/" + def.GetName();
00640 if (toWrite)
00641 MakeDir(dir);
00642 return GetFilePath(dir, def.AsString() + subSet + ".tab", toWrite,
00643 silent);
00644 }
00645
00646
00647 String
00648 GetFilePathPrecomputedKernels(String filename, String otherSet,
00649 bool toWrite, bool silent)
00650 {
00651 String dir = "PrecomputedKernels";
00652 if(otherSet != "")
00653 dir += "/" + otherSet;
00654 if (toWrite)
00655 MakeDir(dir);
00656 return GetFilePath(dir, filename, toWrite, silent);
00657 }
00658
00659
00660 String
00661 GetFilePathRandomForest(FeatureDefinition def, bool toWrite, bool silent)
00662 {
00663 String dir = "Codebooks/Forest/";
00664 if (toWrite)
00665 MakeDir(dir);
00666 return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
00667 }
00668
00669
00670 String
00671 GetFilePathCodebook(FeatureDefinition def, bool toWrite, bool silent)
00672 {
00673 String dir = "Codebooks/";
00674 if (toWrite)
00675 MakeDir(dir);
00676 return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
00677 }
00678
00679
00680 String
00681 GetFilePathSimilarityData(String walkType, String conceptSet, String model,
00682 String featureString, int fileOrDirId,
00683 String fileName, bool toWrite, bool silent)
00684 {
00685
00686 if(walkType == "Image")
00687 walkType = ".";
00688 String dir = PathJoin("SimilarityData", walkType, conceptSet, model,
00689 featureString);
00690 if(mQuidClass == QUID_CLASS_IMAGE)
00691 {
00692 dir = PathJoin(dir, GetSection(fileOrDirId), GetDir(fileOrDirId));
00693 }
00694 else
00695 {
00696 dir = PathJoin(dir, GetAsPath(fileOrDirId));
00697 }
00698 if (toWrite)
00699 MakeDir(dir);
00700 return GetFilePath(dir, fileName, toWrite, silent);
00701 }
00702
00703 String
00704 GetFilePathSimilarityIndex(String conceptSet, String model,
00705 String featureString, String fileName,
00706 bool toWrite, bool silent)
00707 {
00708 String dir = "SimilarityIndex/" + conceptSet + "/" + model + "/"
00709 + featureString;
00710 if (toWrite)
00711 MakeDir(dir);
00712 return GetFilePath(dir, fileName, toWrite, silent);
00713 }
00714
00715 String
00716 GetFilePathAnnotation(String annoFile, bool toWrite, bool silent)
00717 {
00718 String dir = "Annotations";
00719 if (toWrite)
00720 MakeDir(dir);
00721 return GetFilePath(dir, annoFile, toWrite, silent);
00722 }
00723
00724 String
00725 GetFilePathAnnotation(int quidClass, String conceptSet, String annoFile,
00726 bool toWrite, bool silent)
00727 {
00728 String dir = "Annotations/" + QuidClassToString(quidClass) + "/"
00729 + conceptSet;
00730 if (toWrite)
00731 MakeDir(dir);
00732 return GetFilePath(dir, annoFile, toWrite, silent);
00733 }
00734
00735 String
00736 GetFilePathGroundTruth(String truthFile, bool toWrite, bool silent)
00737 {
00738 String dir = "Annotations/groundtruth";
00739 if (toWrite)
00740 MakeDir(dir);
00741 return GetFilePath(dir, truthFile, toWrite, silent);
00742 }
00743
00744 String
00745 GetFilePathConceptModel(String conceptSet, String modelType,
00746 FeatureDefinition def, String modelFile,
00747 bool toWrite, bool silent)
00748 {
00749 String dir = "ConceptModels/" + conceptSet + "/" + modelType + "/"
00750 + def.AsString();
00751 if (toWrite)
00752 MakeDir(dir);
00753 return GetFilePath(dir, modelFile, toWrite, silent);
00754 }
00755
00756 String
00757 GetFilePathReference(int fileId, String suffix, bool toWrite, bool silent)
00758 {
00759 String dir = "Reference/" + GetAsPath(fileId);
00760 if (toWrite)
00761 MakeDir(dir);
00762 return GetFilePath(dir, GetFileBase(fileId) + suffix, toWrite, silent);
00763 }
00764
00765 String
00766 GetFilePathKernelData(String walkType, String referenceSet,
00767 String kernelName, int fileOrDirId,
00768 bool toWrite, bool silent)
00769 {
00770 String dir = PathJoin("KernelData", walkType, referenceSet, kernelName);
00771 if(mQuidClass == QUID_CLASS_IMAGE)
00772 {
00773 dir = PathJoin(dir, GetSection(fileOrDirId), GetDir(fileOrDirId));
00774 }
00775 else
00776 {
00777 dir = PathJoin(dir, GetAsPath(fileOrDirId));
00778 }
00779 if (toWrite)
00780 MakeDir(dir);
00781 return GetFilePath(dir, kernelName + ".tab", toWrite, silent);
00782 }
00783 #endif // REPOSITORY_USED
00784
00785
00786
00787
00788 void
00789 AddFile(String file, String container, int id = -1)
00790 {
00791 int key = -1;
00792 mContainerMap.Get(container, key);
00793
00794 if ((key != -1) && (mLastKeyForWarning != key) &&
00795 (mLastKeyForWarning != -1))
00796 {
00797
00798 ILOG_ERROR("Files are not sorted by directory. " << mLastId <<
00799 " container = " << container);
00800 return;
00801 }
00802
00803 if (id == -1)
00804 {
00805 id = mLastId;
00806 }
00807 else
00808 {
00809 for (int i=0 ; i<mFirstFileId.size() ; i++)
00810 if (mFirstFileId[i] == id)
00811 ILOG_ERROR("Id already in use: " << id << " file: " << file << " container: " << container);
00812 }
00813
00814 if (key == -1)
00815 {
00816 mContainer.push_back(container);
00817 mContainerMap.Add(container, mContainer.size()-1);
00818 mFirstFileId.push_back(id);
00819 mNrFiles.push_back(0);
00820 key = NrDirs() - 1;
00821 }
00822 mFileId.push_back(id);
00823 mFile.push_back(file);
00824 mLastKeyForWarning = key;
00825 mFileKey.push_back(key);
00826 mNrFiles[key] = mNrFiles[key] + 1;
00827 mLastId = id + 1;
00828 if (!file.empty())
00829 mHasFiles = true;
00830 }
00831
00832 void
00833 AddFile(String path, int id)
00834 {
00835 String container(".");
00836 String file("");
00837 ExtractFileAndContainer(path, file, container);
00838 AddFile(file, container, id);
00839 }
00840
00841 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00842 void
00843 Save()
00844 {
00845 Save(mSetName);
00846 }
00847
00848 void
00849 Save(String fileName)
00850 {
00851 Util::IOBuffer* ioBuf = mDb->GetIOBuffer(fileName, false, false, "tmp");
00852 if (! ioBuf)
00853 return;
00854 for (int i=0 ; i<NrFiles() ; i++)
00855 ioBuf->IOBuffer::Puts("\"" + GetAsPath(i) + "\"");
00856 delete ioBuf;
00857 }
00858
00859 void
00860 SaveBinary()
00861 {
00862 if (mSetFileName == String(""))
00863 return;
00864 String fName = mSetFileName + ".bin";
00865 ILOG_DEBUG("saving " << fName);
00866
00867 }
00868 #endif // REPOSITORY_USED
00869
00870 void
00871 Dump(int startDir, int endDir, int startFile, int endFile)
00872 {
00873 if (endDir == -1)
00874 endDir = NrDirs();
00875 std::cout << "Dumping dirs from " << startDir << " to " << endDir
00876 << " (of " << NrDirs() << ")" << std::endl;
00877 for (int i=startDir ; i<endDir ; i++)
00878 std::cout << "dir=" << i << ": " << GetSection(i) << " "
00879 << GetDir(i) << ": nr files = " << GetNrFiles(i)
00880 << ", first file = " << GetFirstFileId(i) << std::endl;
00881 if (!mHasFiles)
00882 {
00883 std::cout << "Does not have files" << std::endl;
00884 return;
00885 }
00886 if (startFile == -1)
00887 {
00888 std::cout << "Skipping files" << std::endl;
00889 return;
00890 }
00891 if (endFile == -1)
00892 endFile = NrFiles();
00893 std::cout << "Dumping files from " << startFile << " to " << endFile
00894 << " (of " << NrFiles() << ")" << std::endl;
00895 for (int j=startFile ; j<endFile ; j++)
00896 {
00897 std::cout << GetFileId(j) << " " << GetSectionOfFile(j) << " "
00898 << GetDirOfFile(j) << " " << GetFile(j) << std::endl;
00899 }
00900 }
00901
00902 virtual String
00903 QuidToString(Quid quid, bool elaborate)
00904 {
00905 QuidObj qObj(quid);
00906 return qObj.ToString();
00907 }
00908
00909
00910 protected:
00911
00912 void
00913 Init(String setName)
00914 {
00915 mLastKeyForWarning = -1;
00916 mSetName = setName;
00917 mSetNameBase = FileNameBase(mSetName);
00918 mSetFileName = "";
00919 mProtocolAndHost = "not_set";
00920 mHasFiles = false;
00921 mLastId = 0;
00922 }
00923
00924 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00925 void
00926 Init(Util::Database* db, String setName, String setFileName)
00927 {
00928 mLastKeyForWarning = -1;
00929 mDb = db;
00930 mSetName = setName;
00931 mSetNameBase = FileNameBase(mSetName);
00932 mSetFileName = setFileName;
00933 mProtocolAndHost = "not_set";
00934 mHasFiles = false;
00935 mLastId = 0;
00936 if (mSetFileName == String(""))
00937 return;
00938 mQuidClass = QUID_CLASS_UNKNOWN;
00939 ILOG_DEBUG("opening data file " << mSetFileName);
00940 Timer timer(1);
00941 Util::IOBuffer* f = mDb->GetIOBuffer(mSetFileName, true, true, "");
00942 if (f->Valid())
00943 {
00944 while (f->Available())
00945 {
00946 String line = f->ReadLine();
00947 if (line[0] && (line[0] != '#'))
00948 {
00949 int id = -1;
00950 String name("");
00951 Util::StringParser p(line);
00952 while (!p.TheEnd())
00953 {
00954 if (p.At("<id="))
00955 {
00956 p.Eat('=');
00957 id = p.GetInt('>', true);
00958 p.Eat('>');
00959 }
00960 else
00961 {
00962 String s = p.GetString2(false);
00963 name = StringReplaceAll(s, "\\", "/");
00964 }
00965 }
00966 AddFile(name, id);
00967 }
00968 }
00969 }
00970 delete f;
00971 ILOG_DEBUG("done read file in " << timer.SplitTime());
00972 }
00973 #endif // REPOSITORY_USED
00974
00975 void
00976 ExtractFileSectionDir(CString src, String& file, String& section, String& dir)
00977 {
00978 StringList sl(src, '/', true);
00979
00980
00981
00982
00983
00984
00985
00986
00987
00988
00989
00990
00991
00992
00993
00994
00995
00996
00997
00998
00999
01000
01001
01002
01003
01004
01005
01006
01007
01008
01009
01010
01011
01012
01013
01014 StringList::reverse_iterator it = sl.rbegin();
01015 file = *it++;
01016 if (it != sl.rend())
01017 dir = *it++;
01018 if (it != sl.rend())
01019 section = *it++;
01020 while (it != sl.rend())
01021 {
01022 String subDir = *it++;
01023 if (subDir != ".")
01024 section = subDir + "/" + section;
01025 }
01026 }
01027
01028 void
01029 ExtractFileAndContainer(CString src, String& file, String& container)
01030 {
01031 StringList sl(src, '/', true);
01032 StringList::reverse_iterator it = sl.rbegin();
01033 file = *it++;
01034 if (it != sl.rend())
01035 container = *it++;
01036 while (it != sl.rend())
01037 {
01038 String subDir = *it++;
01039 if (subDir != ".")
01040 container = subDir + "/" + container;
01041 }
01042 }
01043
01044 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
01045 Util::Database* mDb;
01046 #endif // REPOSITORY_USED
01047 String mSetName;
01048 String mSetNameBase;
01049 String mSetFileName;
01050 String mProtocolAndHost;
01051 bool mHasFiles;
01052 int mLastId;
01053 int mLastKeyForWarning;
01054 std::vector<int> mFileId;
01055 std::vector<String> mFile;
01056 std::vector<int> mFileKey;
01057
01058 std::vector<String> mContainer;
01059 Util::SimpleMap<String,int> mContainerMap;
01060 std::vector<int> mFirstFileId;
01061 std::vector<int> mNrFiles;
01062
01063 int mQuidClass;
01064
01065 ILOG_VAR_DEC;
01066 };
01067
01068 ILOG_VAR_INIT(RawDataSet, Impala.Core.Database);
01069
01070 }
01071 }
01072 }
01073
01074 #endif