|| || || || || ||

---

RawDataSet.h

Go to the documentation of this file.

#ifndef Impala_Core_Database_RawDataSet_h
#define Impala_Core_Database_RawDataSet_h

#include "Basis/FileName.h"
#include "Util/IOBufferFile.h"
#include "Util/Database.h"
#include "Util/StringParser.h"
#include "Util/SimpleMap.h"
#include "Basis/Timer.h"
#include "Basis/QuidObj.h"
#include "Basis/Quids.h"
#include "Core/Feature/FeatureDefinition.h"
#include "Link/DiskImage/DiskImageFuncs.h"
#include "Persistency/Locator.h"

namespace Impala
{
namespace Core
{
namespace Database
{


class RawDataSet
{
public:

    RawDataSet(String setName)
    {
        Init(setName);
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    RawDataSet(Util::Database* db, String setName, String setFileName)
    {
        Init(db, setName, setFileName);
    }

    RawDataSet(Util::Database* db, String setName, String setFileName,
               String nameExt)
    {
        Init(db, setName, setFileName);
        GenerateFileNames(nameExt);
    }
#endif // REPOSITORY_USED

    String
    GetSetName() const
    {
        return mSetName;
    }

    String
    GetSetNameBase() const
    {
        return mSetNameBase;
    }

    void
    SetProtocolAndHost(CString protocolAndHost)
    {
        mProtocolAndHost = protocolAndHost;
    }

    Persistency::Locator
    GetLocator() const
    {
        if (mProtocolAndHost == "not_set")
        {
            CmdOptions& options = CmdOptions::GetInstance();
            ILOG_WARN("[GetLocator] protocol may not be what you expect" <<
                      ", set = " << mSetName);
            return Persistency::Locator(mSetName, options);
        }
        return Persistency::Locator(mProtocolAndHost, mSetName);
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    Util::Database*
    GetDatabase()
    {
        return mDb;
    }

    void
    SetDatabase(Util::Database* db)
    {
        mDb = db;
    }
#endif // REPOSITORY_USED

    virtual int
    GetSetId()
    {
        return -1;
    }

    // Returns QUID_CLASS_FRAME for VideoSets and QUID_CLASS_IMAGE for ImageSets
    int
    GetQuidClass() const
    {
        return mQuidClass;
    }

    int
    NrDirs() const
    {
        return mContainer.size();
    }

    // section

    String
    GetSection(int elemId) const
    {
        String s = FileNamePath2(mContainer[elemId]);
        if (s.empty())
            return ".";
        return s;
    }

    String
    GetSectionOfFile(int fileId) const
    {
        return GetSection(mFileKey[fileId]);
    }

    // directory

    String
    GetDir(int elemId) const
    {
        String s = FileNameTail(mContainer[elemId]);
        if (s.empty())
            return ".";
        return s;
    }

    int
    GetDirId(String dir) const
    {
        for (int i=0 ; i<NrDirs() ; i++)
            if (GetDir(i) == dir)
                return i;
        return -1;
    }

    int
    GetDirId(String section, String dir) const
    {
        for (int i=0 ; i<NrDirs() ; i++)
            if ((GetSection(i) == section) && (GetDir(i) == dir))
                return i;
        return -1;
    }

    int
    GetDirIdOfFile(int fileId) const
    {
        return mFileKey[fileId];
    }

    String
    GetDirOfFile(int fileId) const
    {
        return GetDir(mFileKey[fileId]);
    }

    // file

    int
    GetFirstFileId(int elemId) const
    {
        if ((elemId < 0) || (elemId >= mFirstFileId.size()))
        {
            ILOG_ERROR("GetFirstFileId: id " << elemId << " is out of range");
            return -1;
        }
        return mFirstFileId[elemId];
    }

    int
    GetNrFiles(int elemId) const
    {
        if (!HasFiles())
            return 0;
        return mNrFiles[elemId];
    }

    bool
    HasFiles() const
    {
        return mHasFiles;
    }

    int
    NrFiles() const
    {
        if (!HasFiles())
            return NrDirs();
        return mFile.size();
    }

    // This may be an empty string in case there were no files specified
    // in the set
    String
    GetFile(int i) const
    {
        return mFile[i];
    }

    // The real file if a file was specified in the set, otherwise
    // dir + extension
    String
    GetFileOrDir(int i, String extension) const
    {
        if (mFile[i] != String(""))
            return mFile[i];
        return GetDirOfFile(i) + extension;
    }

    // The real base if a file was specified in the set, otherwise the dir name
    String
    GetFileBase(int i) const
    {
        if (mFile[i] != String(""))
            return FileNameBase(mFile[i]);
        return GetDirOfFile(i);
    }

    // This is new to allow for non-consecutive fileId's
    int
    GetFileId(int idx) const
    {
        if ((idx >= 0) && (idx < mFileId.size()))
            return mFileId[idx];
        ILOG_ERROR("No fileId with index " << idx);
        return -1;
    }

    int
    GetFileId(String file) const
    {
        for (int i=0 ; i<mFile.size() ; i++)
            if (mFile[i] == file)
                return i;
        return -1;
    }

    int
    GetFileId(int dirId, String file) const
    {
        for (int i=0 ; i<mFile.size() ; i++)
            if ((mFileKey[i] == dirId) && (mFile[i] == file))
                return i;
        return -1;
    }

    int
    GetFileId(String section, String dir, String file) const
    {
        if ((section == String("")) && (dir == String("")))
            return GetFileId(file);
        int dirId = GetDirId(section, dir);
        return GetFileId(dirId, file);
    }

    int
    GetFileIdOfQuid(Quid q)
    {
        return (IsImageSet()) ? QuidId(q) : QuidObject(q);
    }

    // Generate file names from dir if no files were specified in the set
    void
    GenerateFileNames(String extension)
    {
        for (int i=0 ; i<mFile.size() ; i++)
            if (mFile[i] == String(""))
                mFile[i] = GetDirOfFile(i) + extension;
        mHasFiles = true;
    }

    String
    GetAsPath(int fileId)
    {
        const int dirId = GetDirIdOfFile(fileId);
        String dir = GetDir(dirId);

        if (Link::DiskImage::DiskImageUsed() && dir.find("//#") != String::npos)
        {
            // prepare for use of escape sequence '0x'
            dir = StringReplaceAll(dir, "0x", "0x300x78", false);
            // treat every file system start token as a subfolder
            dir = StringReplaceAll(dir, "//#", "/0x2f#", false);
            if (dir.find("/") == 0)
                dir = dir.substr(1); // remove leading slash
        }

        return GetSection(dirId) + "/" + dir + "/" + GetFile(fileId);
    }

    // container

    bool
    IsImageSet() const
    {
        return GetQuidClass() == QUID_CLASS_IMAGE;
    }

    int
    GetNrContainers() const
    {
        if (IsImageSet())
            return NrDirs();
        return NrFiles();
    }

    String
    GetContainer(int nr)
    {
        if (IsImageSet())
            return GetContainerDir(nr);
        return GetContainerFile(nr);
    }

    int
    GetContainerNrOfQuid(Quid q)
    {
        int fileId = (IsImageSet()) ? QuidId(q) : QuidObject(q);
        return (IsImageSet()) ? GetDirIdOfFile(fileId) : fileId;
    }

    String
    GetContainerDir(int elemId)
    {
        return GetSection(elemId) + "/" + GetDir(elemId);
    }

    String
    GetContainerDirOfFile(int fileId)
    {
        return GetSectionOfFile(fileId) + "/" + GetDirOfFile(fileId);
    }

    String
    GetContainerFile(int fileId)
    {
        return GetAsPath(fileId);
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    // Any data directory part

    String
    MakeDir(String dir)
    {
        return mDb->MakeDir(dir);
    }

    String
    MakeDir(String dir1, String dir2)
    {
        return mDb->MakeDir(dir1, dir2);
    }

    String
    MakeDir(String dir1, String dir2, String dir3)
    {
        return mDb->MakeDir(dir1, dir2, dir3);
    }

    String
    MakeDir(String dir1, String dir2, String dir3, String dir4)
    {
        return mDb->MakeDir(dir1, dir2, dir3, dir4);
    }

    String
    MakeDir(String dir1, String dir2, String dir3, String dir4, String dir5)
    {
        return mDb->MakeDir(dir1, dir2, dir3, dir4, dir5);
    }

    // Any data file part

    String
    GetFilePath(String dir, String file, bool toWrite, bool silent)
    {
        return mDb->GetFilePath(dir, file, toWrite, silent);
    }

    String
    GetFilePath(String dir1, String dir2, String file, bool toWrite, bool silent)
    {
        return mDb->GetFilePath(dir1, dir2, file, toWrite, silent);
    }

    String
    GetFilePath(String dir1, String dir2, String dir3, String file,
                bool toWrite, bool silent)
    {
        return mDb->GetFilePath(dir1, dir2, dir3, file, toWrite, silent);
    }

    String
    GetFilePath(String dir1, String dir2, String dir3, String dir4, String file,
                bool toWrite, bool silent)
    {
        return mDb->GetFilePath(dir1, dir2, dir3, dir4, file, toWrite, silent);
    }

    String
    GetFilePath(String dir1, String dir2, String dir3, String dir4, String dir5,
                String file, bool toWrite, bool silent)
    {
        return mDb->GetFilePath(dir1, dir2, dir3, dir4, dir5, file, toWrite,
                                silent);
    }

    // "Specialized" interface to Database paths

    String
    GetFilePathVideoData(String setName, bool toWrite, bool silent)
    {
        String dir = "VideoData";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, setName, toWrite, silent);
    }

    String
    GetFilePathVideoData(int nameId, bool toWrite, bool silent)
    {
        String dir = "VideoData/" + GetSectionOfFile(nameId) + "/"
            + GetDirOfFile(nameId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, GetFile(nameId), toWrite, silent);
    }

    String
    GetFilePathImageData(String setName, bool toWrite, bool silent)
    {
        String dir = "ImageData";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, setName, toWrite, silent);
    }

    String
    GetFilePathImageData(int nameId, bool toWrite, bool silent)
    {
        String dir = "ImageData/" + GetSectionOfFile(nameId) + "/"
            + GetDirOfFile(nameId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, GetFile(nameId), toWrite, silent);
    }

    String
    GetFilePathImageArchive(String archiveName, bool toWrite, bool silent)
    {
        String dir = "ImageArchive";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, archiveName, toWrite, silent);
    }

    String
    GetFilePathImageArchive(int fileId, String archiveName, bool toWrite,
                            bool silent)
    {
        String dir = "ImageArchive/" + GetSectionOfFile(fileId) + "/"
            + GetDirOfFile(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, archiveName, toWrite, silent);
    }

    String
    GetFilePathFrames(int fileId, String name, bool toWrite, bool silent)
    {
        String dir = "Frames/" + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, name, toWrite, silent);
    }

    String
    GetFilePathFrames(int fileId, int frameNr, String suffix, bool toWrite,
                      bool silent)
    {
        String dir = "Frames/" + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        char frameName[200];
        sprintf(frameName, "frame_%06d", frameNr);
        return GetFilePath(dir, String(frameName)+suffix, toWrite, silent);
    }

    String
    GetFilePathVideoIndex(String subDir, String file, bool toWrite, bool silent)
    {
        String dir("VideoIndex");
        if (!subDir.empty())
            dir += "/" + subDir;
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, file, toWrite, silent);
    }

    /*
    String
    GetFilePathImageIndex(String subDir, String file, bool toWrite,
                          bool silent)
    {
        String dir = "ImageIndex/" + subDir;
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, file, toWrite, silent);
    }
    */

    String
    GetFilePathMetaData(int fileId, String metaName, bool useDbFile,
                        String suffix, bool toWrite, bool silent)
    {
        String dir = "MetaData/" + GetAsPath(fileId) + "/" + metaName;
        if (toWrite)
            MakeDir(dir);
        String fName = ((useDbFile) ? GetFileBase(fileId) : "") + suffix;
        return mDb->GetFilePath(dir, fName, toWrite, silent);
    }

    String
    GetFilePathMpeg7Shot(int fileId, bool toWrite, bool silent)
    {
        String dir = "MetaData/shots/" + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, GetFileBase(fileId) + ".xml", toWrite, silent);
    }

    String
    GetFilePathMpeg7Anno(int fileId, String conceptSet, String xmlName,
                         bool toWrite, bool silent)
    {
        String dir = "MetaData/annotations/" + conceptSet + "/"
            + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, xmlName, toWrite, silent);
    }

    String
    GetFilePathMpeg7Feature(int fileId, String xmlName, bool toWrite,
                            bool silent)
    {
        String dir = "MetaData/features/" + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, xmlName, toWrite, silent);
    }

    String
    GetFilePathMpeg7Concept(int fileId, String conceptSet, String xmlName,
                            bool toWrite, bool silent)
    {
        String dir = "MetaData/similarities/" + conceptSet + "/"
            + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, xmlName, toWrite, silent);
    }

    typedef Feature::FeatureDefinition FeatureDefinition;

    String
    GetFilePathPrototype(String protoDataFile, String featureName,
                         String fileName, bool toWrite, bool silent)
    {
        String dir = "Codebooks/" + protoDataFile + "/" + featureName;
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, fileName, toWrite, silent);
    }

    // for video sets
    String
    GetFilePathFeatureData(String walkType, FeatureDefinition def,
                           int fileOrDirId, bool partial, int startFrameOrFile,
                           bool toWrite, bool silent)
    {
        // for backward compatibility
        if (walkType == "Image")
            walkType = ".";
 
        String dir = PathJoin("FeatureData", walkType, def.GetName());
        if (mQuidClass == QUID_CLASS_IMAGE)
        {
            dir = PathJoin(dir, GetSection(fileOrDirId), GetDir(fileOrDirId));
        }
        else
        {
            dir = PathJoin(dir, GetAsPath(fileOrDirId));
        }
        if (partial)
            dir = PathJoin(dir, "part_" + MakeString(startFrameOrFile));
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
    }

    // for image sets
    String
    GetFilePathFeatureData(FeatureDefinition def, int dirId, bool partial,
                           int startFile, bool toWrite, bool silent)
    {
        String dir = "FeatureData/" + def.GetName() + "/" + GetSection(dirId)
            + "/" + GetDir(dirId) + "/";
        if (partial)
            dir += "/part_" + MakeString(startFile);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
    }

    String
    GetFilePathFeatureIndex(FeatureDefinition def, String subSet, bool toWrite,
                            bool silent)
    {
        String dir = "FeatureIndex/" + def.GetName();
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, def.AsString() + subSet + ".tab", toWrite,
                           silent);
    }

    // for both?
    String
    GetFilePathPrecomputedKernels(String filename, String otherSet,
                                  bool toWrite, bool silent)
    {
        String dir = "PrecomputedKernels";
        if(otherSet != "")
            dir += "/" + otherSet;
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, filename, toWrite, silent);
    }

    // for both
    String
    GetFilePathRandomForest(FeatureDefinition def, bool toWrite, bool silent)
    {
        String dir = "Codebooks/Forest/";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
   }
    
    // for both
    String
    GetFilePathCodebook(FeatureDefinition def, bool toWrite, bool silent)
    {
        String dir = "Codebooks/";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, def.AsString() + ".tab", toWrite, silent);
   }
    
    // when using walkType
    String
    GetFilePathSimilarityData(String walkType, String conceptSet, String model,
                              String featureString, int fileOrDirId,
                              String fileName, bool toWrite, bool silent)
    {
        // for backwards compatibility
        if(walkType == "Image") 
            walkType = ".";
        String dir = PathJoin("SimilarityData", walkType, conceptSet, model,
                              featureString);
        if(mQuidClass == QUID_CLASS_IMAGE)
        {
            dir = PathJoin(dir, GetSection(fileOrDirId), GetDir(fileOrDirId));
        }
        else
        {
            dir = PathJoin(dir, GetAsPath(fileOrDirId));
        }
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, fileName, toWrite, silent);
    }

    String
    GetFilePathSimilarityIndex(String conceptSet, String model,
                               String featureString, String fileName,
                               bool toWrite, bool silent)
    {
        String dir = "SimilarityIndex/" + conceptSet + "/" + model + "/"
            + featureString;
        if (toWrite)
            MakeDir(dir);        
        return GetFilePath(dir, fileName, toWrite, silent);
    }

    String
    GetFilePathAnnotation(String annoFile, bool toWrite, bool silent)
    {
        String dir = "Annotations";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, annoFile, toWrite, silent);
    }

    String
    GetFilePathAnnotation(int quidClass, String conceptSet, String annoFile,
                          bool toWrite, bool silent)
    {
        String dir = "Annotations/" + QuidClassToString(quidClass) + "/"
            + conceptSet;
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, annoFile, toWrite, silent);
    }

    String
    GetFilePathGroundTruth(String truthFile, bool toWrite, bool silent)
    {
        String dir = "Annotations/groundtruth";
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, truthFile, toWrite, silent);
    }

    String
    GetFilePathConceptModel(String conceptSet, String modelType,
                            FeatureDefinition def, String modelFile,
                            bool toWrite, bool silent)
    {
        String dir = "ConceptModels/" + conceptSet + "/" + modelType + "/"
            + def.AsString();
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, modelFile, toWrite, silent);
    }

    String
    GetFilePathReference(int fileId, String suffix, bool toWrite, bool silent)
    {
        String dir = "Reference/" + GetAsPath(fileId);
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, GetFileBase(fileId) + suffix, toWrite, silent);
    }

    String
    GetFilePathKernelData(String walkType, String referenceSet,
                          String kernelName, int fileOrDirId,
                          bool toWrite, bool silent)
    {
        String dir = PathJoin("KernelData", walkType, referenceSet, kernelName);
        if(mQuidClass == QUID_CLASS_IMAGE)
        {
            dir = PathJoin(dir, GetSection(fileOrDirId), GetDir(fileOrDirId));
        }
        else
        {
            dir = PathJoin(dir, GetAsPath(fileOrDirId));
        }
        if (toWrite)
            MakeDir(dir);
        return GetFilePath(dir, kernelName + ".tab", toWrite, silent);
    }
#endif // REPOSITORY_USED


    // "construction" and I/O part

    void
    AddFile(String file, String container, int id = -1)
    {
        int key = -1;
        mContainerMap.Get(container, key);

        if ((key != -1) && (mLastKeyForWarning != key) &&
            (mLastKeyForWarning != -1))
        {
            // This will cause Walkers to break down
            ILOG_ERROR("Files are not sorted by directory. " << mLastId <<
                       " container = " << container);
            return;
        }

        if (id == -1)
        {
            id = mLastId;
        }
        else
        {
            for (int i=0 ; i<mFirstFileId.size() ; i++)
                if (mFirstFileId[i] == id)
                    ILOG_ERROR("Id already in use: " << id << " file: " << file << " container: " << container);
        }

        if (key == -1)
        {
            mContainer.push_back(container);
            mContainerMap.Add(container, mContainer.size()-1);
            mFirstFileId.push_back(id);
            mNrFiles.push_back(0);
            key = NrDirs() - 1;
        }
        mFileId.push_back(id);
        mFile.push_back(file);
        mLastKeyForWarning = key;
        mFileKey.push_back(key);
        mNrFiles[key] = mNrFiles[key] + 1;
        mLastId = id + 1;
        if (!file.empty())
            mHasFiles = true;
    }

    void
    AddFile(String path, int id)
    {
        String container(".");
        String file("");
        ExtractFileAndContainer(path, file, container);
        AddFile(file, container, id);
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    void
    Save()
    {
        Save(mSetName);
    }

    void
    Save(String fileName)
    {
        Util::IOBuffer* ioBuf = mDb->GetIOBuffer(fileName, false, false, "tmp");
        if (! ioBuf)
            return;
        for (int i=0 ; i<NrFiles() ; i++)
            ioBuf->IOBuffer::Puts("\"" + GetAsPath(i) + "\"");
        delete ioBuf;
    }

    void
    SaveBinary()
    {
        if (mSetFileName == String(""))
            return;
        String fName = mSetFileName + ".bin";
        ILOG_DEBUG("saving " << fName);
        //IOBUfferFile(fName, false, true);
    }
#endif // REPOSITORY_USED

    void
    Dump(int startDir, int endDir, int startFile, int endFile)
    {
        if (endDir == -1)
            endDir = NrDirs();
        std::cout << "Dumping dirs from " << startDir << " to " << endDir
                  << " (of " << NrDirs() << ")" << std::endl;
        for (int i=startDir ; i<endDir ; i++)
            std::cout << "dir=" << i << ": " << GetSection(i) << " "
                      << GetDir(i) << ": nr files = " << GetNrFiles(i)
                      << ", first file = " << GetFirstFileId(i) << std::endl;
        if (!mHasFiles)
        {
            std::cout << "Does not have files" << std::endl;
            return;
        }
        if (startFile == -1)
        {
            std::cout << "Skipping files" << std::endl;
            return;
        }
        if (endFile == -1)
            endFile = NrFiles();
        std::cout << "Dumping files from " << startFile << " to " << endFile
                  << " (of " << NrFiles() << ")" << std::endl;
        for (int j=startFile ; j<endFile ; j++)
        {
            std::cout << GetFileId(j) << " " << GetSectionOfFile(j) << " "
                      << GetDirOfFile(j) << " " << GetFile(j) << std::endl;
        }
    }

    virtual String
    QuidToString(Quid quid, bool elaborate)
    {
        QuidObj qObj(quid);
        return qObj.ToString();
    }


protected:

    void
    Init(String setName)
    {
        mLastKeyForWarning = -1;
        mSetName = setName;
        mSetNameBase = FileNameBase(mSetName);
        mSetFileName = "";
        mProtocolAndHost = "not_set";
        mHasFiles = false;
        mLastId = 0;
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    void
    Init(Util::Database* db, String setName, String setFileName)
    {
        mLastKeyForWarning = -1;
        mDb = db;
        mSetName = setName;
        mSetNameBase = FileNameBase(mSetName);
        mSetFileName = setFileName;
        mProtocolAndHost = "not_set";
        mHasFiles = false;
        mLastId = 0;
        if (mSetFileName == String(""))
            return;
        mQuidClass = QUID_CLASS_UNKNOWN;
        ILOG_DEBUG("opening data file " << mSetFileName);
        Timer timer(1);
        Util::IOBuffer* f = mDb->GetIOBuffer(mSetFileName, true, true, "");
        if (f->Valid())
        {
            while (f->Available())
            {
                String line = f->ReadLine();
                if (line[0] && (line[0] != '#'))
                {
                    int id = -1;
                    String name("");
                    Util::StringParser p(line);
                    while (!p.TheEnd())
                    {
                        if (p.At("<id="))
                        {
                            p.Eat('=');
                            id = p.GetInt('>', true);
                            p.Eat('>');
                        }
                        else
                        {
                            String s = p.GetString2(false);
                            name = StringReplaceAll(s, "\\", "/");
                        }
                    }
                    AddFile(name, id);
                }
            }
        }
        delete f;
        ILOG_DEBUG("done read file in " << timer.SplitTime());
    }
#endif // REPOSITORY_USED

    void
    ExtractFileSectionDir(CString src, String& file, String& section, String& dir)
    {
        StringList sl(src, '/', true);

        /* DK: can this be removed? Needs to be adjusted to use mContainer
        if (Link::DiskImage::DiskImageUsed())
        {
            const String::size_type firstTokenPos = src.find("//#");
            const bool parseDiskImageStyle = (firstTokenPos != String::npos);
            if (parseDiskImageStyle)
            {
                StringList::iterator it = sl.begin();
                section = "";
                // define all until first token as 'section'
                while (!it->empty())
                {
                    if (it != sl.begin())
                        section += "/";
                    section += *it++;
                }
                it++; // skip slash that separates section and path

                // 'dir' will now start with a token
                dir = "";
                while (it != sl.end())
                {
                    String& next = *it;
                    it++;
                    if (it == sl.end())
                        file = next;
                    else
                        dir += "/" + next;
                }
                return;
            }
        }
        */

        StringList::reverse_iterator it = sl.rbegin();
        file = *it++;
        if (it != sl.rend())
            dir = *it++;
        if (it != sl.rend())
            section = *it++;
        while (it != sl.rend())
        {
            String subDir = *it++;
            if (subDir != ".")
                section = subDir + "/" + section;
        }
    }

    void
    ExtractFileAndContainer(CString src, String& file, String& container)
    {
        StringList sl(src, '/', true);
        StringList::reverse_iterator it = sl.rbegin();
        file = *it++;
        if (it != sl.rend())
            container = *it++;
        while (it != sl.rend())
        {
            String subDir = *it++;
            if (subDir != ".")
                container = subDir + "/" + container;
        }
    }

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    Util::Database*     mDb;
#endif // REPOSITORY_USED
    String              mSetName;
    String              mSetNameBase;
    String              mSetFileName;
    String              mProtocolAndHost;
    bool                mHasFiles;
    int                 mLastId;
    int                 mLastKeyForWarning;
    std::vector<int>    mFileId;
    std::vector<String> mFile;
    std::vector<int>    mFileKey;

    std::vector<String>         mContainer;
    Util::SimpleMap<String,int> mContainerMap; // the "reverse" of mContainer
    std::vector<int>            mFirstFileId;
    std::vector<int>            mNrFiles;
    
    int                 mQuidClass;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(RawDataSet, Impala.Core.Database);

} // namespace Database
} // namespace Core
} // namespace Impala

#endif

---