|| || || || || ||

---

RgbDataSrcInfo.h

Go to the documentation of this file.

#ifndef Impala_Core_Stream_RgbDataSrcInfo_h
#define Impala_Core_Stream_RgbDataSrcInfo_h

#include "Basis/FileName.h"
#include "Basis/ILog.h"
#include "Basis/Timer.h"
#include "Util/StringParser.h"
#include "Util/Database.h"
#include "Util/IOBuffer.h"
#include "Core/Stream/RgbDataSrc.h"
#include "Core/Table/TableTem.h"
#include "Core/Table/Write.h"
#include "Core/Table/Read.h"
#include "Core/Column/ColumnTem.h"
#include "Core/Column/FixedStringColumn.h"
#include "Persistency/RgbDataSrcLocator.h"
#include "Persistency/RepositoryInFileSystem.h"

namespace Impala
{
namespace Core
{
namespace Stream
{


class RgbDataSrcInfo : public RgbDataSrc
{
    // frame nr | preceeding key frame nr | preceeding key frame position | frame hash
    typedef Table::TableTem<Column::ColumnTem<Int32>,
                            Column::ColumnTem<Int32>,
                            Column::ColumnTem<UInt64>,
                            Column::FixedStringColumn > IndexTableType;

    typedef Column::FixedStringColumn::ColElemType FixedString;

public:

    static const String DUMMY_FRAME_HASH;
    static const String MISSING_FRAME_HASH;

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    RgbDataSrcInfo(int src, CString srcName, Util::Database* db,
                   CString infoName = "", bool newImpl = false) 
        : RgbDataSrc(src, srcName), mDb(db)
    {
#else // REPOSITORY_USED
    RgbDataSrcInfo(int src, const Persistency::RgbDataSrcLocator& loc, 
                   CString infoName = "", bool newImpl = false) 
        : RgbDataSrc(src, loc.GetName())
    {
        String srcName = loc.GetName();
        mLoc = loc;
#endif // REPOSITORY_USED
        mNewImpl = newImpl;

        //Determine the Info file path
        if (infoName.empty())
        {
#ifdef USE_IFILE
            int posAfterProtocol = srcName.find(":") + 1;
            mInfoPath = srcName.substr(posAfterProtocol) + ".info";
#else
            mInfoPath = srcName + ".info";
#endif
        }
        else
            mInfoPath = infoName;

        InitInfo();

        // SK: Note that the following lines use 'magic numbers' in stead
        // of using corresponding symbolic constants from RgbDataSrcFactory 
        // which would create a mutual dependency between the two classes.
        // To be refactored.
        if ((src == 18) || (src == 19))
            ReadInfo();
        if (src == 19)
            DumpIndex();
    }
    
    //RgbDataSrcInfo(String InfoPath) 
    //    : RgbDataSrc(0,"RgbDataSrcInfo")
    //{
    //    mInfoPath=InfoPath;
    //    InitInfo();
    //}

    virtual
    ~RgbDataSrcInfo()
    {
        if (mIndex)
            delete mIndex;
    }

    virtual bool
    ReadFrameData() = 0;

    virtual bool
    CurIsIFrame()
    {
        int f = FrameNr();
        return GetSeekableFrame(f) == f;
    }

    virtual bool
    Valid() const
    {
        return true;
    }

    Int64
    DumpHashValue(char* &buffer, const int maxChars, int bytesWritten, 
        CString hash) const
    {
        const int hashLen = hash.size();
        if (bytesWritten + hashLen + 1 > maxChars) // add 1 for '\t'
        {
            ILOG_ERROR("Buffer too small (" << maxChars << 
                ") to write all index hashes");
            return -1;
        }
        sprintf(buffer, "%s\t", hash.c_str());
        buffer += (hashLen + 1);
        int written = bytesWritten + (hashLen + 1);
        return written;
    }

    Int64 
    DumpIndexHashes(char* buffer, const int maxChars) const
    {
        const int nrOfIndexedFrames = mIndex->Size();
        const int maxFrameNr = mIndex->Get1(nrOfIndexedFrames - 1);

        int indx = 0;
        Int64 bytesWritten = 0;
        int frameNr = 0;
        while (indx < nrOfIndexedFrames)
        {
            String hash;
            const int frameNrFromIndexEntry = mIndex->Get1(indx);
            if (frameNr < frameNrFromIndexEntry)
            {
                ILOG_WARN("Index contains no entry for frame #" << frameNr);
                hash = MISSING_FRAME_HASH;
            }
            else
            {
                if (frameNr > frameNrFromIndexEntry)
                {
                    ILOG_ERROR("[DumpIndexHashes] Index contains inconsistency for frame #" << 
                        frameNrFromIndexEntry << " at table position " << indx <<
                        "  (zero based)");
                }
                const String hash32(mIndex->Get4(indx).GetData(), HASH_SIZE);
                hash = String(hash32.c_str()); // (dummy) hashes may be shorter
                indx++;
            }
            bytesWritten = DumpHashValue(buffer, maxChars, bytesWritten, hash);
            if (bytesWritten < 0)
                return -1;
            frameNr++;
        }
        return bytesWritten;
    }

protected:

    String
    ReplaceWhite(String result)
    {
        String replaceWhat=" ";
        String replaceWithWhat="_";
        while(1)
        {
            const int pos = result.find(replaceWhat);
            if (pos==-1)
                break;
            result.replace(pos,replaceWhat.size(),replaceWithWhat);
        }
        if(result=="")
        {
            ILOG_DEBUG("String was empty, resetting to _");
            result="_";
        }
        return result;

    }

    String 
    GetInfoFilePath(bool toWrite, bool silent=true) const
    {
        return mInfoPath;
    }

    Util::IOBuffer* 
    GetInfoIOBuffer(bool toWrite, bool silent=true) const
    {
        String fname = GetInfoFilePath(toWrite, silent);
        if (fname.empty())
        {
            if (!silent)
                ILOG_ERROR("Unable to determine info file name!");
            return 0;
        }

        //Util::IOBuffer* buf=db.GetIOBuffer(fname, !toWrite, false, "tmp");
        String tmpFile = (toWrite) ? "tmp" : "";
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
        Util::IOBuffer* buf = mDb->GetIOBuffer(fname, !toWrite, false, tmpFile);
#else // REPOSITORY_USED
        typedef Persistency::RepositoryInFileSystem RepFS;
        Persistency::File file = RepFS::GetInstance().GetFile(mLoc, "", fname,
                                                              toWrite, false);
        Util::IOBuffer* buf = (toWrite) ? file.GetWriteBuffer()
                                        : file.GetReadBuffer();
#endif // REPOSITORY_USED

        if(!buf)
        {
            ILOG_DEBUG("Invalid File! "<<mInfoPath);
            return 0;
        }

        if(!buf->Valid())
        {
            ILOG_DEBUG("Invalid File! "<<mInfoPath);
            delete buf;
            return 0;
        }

        return buf;
    }

    void InitInfo()
    {
        mInfoVersion = "2.2";
        //For now, as long as applications do not register this parameter,
        //the indexes will be created in place (same location as the file)
        mIndexInPlace = true; //options.GetBool("IndexInPlace",true);

        mIndex = new IndexTableType(Column::ColumnTem<Int32>(1),
                                  Column::ColumnTem<Int32>(1),
                                  Column::ColumnTem<UInt64>(1),
                                  Column::FixedStringColumn(HASH_SIZE, 1));
        mIndex->SetColName(1, "frame_nr");
        mIndex->SetColName(2, "key_frame_nr");
        mIndex->SetColName(3, "key_frame_pos");
        mIndex->SetColName(4, "frame_hash");

        mFormatStr = String(
                          "INFO VERSION         : ") + mInfoVersion + "\n"
                        + "==================================\n"
                        + "File name            : %s\n"
                        + "\n"
                        + "Format name          : %s\n"
                        + "Format name (long)   : %s\n"
                        + "Bit rate (bps)       : %d \n"
                        + "Stream count         : %d \n"
                        + "\n"
                        + "Codec name           : %s\n"
                        + "Codec tag            : %s\n"
                        + "Time base            : %d / %d \n"
                        + "Duration (fractions) : %lld \n"
                        + "Duration (seconds)   : %lld \n"
                        + "Duration (human)     : %lldh %lldm %llds \n"
                        + "\n"
                        + "Frame width          : %d \n"
                        + "Frame height         : %d \n"
                        + "Aspect ratio         : %d / %d \n"
                        + "\n"
                        + "Frame count          : %d \n"
                        + "Frame rate (fps)     : %d / %d \n"
                        + "GOP size             : %d \n"
                        + "Leading bad frames   : %d \n"
                        + "Bad frame ID's       : %s \n"
                        + "\n"
                        + "Index exists         : %s \n";
    }

    bool ReadInfo(bool silent = false)
    {
        ILOG_DEBUG("Reading info file");
        Timer timer;
        Util::IOBuffer* infoBuf = GetInfoIOBuffer(false, silent);
        if (!infoBuf)
        {
            ILOG_WARN("Reading canceled");
            return false;
        }

        char buf[BUFFER_SIZE];
        infoBuf->Read(buf, BUFFER_SIZE);
        char srcName[100];
        char formatName[30];
        char formatLName[100];
        char codecName[30];
        char codecTag[5];
        char indexExists[5];
        
        int frameCount;
        char badFramesList[10000];
        int numberOfLeadingBadFrames;
        int res = sscanf(buf, mFormatStr.c_str(),
                    srcName,
                    formatName,
                    formatLName,
                    &mBitRate,
                    &mStreamCount,
                    codecName,
                    codecTag,
                    &mTimeBaseNum, &mTimeBaseDen,
                    &mVideoDuration,
                    &mDuration,
                    &mDurationHour, &mDurationMin, &mDurationSec,
                    &mFrameWidth,
                    &mFrameHeight,
                    &mAspectRatioNum, &mAspectRatioDen,
                    &frameCount,
                    &mFrameRateNum, &mFrameRateDen,
                    &mGopSize,
                    &numberOfLeadingBadFrames,
                    badFramesList,
                    indexExists);
        if (res != 25)
        {
            ILOG_ERROR("RGB data source info format not compatible; " <<
                "expected version is: " << mInfoVersion);
            return false;
        }

        //if (frameCount != mLastFrame + 1)
        //{
        //    ILOG_ERROR("Number of frames according to info file (" << 
        //        frameCount << ") does not match currently " <<
        //        " established frame count (" << (mLastFrame + 1) << ")");
        //    return false;
        //}

        //Copy the scanned data to member variables
        mLastFrame = frameCount - 1;
        mFormatName = formatName;
        mFormatLName = formatLName;
        mCodecName = codecName;
        mCodecTag = codecTag;

        SetLeadingBadFrames(numberOfLeadingBadFrames);
        mBadFrameNrs.clear();
        const String badFramesListString(badFramesList);
        if (badFramesListString != "none")
        {
            Util::StringParser parser(badFramesListString);
            while (!parser.TheEnd())
            {
                const int frameNr = parser.GetInt(';', false, true);
                MarkBadFrame(frameNr);
            }
        }

        if (String(indexExists) == "yes")
            mIndexExists = true;
        else
            mIndexExists = false;

        // for debug log:
        {
            StringList badFramesString;
            for (int i = 0; i < mBadFrameNrs.size(); i++)
                badFramesString << MakeString(mBadFrameNrs[i]);

            sprintf(buf, mFormatStr.c_str(),
                        srcName,
                        formatName,
                        formatLName,
                        mBitRate,
                        mStreamCount,
                        codecName,
                        codecTag,
                        mTimeBaseNum, mTimeBaseDen,
                        mVideoDuration,
                        mDuration,
                        mDurationHour, mDurationMin, mDurationSec,
                        mFrameWidth,
                        mFrameHeight,
                        mAspectRatioNum, mAspectRatioDen,
                        mLastFrame + 1,
                        mFrameRateNum, mFrameRateDen,
                        mGopSize,
                        mBlankFrames,
                        badFramesString.ToString(';').c_str(),
                        indexExists);
            ILOG_DEBUG("\n\n" << buf);
        }

        if (mIndexExists)
        {   
            ILOG_DEBUG("Reading the index attached to info file");
            ILOG_DEBUG("  start read " << timer.SplitTime());
            Read(mIndex, infoBuf);
            ILOG_DEBUG("  done actual read " << timer.SplitTime());
            ILOG_DEBUG(mIndex->Size() << " index entries read");
        }

        delete infoBuf; 
        return true;
    }

    void DumpIndex()
    {
        ILOG_DEBUG("Dumping index entries:");
        for (int i = 0; i < mIndex->Size(); i++)
            ILOG_DEBUG(i << " : " << mIndex->Get1(i) << " "
                         << mIndex->Get2(i) << " " << mIndex->Get3(i));
    }

    void WriteInfo()
    {
        ILOG_DEBUG("Writing info file");

        //Open the Info file for writing
        Util::IOBuffer* infoBuf = GetInfoIOBuffer(true);

        if (!infoBuf)
        {
            ILOG_WARN("Writing canceled: if file already exists, "
                        << "use the --override switch; "
                        << "otherwise make sure the directory is writable.");
            return;
        }

        //Write the video information to file
        char buf[BUFFER_SIZE];
        memset(buf, 0, BUFFER_SIZE);
        StringList badFramesStringList;
        for (int i = 0; i < mBadFrameNrs.size(); i++)
            badFramesStringList << MakeString(mBadFrameNrs[i]);
        String badFramesString = badFramesStringList.ToString(';');
        if (badFramesString.empty())
            badFramesString = "none";
        sprintf(buf, mFormatStr.c_str(),
                    ReplaceWhite(FileNameTail(mSrcName)).c_str(),
                    ReplaceWhite(mFormatName).c_str(),
                    ReplaceWhite(mFormatLName).c_str(),
                    mBitRate,
                    mStreamCount,
                    ReplaceWhite(mCodecName).c_str(),
                    ReplaceWhite(mCodecTag).c_str(),
                    mTimeBaseNum, mTimeBaseDen,
                    mVideoDuration,
                    mDuration,
                    mDurationHour, mDurationMin, mDurationSec,
                    mFrameWidth,
                    mFrameHeight,
                    mAspectRatioNum, mAspectRatioDen,
                    mLastFrame + 1,
                    mFrameRateNum, mFrameRateDen,
                    mGopSize,
                    mBlankFrames,
                    badFramesString.c_str(),
                    (mIndexExists ? "yes" : "no") );
        infoBuf->Write(buf, BUFFER_SIZE);

        if (mIndexExists)
        {
            ILOG_DEBUG("Index exists; writing table");
            Write(mIndex, infoBuf, true);
            ILOG_DEBUG(mIndex->Size() << " index entries written");
        }

        delete infoBuf;
    }

    void 
    AddIndex(int fr, int key, UInt64 pos, CString hash)
    {
        if (fr != (mIndex->Size() + (mNewImpl ? 0 : mBlankFrames))) // for backw.compat.
            ILOG_WARN("Frame numbers mismatch " << fr << " != " << mIndex->Size());
        FixedString fStr(HASH_SIZE, (char*) hash.c_str(), false);
        mIndex->Add(fr, key, pos, fStr);
    }

    int GetSeekableFrame(int fr) const
    {
        return mIndex->Get2(fr);
    }

    void SetKey(int fr, int key)
    {
        mIndex->Set2(fr, key);
    }

    UInt64 GetPos(int fr) const
    {
        const int indexPosition = fr;

        if (mIndexExists)
        {
            if (indexPosition < mIndex->Size())
            {
                if (FrameHash(indexPosition) == DUMMY_FRAME_HASH)
                {
                    ILOG_WARN("No position for invalid frame: " << fr << 
                        "; returning next frame's position instead");
                    return GetPos(fr + 1);
                }
                else
                    return mIndex->Get3(indexPosition);
            }
            else
                ILOG_ERROR("Index position out of bounds: " << indexPosition);
        }
        else
        {
            ILOG_WARN("No external index available; " <<
                "cannot lookup seek position for frame: " << fr);
            if (!mNewImpl)
                return 0; // backw.compat.
        }

        return -1;
    }

    // SK: 28-05-09: To be deleted once no longer used by RgbDataSrcLavc_old
    int GetFrameFromHash(String hash) const
    {
        for (int i = mBlankFrames; i < mIndex->Size(); i++)
            if (FrameHash(i) == hash)
                return i;

        return -1;
    }

    void SetPos(int fr, UInt64 pos)
    {
        mIndex->Set3(fr, pos);
    }

    virtual String 
    FrameHash() const
    {
        if (HasIndex())
            return FrameHash(FrameNr());
        else
            return RgbDataSrc::FrameHash();
    }

    virtual String 
    FrameHash(int fr) const
    {
        return String(mIndex->Get4(fr).GetData(), HASH_SIZE);
    }

    // all bad frames, including leading bad frames
    virtual const int 
    BadFrames() const
    {
        return mBadFrameNrs.size();
    }


protected:
#ifndef REPOSITORY_USED // Here comes the deprecated stuff
    Util::Database* mDb;
#endif // REPOSITORY_USED
    IndexTableType* mIndex;


    virtual bool
    FrameValid(int frameNr) const
    {
        if (!RgbDataSrc::FrameValid(frameNr))
            return false;

        for (int i = 0; i < mBadFrameNrs.size(); i++)
            if (mBadFrameNrs[i] == frameNr)
                return false;

        return true;
    }

    bool
    MarkBadFrame(int frameNr)
    {
        for (int i = 0; i < mBadFrameNrs.size(); i++)
            if (mBadFrameNrs[i] == frameNr)
                return false;

        mBadFrameNrs.push_back(frameNr);
        return true;
    }


private:

    const static int BUFFER_SIZE = 100000;

    String     mFormatStr;
    String     mInfoPath;
    String     mInfoVersion;
    bool            mIndexInPlace;

    std::vector<int> mBadFrameNrs;
    Persistency::RgbDataSrcLocator mLoc;

    ILOG_VAR_DEC;
        
}; // class

ILOG_VAR_INIT(RgbDataSrcInfo, Impala.Core.Stream);

// fixed size 32:
const String RgbDataSrcInfo::DUMMY_FRAME_HASH =   "DUMMY_FRAME_HASH________________";
const String RgbDataSrcInfo::MISSING_FRAME_HASH = "MISSING_FRAME_HASH______________";

} // namespace
} // namespace
} // namespace

#endif

---