|| || || || || ||

---

VideoAccessObject.h

Go to the documentation of this file.

#ifndef Impala_Core_Stream_Lavc_VideoAccessObject_h
#define Impala_Core_Stream_Lavc_VideoAccessObject_h

#include "Core/Array/Arrays.h"
#include "Core/Array/MD5Hash.h"

// include our one and only entry point into libavcodec:
#include "Link/Lavc/Lavc.h"

#include "Link/DiskImage/DiskImageFuncs.h"

#include "Core/Stream/LavcProtocolDataServer.h"
#include "Core/Table/TableTem.h"

//#ifdef USE_IFILE
//#include "Core/Stream/LavcProtocolLocalFile.h"
//#endif

#ifdef USE_IFILE
#include "Core/Stream/LavcProtocolLocalFile.h"
#endif

namespace Impala
{
namespace Core
{
namespace Stream
{
namespace Lavc
{

// size | flags | file position upon read | is video 
typedef Table::TableTem< Column::ColumnTem<Int32>,
                         Column::ColumnTem<Int32>,
                         Column::ColumnTem<UInt64>,
                         Column::ColumnTem<bool> > PacketTrace;

class VideoAccessObject
{

public:

    VideoAccessObject(CString srcAddress) 
    {
        mIsValid = false;

        mFormatCtx = 0;
        mVideoStream = 0;
        mVideoCodecCtx = 0;
        mPacket = 0;
        mFrame = 0;
        mFrameValid = false;
        mRgbFrame = 0;
        mRgbBuffer = 0;
        mSwsContext = 0;

        mSrcAddress = srcAddress;

        ILOG_INFO("libavcodec version  : " << 
            (int)(LIBAVCODEC_VERSION_INT>>16) << "." << 
            (int)((LIBAVCODEC_VERSION_INT>>8)&0xFF) << "." << 
            (int)(LIBAVCODEC_VERSION_INT&0xFF));

        ILOG_INFO("libavformat version : " << 
            (int)(LIBAVFORMAT_VERSION_INT>>16) << "." << 
            (int)((LIBAVFORMAT_VERSION_INT>>8)&0xFF) << "." << 
            (int)(LIBAVFORMAT_VERSION_INT&0xFF));

        //ILOG_INFO("libavutil version   : " << 
        //    (int)(LIBAVUTIL_VERSION_INT>>16) << "." << 
        //    (int)((LIBAVUTIL_VERSION_INT>>8)&0xFF) << "." << 
        //    (int)(LIBAVUTIL_VERSION_INT&0xFF));

        ILOG_INFO("libswscale version  : " << 
            (int)(LIBSWSCALE_VERSION_INT>>16) << "." << 
            (int)((LIBSWSCALE_VERSION_INT>>8)&0xFF) << "." << 
            (int)(LIBSWSCALE_VERSION_INT&0xFF));

        ILOG_INFO("Accessing video source: " << srcAddress);

        if (!InitFFMPEG() ||
            !InitFormatContext() ||
            !InitStreams() ||
            !InitCodecContext() ||
            !InitFrames())
        {
            return;
        }

        mIsValid = true;
    }

    ~VideoAccessObject() 
    {
        if (mRgbBuffer != 0)
            delete [] mRgbBuffer;

        if (mRgbFrame != 0)
            av_free(mRgbFrame);
        if (mFrame != 0)
            av_free(mFrame);

        if (mPacket != 0)
        {
            FreePacket();
            delete mPacket;
        }

        if (mSwsContext != 0)
            sws_freeContext(mSwsContext);

        if (mVideoCodecCtx != 0)
            avcodec_close(mVideoCodecCtx);

        if (mFormatCtx != 0)
            av_close_input_file(mFormatCtx);
    }

    bool
    IsValid() const
    {
        return mIsValid;
    }

    String 
    VideoSrcAddress() const
    {
        return mSrcAddress;
    }

    int
    BitRate() const
    {
        return mFormatCtx->bit_rate;
    }

    String
    FormatNameShort() const
    {
        return mFormatCtx->iformat->name;
    }

    String
    FormatNameLong() const
    {
        return mFormatCtx->iformat->long_name;
    }

    int
    StreamCount() const
    {
        return mFormatCtx->nb_streams;
    } 

    UInt64
    FirstPacketPosition() const
    {
        const Int64& firstPosSigned = mFormatCtx->data_offset;
        if (firstPosSigned < 0)
        {
            ILOG_ERROR("Negative stream start offset unexpected");
            return 0;
        }
        return mFormatCtx->data_offset;
    }

    bool
    NativeIndexValid() const
    {
        return NativeIndexEntries() != 0;
    }

    const AVIndexEntry*
    NativeIndexEntries() const
    {
        AVIndexEntry *idx = mVideoStream->index_entries;
        if (idx && (mFormatCtx->flags & AVFMT_FLAG_IGNIDX))
            idx = 0;
        return idx;
    }

    int
    NativeIndexSize() const
    {
        if (NativeIndexValid())
            return mVideoStream->nb_index_entries;
        return 0;
    }

    int 
    VideoCodecId() const
    {
        return mVideoCodecCtx->codec_id;
    } 

    String 
    VideoCodecName() const
    {
        return mVideoCodecCtx->codec->name;
    } 

    String 
    VideoCodecTag() const
    {
        char buf[5];
        const unsigned int tagCode = mVideoCodecCtx->codec_tag;
#ifdef WIN32
        _snprintf(
#else
        snprintf(
#endif
            buf, sizeof(buf), "%c%c%c%c",
            (tagCode)       & 0xff,
            (tagCode >> 8)  & 0xff,
            (tagCode >> 16) & 0xff,
            (tagCode >> 24) & 0xff);
        return String(buf);
    }

    int
    GopSize() const
    {
        return mVideoCodecCtx->gop_size;
    }

    int
    FrameWidth() const
    {
        return mVideoCodecCtx->width;
    }

    int
    FrameHeight() const
    {
        return mVideoCodecCtx->height;
    }

    int
    AspectRatioNum() const
    {
        return mVideoCodecCtx->sample_aspect_ratio.num;
    }

    int
    AspectRatioDen() const
    {
        return mVideoCodecCtx->sample_aspect_ratio.den;
    }

    int 
    VideoDuration() const
    {
        // number of (time) fractions
        return mVideoStream->duration; 
    }

    int
    VideoTimeBaseNumerator() const
    {
        const AVRational videoTimeBase = 
            mVideoStream->time_base; // time fraction duration (seconds)
        return videoTimeBase.num;
    }

    int
    VideoTimeBaseDenominator() const
    {
        const AVRational videoTimeBase = 
            mVideoStream->time_base; // time fraction duration (seconds)
        return videoTimeBase.den;
    }

    int
    FrameRateNumerator() const
    {
        return mVideoStream->r_frame_rate.num;
    }

    int
    FrameRateDenominator() const
    {
        return mVideoStream->r_frame_rate.den;
    }



    // ------------------------ 

    bool
    Reset() const
    {
        mFrameValid = false;
        FreePacket();
        FlushCodecContext();

        if (!Seek(FirstPacketPosition(), AVSEEK_FLAG_BYTE))
        {
            ILOG_ERROR("Video source reset failed");
            return false;
        }

        ILOG_DEBUG("Video source has been reset");
        return true;
    }

    bool
    AtEof() const
    {
#ifdef FFMPEG_52
        return (url_feof(mFormatCtx->pb) != 0);
#else
        return (url_feof(&mFormatCtx->pb) != 0);
#endif
    }

    UInt8*
    RgbDataPtr() const
    {
        return mRgbBuffer;
    }
    
    char
    FrameType() const
    {
        switch (mFrame->pict_type)
        {
            case FF_I_TYPE : return 'I';
            case FF_P_TYPE : return 'P';
            case FF_B_TYPE : return 'B';
        }
        return '?';
    }

    bool 
    Seek(const UInt64& target, int flags) const
    {
        if ((Int64) target < 0)
        {
            ILOG_ERROR("File position value " << target << 
                " not supported (should fit in Int64)");
            return false;
        }

        avcodec_flush_buffers(mVideoCodecCtx);

        const int result = 
            av_seek_frame(mFormatCtx, mVideoStreamIndex, (Int64) target, flags);

        if (result < 0)
        {
            ILOG_ERROR("av_seek_frame returned a negative value: " << result << 
                " (target=" << target << ", flags=" << flags << ")");
            return false;
        }
        return true;
    }

    int 
    ReadPacket() const
    {
        FreePacket();
        // note that this method's name is correct since in fact it reads a packet
        return av_read_frame(mFormatCtx, mPacket);
    }

    int
    CurrentPacketSize() const
    {
        return mPacket->size;
    }

    int
    CurrentPacketFlags() const
    {
        return mPacket->flags;
    }

    bool
    CurrentPacketIsVideo() const
    {
        return mPacket->stream_index == mVideoStreamIndex;
    }

    UInt64
    CurrentFilePosition() const
    {
        const ByteIOContext* const b = 
#ifdef FFMPEG_52
            mFormatCtx->pb;
#else
            &(mFormatCtx->pb);
#endif
        Int64 pos = b->pos - (b->buf_end - b->buf_ptr);
        if (pos < 0)
        {
            ILOG_ERROR("Negative file position " << pos << 
                " unexpected (should fit in UInt64)");
            return 0;
        }
        return pos;
    }

    int
    DecodeFrame(int* resultCode) const
    {
        mFrameValid = false;

        if (!CurrentPacketIsVideo())
        {
            ILOG_ERROR("Attempt to decode video frame from non-video packet");
            return -1;
        }

        const int len = avcodec_decode_video(mVideoCodecCtx, 
                            mFrame, resultCode, mPacket->data, mPacket->size);
        // from avcodec.h (v5.1): 
        //   resultCode : 0 if no frame could be decompressed, 
        //                otherwise, it is nonzero.
        //   len        : On error a negative value is returned, otherwise the 
        //                number of bytes used or zero if no frame could be 
        //                decompressed.

        if (len < 0)
        {
            // indicates an error condition according to avcodec.h

            //if (len < -1)
            //{
                ILOG_DEBUG("avcodec_decode_video returned " << len <<
                    " (negative value) indicating an error occurred"); 
            //}
            //else
            //{
            //}
        }
        else if (len == 0)
        {
            if (*resultCode != 0)
                ILOG_DEBUG("Failed assumption on avcodec_decode_video: " << 
                    "len=0 (no frame decompressed, but not an error) " << 
                    "but resultCode=" << *resultCode <<" (non-zero)"); 
        }
        //else if (*resultCode == 0)
        //{
        //    ILOG_DEBUG("Failed assumption on avcodec_decode_video: " << 
        //        "len>0 (bytes used for decompressed frame, no error occurred) " << 
        //        "but resultCode=0 (no frame could be decompressed)"); 
        //}

        mFrameValid = true;
        return len;
    }

    bool
    FrameIsKey() const
    {
        return mFrame->key_frame == 1;
    }

    bool
    CurrentFrameValid() const
    {
        return mFrameValid;
    }

    String
    CurrentHash() const
    {
        if (!CurrentFrameValid())
            return "DUMMY";

        Array::Array2dVec3UInt8* arr = new Array::Array2dVec3UInt8(
            mVideoCodecCtx->width, mVideoCodecCtx->height, 0, 0,
                mRgbBuffer, true, false);
        String hash = Array::MD5Hash(arr);
        delete arr;

        return hash;
    }

    String
    CurrentHashBeforeConversion() const
    {
        if (!CurrentFrameValid())
            return "DUMMY";

        int nrOfPicturePlanes = 0;
        int maxLineSize = 1;
        for ( ; nrOfPicturePlanes < 4; nrOfPicturePlanes++)
        {
            int lineSize = mFrame->linesize[nrOfPicturePlanes];
            if (lineSize <= 0)
                break;
            maxLineSize = Max(lineSize, maxLineSize);
        }

        // construct an array holding the original (unconverted) picture's data
        Array::Array2dScalarUInt8* arr = new Array::Array2dScalarUInt8(
            maxLineSize, nrOfPicturePlanes, 0, 0);
        size_t numBytes = sizeof(UInt8) * nrOfPicturePlanes * maxLineSize;
        memset((void*) arr->PB(), 0, numBytes);
        for (int p = 0; p < nrOfPicturePlanes; p++)
        {
            int lineSize = mFrame->linesize[p];
            for (int e = 0; e < lineSize; e++)
            {
                UInt8 value = mFrame->data[p][e];
                arr->SetValue(value, e, p);
            }
        }
        String hash = Array::MD5Hash(arr);
        delete arr;
        return hash;
    }

    // Convert the image from native format to RGB
    int
    CurrentFrameToRgb() const
    {
        ClearRgbBuffer();
        int result = -1;
        if (CurrentFrameValid())
        {
            result = sws_scale(mSwsContext, 
                mFrame->data, mFrame->linesize, 
                0, mVideoCodecCtx->height, 
                mRgbFrame->data, mRgbFrame->linesize);
#ifdef FFMPEG_52
            if (result != mVideoCodecCtx->height)
                ILOG_WARN("Image height after scaling (" << result << 
                ") is expected to equal the original image's height (" << 
                mVideoCodecCtx->height << ").");
#endif
        }
        return result;
    }

    void
    FlushCodecContext() const
    {
        if (mVideoCodecCtx->codec->flush != NULL)
            mVideoCodecCtx->codec->flush(mVideoCodecCtx);

        avcodec_flush_buffers(mVideoCodecCtx);
    }

    // For dev purposes only;
    void
    DumpState() const
    {
        //std::cout << "------------------------------------" << std::endl;
        AVFormatContext* const s = mFormatCtx;
        //const UInt8 invalidPtr = 0;
        //const UInt8* curPtr = &invalidPtr;
        //if (s->cur_ptr)
        //    curPtr = (s->cur_ptr);
        //std::cout << "Current AVFormatContext: " <<
        //    "data_offset=" << s->data_offset << ", " <<
        //    "cur_ptr=" << curPtr << ", " <<
        //    "cur_len=" << s->cur_len << ", " <<
        //    "packet_size=" << s->packet_size << 
        //    std::endl;
 
        const ByteIOContext* const b = 
#ifdef FFMPEG_52
            s->pb;
#else
            &(s->pb);
#endif
        std::cout << "Current ByteIOContext: " <<
            //"total.bytes=" << url_fsize(&(s->pb)) << ", " <<
            //"buffer_size=" << b->buffer_size << ", " <<
            "act.buffer.size=" << b->buf_end - b->buffer << ", " <<
            "pos.of.buffer=" << b->pos - (b->buf_end - b->buffer) << ", " <<
            "pos.in.buffer=" << b->buf_ptr - b->buffer << ", " <<
            "pos.in.file=" << b->pos - (b->write_flag ? 0 : (b->buf_end - b->buf_ptr)) << ", " <<
            "eof_reached=" << b->eof_reached << ", " <<
            //"checksum=" << b->checksum << ", " <<
            //"must_flush=" << b->must_flush <<
            //"write_flag=" << b->write_flag << ", " <<
            //"is_streamed=" << b->is_streamed << 
            std::endl;
    }


private:
    
    bool 
    InitFFMPEG()
    {

        // Set libavcodec's log output level
        //av_log_set_level(AV_LOG_DEBUG);
        //av_log_set_level(AV_LOG_ERROR);
        //av_log_set_level(AV_LOG_FATAL);
        av_log_set_level(AV_LOG_QUIET);


        // Register all formats and codecs 
        av_register_all();
        avcodec_register_all();

        register_protocol(&LavcProtocolDataServer_protocol);

        if (Link::DiskImage::DiskImageUsed())
        {
            ILOG_DEBUG("Registering disk image protocol..");
            Link::DiskImage::RegisterProtocol();
        }

#ifdef USE_IFILE
        ILOG_DEBUG("Registering Impala file protocol..");
        register_protocol(&LavcProtocolLocalFile_protocol);
#endif
        return true;
    }

    bool 
    InitFormatContext()
    {
        int result = av_open_input_file(&mFormatCtx, 
                                 mSrcAddress.c_str(), NULL, 0, NULL);

        if (result != 0)
        {
            ILOG_ERROR("Unable to open: " << mSrcAddress << 
                " (" << result << ")");
            return false;
        }

#if LIBAVFORMAT_VERSION_INT >= ((51<<16)+(14<<8)+0) // from version 51.14.0 on
        if (mFormatCtx->iformat->flags & AVFMT_GENERIC_INDEX)
        {
            // if flag not initially cleared, may result in invalid data;
            // mFormatCtx is not available until input file is opened
            mFormatCtx->iformat->flags -= AVFMT_GENERIC_INDEX;
            ILOG_DEBUG("Flag cleared: mFormatCtx->iformat->flags -= " <<
                "AVFMT_GENERIC_INDEX");

            // re-open the file
            result = av_open_input_file(&mFormatCtx, 
                                 mSrcAddress.c_str(), NULL, 0, NULL);
        }
#endif

        // Retrieve stream information
        result = av_find_stream_info(mFormatCtx);
        if (result < 0) 
        {
            ILOG_ERROR("Could not find stream info in: " << mSrcAddress <<
                " (" << result << ")");
            return false;
        }

        // Dump information about file onto standard error
        dump_format(mFormatCtx, 0, mSrcAddress.c_str(), false);

        mFormatCtx->flags |= AVFMT_FLAG_GENPTS;

        return true;
    }

    bool
    InitStreams()
    {
        mVideoStreamIndex = -1;
        mVideoStream = 0;
        int videoStreamCount = 0;

        // find the first video stream 
        for (int i = 0; i < StreamCount(); i++)
        {
            if (mFormatCtx->streams[i]->codec->codec_type == CODEC_TYPE_VIDEO)
            {
                if (++videoStreamCount > 1)
                    continue;
                    // Note that this obfuscates the fact that more video data 
                    // is present which is inappropriate in forensic settings.

                mVideoStreamIndex = i;
                mVideoStream = mFormatCtx->streams[i];
            }
        }

        if (videoStreamCount == 0) 
        {
            ILOG_ERROR("Did not find video stream in: " << mSrcAddress);
            return false;
        }

        if (videoStreamCount > 1)
            ILOG_WARN("Found " << videoStreamCount << " video streams; using " <<
                "just one from: " << mSrcAddress);

        return true;
    }

    bool 
    InitCodecContext()
    {
        mVideoCodecCtx = mVideoStream->codec;
        
        AVCodec* codec = avcodec_find_decoder(mVideoCodecCtx->codec_id);
        if (codec == NULL) 
        {
            ILOG_ERROR("Codec not found for: " << mSrcAddress);
            return false;
        }

        int result = avcodec_open(mVideoCodecCtx, codec);
        if (result < 0) 
        {
            ILOG_ERROR("Could not open codec " << codec->name << " (id: " << 
                codec->id << ") for: " << mSrcAddress << " (" << result << ")");
            return false;
        }

#ifdef FFMPEG_52
        if (sws_isSupportedInput(mVideoCodecCtx->pix_fmt) < 0)
        {
            ILOG_ERROR("sws conversion does not support pixel format (" << 
                mVideoCodecCtx->pix_fmt << ") of: " << mSrcAddress);
            return false;
        }
#endif

        mSwsContext = sws_getContext(
            mVideoCodecCtx->width, mVideoCodecCtx->height, mVideoCodecCtx->pix_fmt, 
            mVideoCodecCtx->width, mVideoCodecCtx->height, PIX_FMT_RGB24, 
            SWS_BICUBIC, NULL, NULL, NULL);
        if (mSwsContext == NULL) 
        {
            ILOG_ERROR("Failed to initialize image conversion context for: " << 
                mSrcAddress);
            return false;
        }

        if (codec != mVideoCodecCtx->codec)
        {
            ILOG_ERROR("Codec found is not same as codec stored (" << 
                MakeString(codec->id) << " vs. " << 
                MakeString(mVideoCodecCtx->codec->id) << ") for: " << 
                mSrcAddress);
        }

        return true;
    }

    bool 
    InitFrames()
    {
        mPacket = new AVPacket;
        mPacket->data = 0;

        // Allocate frame for reading
        mFrame = avcodec_alloc_frame();

        // Allocate frame for RGB conversion
        mRgbFrame = avcodec_alloc_frame();

        if ((mFrame == NULL) || (mRgbFrame == NULL)) 
        {
            ILOG_ERROR("Could not allocate frame memory for: " << mSrcAddress);
            return false;
        }

        // allocate buffer to hold frame data converted to RGB;
        // (note the comments in avcodec.h for avpicture_get_size)
        mRgbBufferSize = avpicture_fill( (AVPicture*) mRgbFrame, 
            NULL, PIX_FMT_RGB24, mVideoCodecCtx->width, mVideoCodecCtx->height);
        mRgbBuffer = new UInt8[mRgbBufferSize];
        //ClearRgbBuffer();

        // Assign appropriate parts of buffer to image planes in mRgbFrame
        int result = avpicture_fill( (AVPicture*) mRgbFrame, 
            mRgbBuffer, PIX_FMT_RGB24, mVideoCodecCtx->width, mVideoCodecCtx->height);

        return true;
    }

    void
    FreePacket() const
    {
        if (mPacket->data != 0)
        {
            av_free_packet(mPacket);
            //mPacket->data = 0;
        }
    }

    void
    ClearRgbBuffer() const
    {
        std::memset( (void*) mRgbBuffer, 0, mRgbBufferSize);
    }


    bool mIsValid;
    String mSrcAddress;

    AVFormatContext* mFormatCtx;
    int mVideoStreamIndex;
    AVStream* mVideoStream;
    AVCodecContext* mVideoCodecCtx;
    AVPacket* mPacket;
    AVFrame* mFrame;
    mutable bool mFrameValid;
    AVFrame* mRgbFrame;
    int mRgbBufferSize;
    mutable UInt8* mRgbBuffer;
    struct SwsContext* mSwsContext;

    ILOG_VAR_DECL;

}; //class

ILOG_VAR_INIT(VideoAccessObject, Impala.Core.Stream.Lavc);


}}}} // namespace Impala::Core::Stream::Lavc

#endif

---