|| || || || || || || ||

---

IOBufferFile.h

Go to the documentation of this file.

#ifndef Impala_Util_IOBufferFile_h
#define Impala_Util_IOBufferFile_h

#include "Basis/NativeTypeFormats.h"
#include "Basis/String.h"
#include "Util/IOBuffer.h"

namespace Impala
{
namespace Util
{


class IOBufferFile : public IOBuffer
{
public:

    static bool
    FileExists(CString path)
    {
        FILE* fp = Open(path, "r");
        if (fp)
        {
            fclose(fp);
            return true;
        }
        return false;
    }

    IOBufferFile(CString filename, bool readMode, bool useMemory)
    {
        mReadMode = readMode;
        mUseMemory = useMemory;

        mFile = Open(filename, (mReadMode ? "rb" : "w+b"));
        if (!mFile)
        {
            ILOG_ERROR("Failed to open " << filename);
            return;
        }

        if (mReadMode)
        {
            FSeek(mFile, 0, SEEK_END);
            PositionType size = FTell(mFile);
            FSeek(mFile, 0, SEEK_SET);
            if (mUseMemory)
            {
                unsigned char* buf = new unsigned char[size];
                fread(buf, 1, size, mFile);
                SetBuffer(buf, size);
                fclose(mFile);
                mFile = 0;
            }
            else
            {
                SetSize(size);
            }
        }
    }

    virtual
    ~IOBufferFile()
    {
        CheckDataChannelWrite();
        if (mFile)
        {
            if (mUseMemory && !mReadMode)
                fwrite(GetBuffer(), 1, GetPosition(), mFile);
            fclose(mFile);
        }
    }

    virtual bool
    Valid()
    {
        if (mReadMode && mUseMemory)
            return (GetBuffer() != 0);
        return (mFile != 0);
    }

    virtual void
    SetPosition(PositionType position)
    {
        if (GetPosition() == position)
            return;
        if (!mUseMemory)
            FSeek(mFile, position, SEEK_SET);
        IOBuffer::SetPosition(position);
    }

    virtual Int64
    Read(void* buf, Int64 bytesToRead)
    {
        Int64 available = Available();
        if (available < bytesToRead)
            bytesToRead = available;
        if (mUseMemory)
        {
            memcpy(buf, GetBuffer() + GetPosition(), bytesToRead);
        }
        else
        {
            FSeek(mFile, GetPosition(), SEEK_SET);
            Int64 nRead = fread(buf, 1, bytesToRead, mFile);
        }
        SetPosition(GetPosition() + bytesToRead);
        return bytesToRead;
    }

    virtual String
    ReadLine()
    {
        if (mUseMemory)
            return IOBuffer::ReadLine();

        // todo : this is a very inefficient implementation
        Int64 available = Available();
        Int64 maxLineSize = 1048576; // 1 Mb, todo : give user control
        if (available < maxLineSize)
            maxLineSize = available + 1; // extra space for '\0'
        // RvB: In debug mode this caused waiting time of up to 8 minutes
        //char* buf = new char[maxLineSize];
        static char buf[1024 * 1024];
        FSeek(mFile, GetPosition(), SEEK_SET);
        fgets(buf, maxLineSize, mFile);
        SetPosition(FTell(mFile));
        for (int i=0 ; i<maxLineSize ; i++)
        {
            if ((buf[i] == '\r') || (buf[i] == '\n'))
            {
                buf[i] = '\0';
                break;
            }
        }
        String res(buf);
        //delete buf;
        return res;
    }

    virtual Int64
    Gets(char* buf, Int64 bytesToRead)
    {
        if (mUseMemory)
            return IOBuffer::Gets(buf, bytesToRead);

        Int64 available = Available();
        if (available < bytesToRead)
            bytesToRead = available;
        FSeek(mFile, GetPosition(), SEEK_SET);
        fgets(buf, bytesToRead, mFile);
        SetPosition(FTell(mFile));
        return bytesToRead;
    }

    virtual void
    NativeTypeRead(Int8* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt8* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Int16* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt16* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Int32* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt32* ptr)
    {
        DoNativeTypeRead(ptr);
    }
    
    virtual void
    NativeTypeRead(Int64* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt64* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Real32* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Real64* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    Write(const void* buf, Int64 bytesToWrite)
    {
        if (mUseMemory)
            return IOBuffer::Write(buf, bytesToWrite);

        fwrite(buf, 1, bytesToWrite, mFile);
        SetPositionAndSize(FTell(mFile));
    }

    virtual Int64
    Puts(const char* buf)
    {
        if (mUseMemory)
            return IOBuffer::Puts(buf);

        fprintf(mFile, "%s\n", buf);
        SetPositionAndSize(FTell(mFile));
        return 1;
    }

    virtual void
    NativeTypeWrite(Int8 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt8 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Int16 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt16 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Int32 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt32 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Int64 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt64 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Real32 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Real64 val)
    {
        DoNativeTypeWrite(val);
    }

private:

    static FILE*
    Open(CString path, CString mode)
    {
#ifdef WIN32
        if (path.size() >= _MAX_PATH)
        {
            static const String longPathPrefix = "\\\\?\\"; // See MS help on _MAX_PATH
            const String longPathSpec = longPathPrefix + StringReplaceAll(path, "/", "\\");
            wchar_t longPathSpecWC[32768+1]; // reserve one position for null term.
            for (int i = 0; i < longPathSpec.size(); i++)
                longPathSpecWC[i] = (wchar_t) longPathSpec.at(i);
            longPathSpecWC[longPathSpec.size()] = L'\0';
            wchar_t modeWC[10];
            for (int i = 0; i < mode.size(); i++)
                modeWC[i] = (wchar_t) mode.at(i);
            modeWC[mode.size()] = L'\0';
            FILE* fp = _wfopen(longPathSpecWC, modeWC);
            return fp;
        }
#endif
        return fopen(path.c_str(), mode.c_str());
    }

    template <class NativeType>
    void
    DoNativeTypeRead(NativeType* ptr)
    {
        if (mUseMemory)
            return IOBuffer::DoNativeTypeRead(ptr);

        String fs = NativeTypeFormat<NativeType>(0);
        FSeek(mFile, GetPosition(), SEEK_SET);
        fscanf(mFile, fs.c_str(), ptr);
        SetPosition(FTell(mFile));
    }

    template <class NativeType>
    void
    DoNativeTypeWrite(NativeType val)
    {
        String fs = NativeTypeFormat<NativeType>(0) + " ";
        if (mUseMemory)
        {
            char* start = (char*) (GetBuffer() + GetPosition());
            sprintf(start, fs.c_str(), val);
            Int64 i = 0;
            while (start[i] != ' ')
                i++;
            SetPosition(GetPosition() + i + 1);
        }
        else
        {
            fprintf(mFile, fs.c_str(), val);
            SetPositionAndSize(FTell(mFile));
        }
    }

    PositionType
    FTell(FILE* fp)
    {
#ifdef WIN32
#if _MSC_VER <= 1310
        return ftell(fp);
#else
        return _ftelli64(fp);
#endif
#else
        return ftello(fp);
#endif
}

    int
    FSeek(FILE* fp, PositionType offset, int origin)
    {
#ifdef WIN32
#if _MSC_VER <= 1310
        return fseek(fp, offset, origin);
#else
        return _fseeki64(fp, offset, origin);
#endif
#else
        return fseeko(fp, offset, origin);
#endif
    }

    bool  mReadMode;
    bool  mUseMemory;
    FILE* mFile;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(IOBufferFile, Impala.Util);

} // namespace Util
} // namespace Impala

#endif

---