|| || || || || ||

---

IOBufferFile.h

Go to the documentation of this file.

#ifndef Impala_Util_IOBufferFile_h
#define Impala_Util_IOBufferFile_h

#include <string>

#include "Basis/NativeTypeFormats.h"
#include "Basis/String.h"
#include "Util/IOBuffer.h"

namespace Impala
{
namespace Util
{


class IOBufferFile : public IOBuffer
{
public:

    static bool
    FileExists(CString path)
    {
        FILE* fp = Open(path, "r");
        if (fp)
        {
            fclose(fp);
            return true;
        }
        return false;
    }

    IOBufferFile(CString filename, bool readMode, bool useMemory)
    {
        if (!readMode && useMemory)
        {
            ILOG_ERROR("useMemory not supported for write mode");
            return;
        }

        mReadMode = readMode;
        mUseMemory = useMemory;

        mFile = Open(filename, (mReadMode ? "rb" : "w+b"));
        if (!mFile)
        {
            ILOG_ERROR("Failed to open " << filename);
            return;
        }

        if (mReadMode)
        {
            FSeek(mFile, 0, SEEK_END);
            PositionType size = FTell(mFile);
            FSeek(mFile, 0, SEEK_SET);
            if (mUseMemory)
            {
                unsigned char* buf = new unsigned char[size];
                fread(buf, 1, size, mFile);
                SetBuffer(buf, size);
                fclose(mFile);
                mFile = 0;
            }
            else
            {
                SetSize(size);
            }
        }
    }

    virtual
    ~IOBufferFile()
    {
        CheckDataChannelWrite();
        if (mFile)
        {
            fclose(mFile);
        }
    }

    virtual bool
    Valid()
    {
        if (mReadMode && mUseMemory)
            return (GetBuffer() != 0);
        return (mFile != 0);
    }

    virtual void
    SetPosition(PositionType position)
    {
        if (GetPosition() == position)
            return;
        if (!mUseMemory)
            FSeek(mFile, position, SEEK_SET);
        IOBuffer::SetPosition(position);
    }

    virtual Int64
    Read(void* buf, Int64 bytesToRead)
    {
        Int64 available = Available();
        if (available < bytesToRead)
            bytesToRead = available;
        if (mUseMemory)
        {
            memcpy(buf, GetBuffer() + GetPosition(), bytesToRead);
        }
        else
        {
            FSeek(mFile, GetPosition(), SEEK_SET);
            Int64 nRead = fread(buf, 1, bytesToRead, mFile);
        }
        SetPosition(GetPosition() + bytesToRead);
        return bytesToRead;
    }

    virtual String
    ReadLine()
    {
        if (mUseMemory)
            return IOBuffer::ReadLine();

        // todo : this is a very inefficient implementation
        Int64 available = Available();
        Int64 maxLineSize = 1048576; // 1 Mb, todo : give user control
        if (available < maxLineSize)
            maxLineSize = available + 1; // extra space for '\0'
        // RvB: In debug mode this caused waiting time of up to 8 minutes
        //char* buf = new char[maxLineSize];
        static char buf[1024 * 1024];
        FSeek(mFile, GetPosition(), SEEK_SET);
        fgets(buf, maxLineSize, mFile);
        SetPosition(FTell(mFile));
        for (int i=0 ; i<maxLineSize ; i++)
        {
            if ((buf[i] == '\r') || (buf[i] == '\n'))
            {
                buf[i] = '\0';
                break;
            }
        }
        String res(buf);
        //delete buf;
        return res;
    }

    virtual Int64
    Gets(char* buf, Int64 bytesToRead)
    {
        if (mUseMemory)
            return IOBuffer::Gets(buf, bytesToRead);

        Int64 available = Available();
        if (available < bytesToRead)
            bytesToRead = available;
        FSeek(mFile, GetPosition(), SEEK_SET);
        fgets(buf, bytesToRead, mFile);
        SetPosition(FTell(mFile));
        return bytesToRead;
    }

    virtual void
    NativeTypeRead(Int8* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt8* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Int16* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt16* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Int32* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt32* ptr)
    {
        DoNativeTypeRead(ptr);
    }
    
    virtual void
    NativeTypeRead(Int64* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(UInt64* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Real32* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual void
    NativeTypeRead(Real64* ptr)
    {
        DoNativeTypeRead(ptr);
    }

    virtual Int64
    Write(const void* buf, Int64 bytesToWrite)
    {
        size_t bytesWritten = fwrite(buf, 1, bytesToWrite, mFile);
        if (bytesWritten != bytesToWrite)
            ILOG_ERROR("Bytes written to file (" << bytesWritten << ") " <<
            "differs from bytes to write (" << bytesToWrite << ")");
        SetPositionAndSize(FTell(mFile));
        return bytesWritten;
    }

    virtual Int64
    Puts(const char* buf)
    {
        fprintf(mFile, "%s\n", buf);
        SetPositionAndSize(FTell(mFile));
        return 1;
    }

    virtual void
    NativeTypeWrite(Int8 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt8 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Int16 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt16 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Int32 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt32 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Int64 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(UInt64 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Real32 val)
    {
        DoNativeTypeWrite(val);
    }

    virtual void
    NativeTypeWrite(Real64 val)
    {
        DoNativeTypeWrite(val);
    }

private:

    static FILE*
    Open(CString path, CString mode)
    {
        String thePath = path;
#ifdef WIN32
        thePath = StringReplaceAll(thePath, "/", "\\");
        thePath = StringReplaceAll(thePath, "\\\\", "\\", true, 2);
        thePath = StringReplaceAll(thePath, "\\.\\", "\\"); // for _fullpath
        bool isAbs = (thePath.at(0) == '\\') || (thePath.find(":") != String::npos);
        int pathSize = thePath.size();

        static const long maxSimplePath = _MAX_PATH;
        if ((pathSize > maxSimplePath) || !isAbs)
        {
            if (!isAbs)
            {
                if (pathSize > maxSimplePath)
                {
                    ILOG_ERROR("Relative path over " << maxSimplePath << 
                               " characters long: " << thePath);
                    return NULL;
                }
                static const int relPathWarnLength = maxSimplePath / 2;
                if (pathSize > relPathWarnLength)
                    ILOG_WARN("Relative path over " << relPathWarnLength << 
                               " characters long; it may not be possible to" << 
                               " resolve the absolute path for: " << thePath);

                // _fullpath cannot deal with long (relative) paths;
                // to extend the range of paths we are able to handle we apply
                // _fullpath to only part of the relative path (if possible)
                String pathHead = thePath;
                String pathTail = "";
                int firstSepPos = thePath.find("\\");
                if (firstSepPos != String::npos)
                {
                    int secondSepPos = thePath.find("\\", firstSepPos + 1);
                    if (secondSepPos != String::npos)
                    {
                        pathHead = thePath.substr(0, secondSepPos);
                        pathTail = thePath.substr(secondSepPos);
                    }
                }
                char absPath[maxSimplePath + 1];
                if (!_fullpath(absPath, pathHead.c_str(), maxSimplePath + 1))
                {
                    ILOG_ERROR("Failed to compose absolute path for " << 
                               thePath << " (" << strerror(errno) << ")");
                    return NULL;
                }
                thePath = String(absPath) + pathTail;
            }

            static const String longPathPrefix = "\\\\?\\";
            String longPath = longPathPrefix;
            if (thePath.substr(0, 2) == "\\\\")
                longPath += "UNC" + thePath.substr(1);
            else
                longPath += thePath;
            // Note that this long path specification does not support '..\'
            // (nor '.\'), but that we currently do not check for this!

            static const int maxLongPath = 32 * 1024;
            wchar_t longPathWC[maxLongPath + 1]; // reserve one for null char
            for (int i = 0; i < longPath.size(); i++)
                longPathWC[i] = (wchar_t) longPath.at(i);
            longPathWC[longPath.size()] = L'\0';
            wchar_t modeWC[10];
            for (int i = 0; i < mode.size(); i++)
                modeWC[i] = (wchar_t) mode.at(i);
            modeWC[mode.size()] = L'\0';
            FILE* fp = _wfopen(longPathWC, modeWC); // non-relative paths only
            return fp;
        } // if ..
#endif
        return fopen(thePath.c_str(), mode.c_str());
    }

    template <class NativeType>
    void
    DoNativeTypeRead(NativeType* ptr)
    {
        if (mUseMemory)
            return IOBuffer::DoNativeTypeRead(ptr);

        String fs = NativeTypeFormat<NativeType>(0);
        FSeek(mFile, GetPosition(), SEEK_SET);
        fscanf(mFile, fs.c_str(), ptr);
        SetPosition(FTell(mFile));
    }

    template <class NativeType>
    void
    DoNativeTypeWrite(NativeType val)
    {
        String fs = NativeTypeFormat<NativeType>(0) + " ";
        if (mUseMemory)
        {
            char* start = (char*) (GetBuffer() + GetPosition());
            sprintf(start, fs.c_str(), val);
            Int64 i = 0;
            while (start[i] != ' ')
                i++;
            SetPosition(GetPosition() + i + 1);
        }
        else
        {
            fprintf(mFile, fs.c_str(), val);
            SetPositionAndSize(FTell(mFile));
        }
    }

    PositionType
    FTell(FILE* fp)
    {
#ifdef WIN32
#if _MSC_VER <= 1310
        return ftell(fp);
#else
        return _ftelli64(fp);
#endif
#else
        return ftello(fp);
#endif
}

    int
    FSeek(FILE* fp, PositionType offset, int origin)
    {
#ifdef WIN32
#if _MSC_VER <= 1310
        return fseek(fp, offset, origin);
#else
        return _fseeki64(fp, offset, origin);
#endif
#else
        return fseeko(fp, offset, origin);
#endif
    }

    bool  mReadMode;
    bool  mUseMemory;
    FILE* mFile;

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(IOBufferFile, Impala.Util);

} // namespace Util
} // namespace Impala

#endif

---