|| || || || || ||

---

ReadRaw.h

Go to the documentation of this file.

#ifndef Impala_Core_Array_ReadRaw_h
#define Impala_Core_Array_ReadRaw_h

#include <vector>
#include "Persistency/File.h"
#include "Util/Database.h"
#include "Core/Array/Element/TypeString.h"
#include "Basis/NativeTypeRead.h"
#include "Core/Array/ArrayType.h"
#include "Core/Array/Arrays.h"
#include "Core/Array/Endian.h"

namespace Impala
{
namespace Core
{
namespace Array
{


inline ArrayType
ReadRawArrayType(Util::IOBuffer* buffer)
{
    ILOG_VAR(Impala.Core.Array.ReadRawArrayType);
    char buf[200];
    buffer->Read(buf, 200);
    String line(buf);

    // parse type:
    String::size_type p = line.find("type:");
    if (p == String::npos)
    {
        ILOG_ERROR("type not found");
        return ARRAY2DUNKNOWN;
    }
    p += 6;
    String::size_type p2 = line.find(',', p);
    String type = line.substr(p, p2-p);

    // parse elemSize:
    p = line.find("elemSize:");
    if (p == String::npos)
    {
        ILOG_ERROR("elemSize not found");
        return ARRAY2DUNKNOWN;
    }
    p += 10;
    p2 = line.find(',', p);
    String elemSizeStr = line.substr(p, p2-p);
    Int64 elemSize = atoi(elemSizeStr);

    // translate to ArrayType
    switch (elemSize)
    {
    case 1:
        if (type == "uint8")
            return ARRAY2DSCALARUINT8;
        if (type == "int32")
            return ARRAY2DSCALARINT32;
        if (type == "uint64")
            return ARRAY2DSCALARUINT64;
        if (type == "real32")
            return ARRAY2DSCALARREAL32;
        if (type == "real64")
            return ARRAY2DSCALARREAL64;
        break;
    case 2:
        if (type == "real64")
            return ARRAY2DVEC2REAL64;
        break;
    case 3:
        if (type == "uint8")
            return ARRAY2DVEC3UINT8;
        if (type == "real64")
            return ARRAY2DVEC3REAL64;
        break;
    }
    ILOG_ERROR("unknown type " << type << " elemSize " << elemSize);
    return ARRAY2DUNKNOWN;
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
inline ArrayType
ReadRawArrayType(String fileName, Util::Database* db)
{
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "", 0, true);
    ArrayType res = ReadRawArrayType(buf);
    delete buf;
    return res;
}
#endif // REPOSITORY_USED

inline ArrayType
ReadRawArrayType(Persistency::File file)
{
    Util::IOBuffer* buf = file.GetReadBuffer();
    ArrayType res = ReadRawArrayType(buf);
    delete buf;
    return res;
}

template<class ArrayT>
inline bool
ReadRawHeader(Util::IOBuffer* ioBuf, Int64* version, Int64* binary,
              Int64* elemSize, Int64* width, Int64* height, Int64* nrA,
              String& compressor,
              Int64* linesPerCompressedBlock)
{
    typedef typename ArrayT::StorType StorT;

    ILOG_VAR(Impala.Core.Array.ReadRawHeader);
    char buf[200];
    for (int i=0 ; i<200 ; i++)
        buf[i] = 0;
    ioBuf->Gets(buf, 200);
    int res = sscanf(buf, "version: %lld", version);
    if (res != 1)
    {
        ILOG_ERROR("Could not determine version");
        return false;
    }

    *linesPerCompressedBlock = 0;
    String typeStr = Element::TypeString<StorT>(0);
    if (*version == 1)
    {
        String formatStr = "version: 1, binary: %lld, type: " + typeStr +
            ", elemSize: %lld, width: %lld, height: %lld, nr: %lld";
        int res = sscanf(buf, formatStr.c_str(),
                         binary, elemSize, width, height, nrA);
        if ((res != 5) || (*elemSize != ArrayT::ElemSize()))
        {
            ILOG_ERROR("Header type 1 is not compatible: res==" << res <<
                       " (should be 5), elemSize==" << elemSize << " (should be "
                       << ArrayT::ElemSize() << "), buf==[" << buf << "]");
            return false;
        }
        return true;
    }
    if (*version == 2)
    {
        String formatStr = "version: 2, binary: %lld, type: " + typeStr +
            ", elemSize: %lld, nr: %lld";
        int res = sscanf(buf, formatStr.c_str(), binary, elemSize, nrA);
        if ((res != 3) || (*elemSize != ArrayT::ElemSize()))
        {
            ILOG_ERROR("Header type 2 is not compatible: res==" << res <<
                       " (should be 3), elemSize==" << elemSize << " (should be "
                       << ArrayT::ElemSize() << "), buf==[" << buf << "]");
            return false;
        }
        return true;
    }
    if (*version == 3)
    {
        char compressor2[5];
        String formatStr = "version: 3, binary: %lld, type: " + typeStr +
            ", elemSize: %lld, width: %lld, height: %lld, nr: %lld, %4s: %lld";
        int res = sscanf(buf, formatStr.c_str(),
                         binary, elemSize, width, height, nrA, compressor2, 
                         linesPerCompressedBlock);
        compressor2[4] = '\0';
        compressor = String(compressor2);
        if ((res != 7) || (*elemSize != ArrayT::ElemSize()))
        {
            ILOG_ERROR("Header type 3 is not compatible: res==" << res <<
                       " (should be 6), elemSize==" << elemSize << " (should be "
                       << ArrayT::ElemSize() << "), buf==[" << buf << "]");
            return false;
        }
        return true;
    }
    ILOG_ERROR("Unknown version " << *version);
    return false;
}

template<class ArrayT>
inline void
ReadRaw(ArrayT*& dst, Util::IOBuffer* buffer)
{
    typedef typename ArrayT::StorType StorT;

    ILOG_VAR(Impala.Core.Array.ReadRaw);
    Int64 version;
    Int64 binary;
    Int64 elemSize;
    Int64 width;
    Int64 height;
    Int64 nrA;
    String compressor;
    Int64 linesPerCompressedBlock;
    Util::IOBuffer::PositionType startPos = buffer->GetPosition();
    if (! ReadRawHeader<ArrayT>(buffer, &version, &binary, &elemSize, &width,
                                &height, &nrA, compressor, 
                                &linesPerCompressedBlock))
    {
        return;
    }

    if (dst == 0)
        dst = ArrayCreate<ArrayT>(width, height);
    StorT* dstPtr = dst->CPB(0, 0);
    if (binary)
    {
        Int64 lineSize = sizeof(StorT) * width * elemSize;
        buffer->SetPosition(startPos + 200);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            if (linesPerCompressedBlock > 0)
            {
                int y = 0;
                while (y < height)
                {
                    int end = y + linesPerCompressedBlock;
                    if (end > height)
                        end = height;
                    int dY = end - y;
                    dstPtr = dst->CPB(0, y);
                    size_t nrBytes = lineSize * dY;
                    size_t nrRead = 0;
                    if(compressor == "bzip")
                    {
                        nrRead = buffer->ReadBzip(dstPtr, nrBytes);
                    }
                    else if(compressor == "zlib")
                    {
                        nrRead = buffer->ReadZlib(dstPtr, nrBytes);
                    }
                    else
                    {
                        ILOG_ERROR("Unknown compressor: " << compressor);
                    }
                    if (nrRead != nrBytes)
                    {
                        ILOG_ERROR("read " << nrRead << " bytes instead of "
                                   << nrBytes);
                    }
                    y += dY;
                }
            }
            else
            {
                size_t nrBytes = lineSize * height;
                size_t nrRead = buffer->Read(dstPtr, nrBytes);
                if (nrRead != nrBytes)
                {
                    ILOG_ERROR("read " << nrRead << " bytes instead of "
                               << nrBytes);
                }
            }
        }
        Endian(dst, dst);
    }
    else
    {
        for (Int64 n=0 ; n<nrA ; n++)
        {
            for (Int64 i=0 ; i<height ; i++)
            {
                for (Int64 j=0 ; j<width ; j++)
                {
                    for (Int64 k=0 ; k<elemSize ; k++)
                    {
                        buffer->NativeTypeRead(dstPtr);
                        dstPtr++;
                    }
                }
            }
        }
        buffer->ReadLine();
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
ReadRaw(ArrayT*& dst, String fileName, Util::Database* db)
{
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "", 0, true);
    if (buf)
    {
        ReadRaw(dst, buf);
        delete buf;
    }
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
ReadRaw(ArrayT*& dst, Persistency::File file)
{
    Util::IOBuffer* buf = file.GetReadBuffer();
    if (buf)
    {
        ReadRaw(dst, buf);
        delete buf;
    }
}

template<class ArrayT>
inline void
ReadRawList(std::vector<ArrayT*>& list, Util::IOBuffer* buffer)
{
    typedef typename ArrayT::StorType StorT;
    ILOG_VAR(Impala.Core.Array.ReadRawList);

    Int64 version;
    Int64 binary;
    Int64 elemSize;
    Int64 width;
    Int64 height;
    Int64 nrA;
    String compressor;
    Int64 linesPerCompressedBlock;
    Util::IOBuffer::PositionType startPos = buffer->GetPosition();
    if (! ReadRawHeader<ArrayT>(buffer, &version, &binary, &elemSize, &width,
                                &height, &nrA, compressor,
                                &linesPerCompressedBlock))
    {
        return;
    }

    for (Int64 n=0 ; n<nrA ; n++)
        if (n+1 > list.size())
            list.push_back(ArrayCreate<ArrayT>(width, height));
    if (binary)
    {
        Int64 lineSize = sizeof(StorT) * width * elemSize;
        buffer->SetPosition(startPos + 200);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            if (linesPerCompressedBlock > 0)
            {
                int y = 0;
                while (y < height)
                {
                    int end = y + linesPerCompressedBlock;
                    if (end > height)
                        end = height;
                    int dY = end - y;
                    StorT* dstPtr = list[n]->CPB(0, y);
                    size_t nrBytes = lineSize * dY;
                    size_t nrRead = 0;
                    if(compressor == "bzip")
                    {
                        nrRead = buffer->ReadBzip(dstPtr, nrBytes);
                    }
                    else if(compressor == "zlib")
                    {
                        nrRead = buffer->ReadZlib(dstPtr, nrBytes);
                    }
                    else
                    {
                        ILOG_ERROR("Unknown compressor: " << compressor);
                    }
                    if (nrRead != nrBytes)
                    {
                        ILOG_ERROR("read " << nrRead << " bytes instead of "
                                   << nrBytes);
                    }
                    y += dY;
                }
            }
            else
            {
                StorT* dstPtr = list[n]->CPB(0, 0);
                size_t nrBytes = lineSize * height;
                size_t nrRead = buffer->Read(dstPtr, nrBytes);
                if (nrRead != nrBytes)
                {
                    ILOG_ERROR("read " << nrRead << " bytes instead of "
                               << nrBytes);
                }
            }
            Endian(list[n], list[n]);
        }
    }
    else
    {
        for (Int64 n=0 ; n<nrA ; n++)
        {
            for (Int64 i=0 ; i<height ; i++)
            {
                StorT* dstPtr = list[n]->CPB(0, i);
                for (Int64 j=0 ; j<width ; j++)
                {
                    for (Int64 k=0 ; k<elemSize ; k++)
                    {
                        buffer->NativeTypeRead(dstPtr);
                        dstPtr++;
                    }
                }
            }
        }
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
ReadRawList(std::vector<ArrayT*>& list, String fileName, Util::Database* db)
{
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "", 0, true);
    if (buf)
    {
        ReadRawList(list, buf);
        delete buf;
    }
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
ReadRawList(std::vector<ArrayT*>& list, Persistency::File file)
{
    Util::IOBuffer* buf = file.GetReadBuffer();
    if (buf)
    {
        ReadRawList(list, buf);
        delete buf;
    }
}

template<class ArrayT>
inline void
ReadRawListVar(std::vector<ArrayT*>& list, Util::IOBuffer* buffer)
{
    typedef typename ArrayT::StorType StorT;

    ILOG_VAR(Impala.Core.Array.ReadRawListVar);
    Int64 ver;
    Int64 binary;
    Int64 elemSize;
    Int64 nrA;
    String compressor;
    Int64 linesPerCompressedBlock;
    Util::IOBuffer::PositionType startPos = buffer->GetPosition();
    if (! ReadRawHeader<ArrayT>(buffer, &ver, &binary, &elemSize, 0, 0, &nrA, 
                                compressor,
                                &linesPerCompressedBlock))
    {
        return;
    }
    if (binary)
    {
        buffer->SetPosition(startPos + 200);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            Int64 width;
            Int64 height;
            char buf[20];
            buffer->Read(buf, 20);
            sscanf(buf, "%lld %lld ", &width, &height);
            size_t blockSize = width*height*elemSize;
            ArrayT* a = ArrayCreate<ArrayT>(width, height);
            StorT* dstPtr = a->CPB(0, 0);
            size_t nrRead = buffer->Read(dstPtr, sizeof(StorT) * blockSize);
            if (nrRead != sizeof(StorT) * blockSize)
            {
                ILOG_ERROR("read " << nrRead << " bytes instead of "
                           << sizeof(StorT) * blockSize);
            }
            Endian(a, a);
            list.push_back(a);
        }
    }
    else
    {
        for (Int64 n=0 ; n<nrA ; n++)
        {
            Int64 width;
            Int64 height;
            buffer->NativeTypeRead(&width);
            buffer->NativeTypeRead(&height);
            Int64 blockSize = width*height*elemSize;
            ArrayT* a = ArrayCreate<ArrayT>(width, height);
            for(Int64 i=0 ; i<height ; i++)
            {
                StorT* dstPtr = a->CPB(0, i);
                for(Int64 j=0 ; j<width ; j++)
                {
                    for (Int64 k=0 ; k<elemSize ; k++)
                    {
                        buffer->NativeTypeRead(dstPtr);
                        dstPtr++;
                    }
                }
            }
            list.push_back(a);
        }
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
ReadRawListVar(std::vector<ArrayT*>& list, String fileName, Util::Database* db)
{
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "", 0, true);
    if (buf)
    {
        ReadRawListVar(list, buf);
        delete buf;
    }
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
ReadRawListVar(std::vector<ArrayT*>& list, Persistency::File file)
{
    Util::IOBuffer* buf = file.GetReadBuffer();
    if (buf)
    {
        ReadRawListVar(list, buf);
        delete buf;
    }
}

// create an index for beginning of array data within file, used in ImageSet
template<class ArrayT>
inline void
ReadRawListVarIndex(std::vector<Int64>& list, Util::IOBuffer* buffer)
{
    typedef typename ArrayT::StorType StorT;
  
    ILOG_VAR(Impala.Core.Array.ReadRawListVarIndex);
    String typeStr = Element::TypeString<StorT>(0);
    String formatStr = "version: 2, binary: %lld, type: " + typeStr +
                       ", elemSize: %lld, nr: %lld";

    Int64 binary;
    Int64 elemSize;
    Int64 nrA;
    char buf[200];
    Util::IOBuffer::PositionType startPos = buffer->GetPosition();
    buffer->Gets(buf, 200);
    int res = sscanf(buf, formatStr.c_str(), &binary, &elemSize, &nrA);
    if ((res != 3) || (elemSize != ArrayT::ElemSize()))
    {
        ILOG_ERROR("Header is not compatible");
        return;
    }
    if (binary)
    {
        buffer->SetPosition(startPos + 200);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            Int64 curPos = buffer->GetPosition();
            list.push_back(curPos);
            Int64 width;
            Int64 height;
            buffer->Read(buf, 20);
            sscanf(buf, "%lld %lld ", &width, &height);
            size_t blockSize = width*height*elemSize;
            buffer->SetPosition(buffer->GetPosition() + blockSize);
        }
    }
    else
    {
        ILOG_ERROR("mode not implemented");
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
ReadRawListVarIndex(std::vector<Int64>& list, String fileName, Util::Database* db)
{
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, true, false, "", 0, true);
    if (buf)
    {
        ReadRawListVarIndex<ArrayT>(list, buf);
        delete buf;
    }
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
ReadRawListVarIndex(std::vector<Int64>& list, Persistency::File file)
{
    Util::IOBuffer* buf = file.GetReadBuffer();
    if (buf)
    {
        ReadRawListVarIndex<ArrayT>(list, buf);
        delete buf;
    }
}

} // namespace Array
} // namespace Core
} // namespace Impala

#endif

---