|| || || || || ||

---

WriteRaw.h

Go to the documentation of this file.

#ifndef Impala_Core_Array_WriteRaw_h
#define Impala_Core_Array_WriteRaw_h

#include <vector>
#include "Persistency/File.h"
#include "Util/DatabaseWriteNative.h"
#include "Core/Array/Element/TypeString.h"
#include "Core/Array/ArraySet.h"
#include "Core/Array/Endian.h"
#include "Util/IOBuffer.h"

namespace Impala
{
namespace Core
{
namespace Array
{


// mode : 0=ascii, 1=binary, 2=bzip, 3=zlib
template<class ArrayT>
inline void
WriteRaw(ArrayT* src, Util::IOBuffer* buffer, int mode = 1)
{
    typedef typename ArrayT::StorType StorT;
    ILOG_VAR(Impala.Core.Array.WriteRaw);

    if (mode == 0)
    {
        ILOG_ERROR("Non-binary write has been eliminated");
        return;
    }

#ifndef BZIP2_USED
    if (mode == 2)
        mode = 1;
#endif // BZIP2_USED
#ifndef ZLIB_USED
    if (mode == 3)
        mode = 1;
#endif //ZLIB_USED

    String typeStr = Element::TypeString<StorT>(0);
    String formatStr;
    if (mode == 2)
    {
        formatStr = "version: 3, binary: 1, type: " + typeStr +
            ", elemSize: %lld, width: %lld, height: %lld, nr: %lld, bzip: %lld";
    }
    else if (mode == 3)
    {
        formatStr = "version: 3, binary: 1, type: " + typeStr +
            ", elemSize: %lld, width: %lld, height: %lld, nr: %lld, zlib: %lld";
    }
    else
    {
        formatStr = "version: 1, binary: 1, type: " + typeStr +
            ", elemSize: %lld, width: %lld, height: %lld, nr: %lld";
    }

    Int64 width = src->CW();
    Int64 height = src->CH();
    Int64 elemSize = ArrayT::ElemSize();
    Int64 nrA = 1;
    Int64 lineSize = sizeof(StorT) * width * elemSize;
    Int64 bzipI = 0;
    if (mode > 1)
    {
        bzipI = Util::IOBuffer::cBzipBlockSize / lineSize;
        if (src->BW() != 0)
            bzipI = 1;
    }
    char buf[200];
    memset(buf, 0, 200);
    if (mode > 1)
        sprintf(buf, formatStr.c_str(), elemSize, width, height, nrA, bzipI);
    else
        sprintf(buf, formatStr.c_str(), elemSize, width, height, nrA);
    buffer->Write(buf, 200);

    Endian(src, src);
    if (bzipI > 0)
    {
        int y = 0;
        while (y < height)
        {
            int end = y + bzipI;
            if (end > height)
                end = height;
            int dY = end - y;
            StorT* srcPtr = src->CPB(0, y);
            size_t nrBytes = lineSize * dY;
            if (mode == 3)
                buffer->WriteZlib(srcPtr, nrBytes);
            else
                buffer->WriteBzip(srcPtr, nrBytes);
            y += dY;
        }
    }
    else
    {
        if (src->BW() == 0)
        {
            StorT* srcPtr = src->CPB(0, 0);
            size_t nrBytes = lineSize * height;
            buffer->Write(srcPtr, nrBytes);
        }
        else
        {
            for (Int64 y=0 ; y<height ; y++)
            {
                StorT* srcPtr = src->CPB(0, y);
                buffer->Write(srcPtr, lineSize);
            }
        }
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
WriteRaw(ArrayT* src, String fileName, Util::Database* db, int mode)
{
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp", 0, true);
    if (buf)
    {
        WriteRaw(src, buf, mode);
        delete buf;
    }
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
WriteRaw(ArrayT* src, Persistency::File file, int mode)
{
    Util::IOBuffer* buf = file.GetWriteBuffer();
    if (buf)
    {
        WriteRaw(src, buf, mode);
        delete buf;
    }
}

template<class ArrayT>
inline void
WriteRawList(std::vector<ArrayT*> list, Util::IOBuffer* buffer, bool binary)
{
    typedef typename ArrayT::StorType StorT;

    String typeStr = Element::TypeString<StorT>(0);
    String formatStr = "version: 1, binary: %d, type: " + typeStr +
                       ", elemSize: %lld, width: %lld, height: %lld, nr: %lld";
    Int64 width = list[0]->CW();
    Int64 height = list[0]->CH();
    Int64 elemSize = ArrayT::ElemSize();
    Int64 nrA = list.size();
    char buf[200];
    memset(buf, 0, 200);
    sprintf(buf, formatStr.c_str(), binary, elemSize, width, height, nrA);

    if (binary)
    { // write binary data
        buffer->Write(buf, 200);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            Endian(list[n], list[n]);
            for (Int64 y=0 ; y<height ; y++)
            {
                StorT* srcPtr = list[n]->CPB(0, y);
                buffer->Write(srcPtr, sizeof(StorT) * width * elemSize);
            }
        }
    }
    else
    { // write text data
        buffer->Puts(buf);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            for(Int64 i=0 ; i<height ; i++)
            {
                StorT* srcPtr = list[n]->CPB(0, i);
                for(Int64 j=0 ; j<width ; j++)
                    for (Int64 k=0 ; k<elemSize ; k++)
                        buffer->NativeTypeWrite(*srcPtr++);
                buffer->Puts(""); // adds newline
            }
        }
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
WriteRawList(std::vector<ArrayT*> list, String fileName, Util::Database* db,
             bool binary)
{
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp", 0, true);
    if (buf)
    {
        WriteRawList(list, buf, binary);
        delete buf;
    }
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
WriteRawList(std::vector<ArrayT*> list, Persistency::File file, bool binary)
{
    Util::IOBuffer* buf = file.GetWriteBuffer();
    if (buf)
    {
        WriteRawList(list, buf, binary);
        delete buf;
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
// DK: this code seems to be broken and not used
template<class ArrayT>
inline void
WriteRawList(ArraySet<ArrayT>& aSet, String fileName, bool binary)
{
    std::vector<ArrayT*> list;
    for (Int64 i=0 ; i<aSet.Size() ; i++)
        list.push_back(aSet.Array(i));
    WriteRawList(list, fileName, binary);
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
WriteRawList(ArraySet<ArrayT>& aSet, Persistency::File file, bool binary)
{
    std::vector<ArrayT*> list;
    for (Int64 i=0 ; i<aSet.Size() ; i++)
        list.push_back(aSet.Array(i));
    WriteRawList(list, file, binary);
}

template<class ArrayT>
inline void
WriteRawListVar(std::vector<ArrayT*> list, Util::IOBuffer* buffer, bool binary)
{
    typedef typename ArrayT::StorType StorT;

    String typeStr = Element::TypeString<StorT>(0);
    String formatStr = "version: 2, binary: %d, type: " + typeStr +
                       ", elemSize: %lld, nr: %lld";
    Int64 elemSize = ArrayT::ElemSize();
    Int64 nrA = list.size();
    char buf[200];
    memset(buf, 0, 200);
    sprintf(buf, formatStr.c_str(), binary, elemSize, nrA);

    if (binary)
    { // write binary data
        buffer->Write(buf, 200);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            Endian(list[n], list[n]);
            Int64 width = list[n]->CW();
            Int64 height = list[n]->CH();
            memset(buf, 0, 200);
            sprintf(buf, "%lld %lld", width, height);
            buffer->Write(buf, 20);
            for (Int64 y=0 ; y<height ; y++)
            {
                StorT* srcPtr = list[n]->CPB(0, y);
                buffer->Write(srcPtr, sizeof(StorT) * width * elemSize);
            }
        }
    }
    else
    { // write text data
        buffer->Puts(buf);
        for (Int64 n=0 ; n<nrA ; n++)
        {
            Int64 width = list[n]->CW();
            Int64 height = list[n]->CH();
            buffer->NativeTypeWrite(width);
            buffer->NativeTypeWrite(height);
            for(Int64 i=0 ; i<height ; i++)
            {
                StorT* srcPtr = list[n]->CPB(0, i);
                for(Int64 j=0 ; j<width ; j++)
                    for (Int64 k=0 ; k<elemSize ; k++)
                        buffer->NativeTypeWrite(*srcPtr++);
                buffer->Puts(""); // adds newline
            }
        }
    }
}

#ifndef REPOSITORY_USED // Here comes the deprecated stuff
template<class ArrayT>
inline void
WriteRawListVar(std::vector<ArrayT*> list, String fileName, Util::Database* db,
                bool writeIndex, bool binary)
{
    ILOG_VAR(Impala.Core.Array.WriteRawListVar);
    //Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp");
    Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp", 0, true);
    if (!buf)
        return;

    WriteRawListVar(list, buf, binary);
    delete buf;

    if (!writeIndex)
        return;
    if (!binary)
    {
        ILOG_ERROR("Cannot write index for non-binary files");
        return;
    }

    size_t storSize = sizeof(typename ArrayT::StorType);
    Int64 elemSize = ArrayT::ElemSize();
    std::vector<Int64> v;
    Int64 pos = 200;
    for (Int64 i=0 ; i<list.size() ; i++)
    {
        v.push_back(pos);
        pos += 20 + storSize * list[i]->CW() * list[i]->CH() * elemSize;
    }
    Util::DatabaseWriteNative(fileName + ".idx", db, v.begin(), v.end());
}
#endif // REPOSITORY_USED

template<class ArrayT>
inline void
WriteRawListVar(std::vector<ArrayT*> list, Persistency::File file,
                bool writeIndex, bool binary)
{
    ILOG_VAR(Impala.Core.Array.WriteRawListVar);
    Util::IOBuffer* buf = file.GetWriteBuffer();
    if (!buf)
        return;

    WriteRawListVar(list, buf, binary);
    delete buf;

    if (!writeIndex)
        return;
    if (!binary)
    {
        ILOG_ERROR("Cannot write index for non-binary files");
        return;
    }

    size_t storSize = sizeof(typename ArrayT::StorType);
    Int64 elemSize = ArrayT::ElemSize();
    std::vector<Int64> v;
    Int64 pos = 200;
    for (Int64 i=0 ; i<list.size() ; i++)
    {
        v.push_back(pos);
        pos += 20 + storSize * list[i]->CW() * list[i]->CH() * elemSize;
    }
    Persistency::File idxFile(file.GetPath() + ".idx", file.GetFileSystem(),
                              file.IsWritable());
    idxFile.WriteNative(v.begin(), v.end());
}

} // namespace Array
} // namespace Core
} // namespace Impala

#endif

---