00001 #ifndef Impala_Core_Array_WriteRaw_h
00002 #define Impala_Core_Array_WriteRaw_h
00003
00004 #include <vector>
00005 #include "Persistency/File.h"
00006 #include "Util/DatabaseWriteNative.h"
00007 #include "Core/Array/Element/TypeString.h"
00008 #include "Core/Array/ArraySet.h"
00009 #include "Core/Array/Endian.h"
00010 #include "Util/IOBuffer.h"
00011
00012 namespace Impala
00013 {
00014 namespace Core
00015 {
00016 namespace Array
00017 {
00018
00019
00020
00021 template<class ArrayT>
00022 inline void
00023 WriteRaw(ArrayT* src, Util::IOBuffer* buffer, int mode = 1)
00024 {
00025 typedef typename ArrayT::StorType StorT;
00026 ILOG_VAR(Impala.Core.Array.WriteRaw);
00027
00028 if (mode == 0)
00029 {
00030 ILOG_ERROR("Non-binary write has been eliminated");
00031 return;
00032 }
00033
00034 #ifndef BZIP2_USED
00035 if (mode == 2)
00036 mode = 1;
00037 #endif // BZIP2_USED
00038 #ifndef ZLIB_USED
00039 if (mode == 3)
00040 mode = 1;
00041 #endif //ZLIB_USED
00042
00043 String typeStr = Element::TypeString<StorT>(0);
00044 String formatStr;
00045 if (mode == 2)
00046 {
00047 formatStr = "version: 3, binary: 1, type: " + typeStr +
00048 ", elemSize: %lld, width: %lld, height: %lld, nr: %lld, bzip: %lld";
00049 }
00050 else if (mode == 3)
00051 {
00052 formatStr = "version: 3, binary: 1, type: " + typeStr +
00053 ", elemSize: %lld, width: %lld, height: %lld, nr: %lld, zlib: %lld";
00054 }
00055 else
00056 {
00057 formatStr = "version: 1, binary: 1, type: " + typeStr +
00058 ", elemSize: %lld, width: %lld, height: %lld, nr: %lld";
00059 }
00060
00061 Int64 width = src->CW();
00062 Int64 height = src->CH();
00063 Int64 elemSize = ArrayT::ElemSize();
00064 Int64 nrA = 1;
00065 Int64 lineSize = sizeof(StorT) * width * elemSize;
00066 Int64 bzipI = 0;
00067 if (mode > 1)
00068 {
00069 bzipI = Util::IOBuffer::cBzipBlockSize / lineSize;
00070 if (src->BW() != 0)
00071 bzipI = 1;
00072 }
00073 char buf[200];
00074 memset(buf, 0, 200);
00075 if (mode > 1)
00076 sprintf(buf, formatStr.c_str(), elemSize, width, height, nrA, bzipI);
00077 else
00078 sprintf(buf, formatStr.c_str(), elemSize, width, height, nrA);
00079 buffer->Write(buf, 200);
00080
00081 Endian(src, src);
00082 if (bzipI > 0)
00083 {
00084 int y = 0;
00085 while (y < height)
00086 {
00087 int end = y + bzipI;
00088 if (end > height)
00089 end = height;
00090 int dY = end - y;
00091 StorT* srcPtr = src->CPB(0, y);
00092 size_t nrBytes = lineSize * dY;
00093 if (mode == 3)
00094 buffer->WriteZlib(srcPtr, nrBytes);
00095 else
00096 buffer->WriteBzip(srcPtr, nrBytes);
00097 y += dY;
00098 }
00099 }
00100 else
00101 {
00102 if (src->BW() == 0)
00103 {
00104 StorT* srcPtr = src->CPB(0, 0);
00105 size_t nrBytes = lineSize * height;
00106 buffer->Write(srcPtr, nrBytes);
00107 }
00108 else
00109 {
00110 for (Int64 y=0 ; y<height ; y++)
00111 {
00112 StorT* srcPtr = src->CPB(0, y);
00113 buffer->Write(srcPtr, lineSize);
00114 }
00115 }
00116 }
00117 }
00118
00119 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00120 template<class ArrayT>
00121 inline void
00122 WriteRaw(ArrayT* src, String fileName, Util::Database* db, int mode)
00123 {
00124
00125 Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp", 0, true);
00126 if (buf)
00127 {
00128 WriteRaw(src, buf, mode);
00129 delete buf;
00130 }
00131 }
00132 #endif // REPOSITORY_USED
00133
00134 template<class ArrayT>
00135 inline void
00136 WriteRaw(ArrayT* src, Persistency::File file, int mode)
00137 {
00138 Util::IOBuffer* buf = file.GetWriteBuffer();
00139 if (buf)
00140 {
00141 WriteRaw(src, buf, mode);
00142 delete buf;
00143 }
00144 }
00145
00146 template<class ArrayT>
00147 inline void
00148 WriteRawList(std::vector<ArrayT*> list, Util::IOBuffer* buffer, bool binary)
00149 {
00150 typedef typename ArrayT::StorType StorT;
00151
00152 String typeStr = Element::TypeString<StorT>(0);
00153 String formatStr = "version: 1, binary: %d, type: " + typeStr +
00154 ", elemSize: %lld, width: %lld, height: %lld, nr: %lld";
00155 Int64 width = list[0]->CW();
00156 Int64 height = list[0]->CH();
00157 Int64 elemSize = ArrayT::ElemSize();
00158 Int64 nrA = list.size();
00159 char buf[200];
00160 memset(buf, 0, 200);
00161 sprintf(buf, formatStr.c_str(), binary, elemSize, width, height, nrA);
00162
00163 if (binary)
00164 {
00165 buffer->Write(buf, 200);
00166 for (Int64 n=0 ; n<nrA ; n++)
00167 {
00168 Endian(list[n], list[n]);
00169 for (Int64 y=0 ; y<height ; y++)
00170 {
00171 StorT* srcPtr = list[n]->CPB(0, y);
00172 buffer->Write(srcPtr, sizeof(StorT) * width * elemSize);
00173 }
00174 }
00175 }
00176 else
00177 {
00178 buffer->Puts(buf);
00179 for (Int64 n=0 ; n<nrA ; n++)
00180 {
00181 for(Int64 i=0 ; i<height ; i++)
00182 {
00183 StorT* srcPtr = list[n]->CPB(0, i);
00184 for(Int64 j=0 ; j<width ; j++)
00185 for (Int64 k=0 ; k<elemSize ; k++)
00186 buffer->NativeTypeWrite(*srcPtr++);
00187 buffer->Puts("");
00188 }
00189 }
00190 }
00191 }
00192
00193 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00194 template<class ArrayT>
00195 inline void
00196 WriteRawList(std::vector<ArrayT*> list, String fileName, Util::Database* db,
00197 bool binary)
00198 {
00199
00200 Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp", 0, true);
00201 if (buf)
00202 {
00203 WriteRawList(list, buf, binary);
00204 delete buf;
00205 }
00206 }
00207 #endif // REPOSITORY_USED
00208
00209 template<class ArrayT>
00210 inline void
00211 WriteRawList(std::vector<ArrayT*> list, Persistency::File file, bool binary)
00212 {
00213 Util::IOBuffer* buf = file.GetWriteBuffer();
00214 if (buf)
00215 {
00216 WriteRawList(list, buf, binary);
00217 delete buf;
00218 }
00219 }
00220
00221 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00222
00223 template<class ArrayT>
00224 inline void
00225 WriteRawList(ArraySet<ArrayT>& aSet, String fileName, bool binary)
00226 {
00227 std::vector<ArrayT*> list;
00228 for (Int64 i=0 ; i<aSet.Size() ; i++)
00229 list.push_back(aSet.Array(i));
00230 WriteRawList(list, fileName, binary);
00231 }
00232 #endif // REPOSITORY_USED
00233
00234 template<class ArrayT>
00235 inline void
00236 WriteRawList(ArraySet<ArrayT>& aSet, Persistency::File file, bool binary)
00237 {
00238 std::vector<ArrayT*> list;
00239 for (Int64 i=0 ; i<aSet.Size() ; i++)
00240 list.push_back(aSet.Array(i));
00241 WriteRawList(list, file, binary);
00242 }
00243
00244 template<class ArrayT>
00245 inline void
00246 WriteRawListVar(std::vector<ArrayT*> list, Util::IOBuffer* buffer, bool binary)
00247 {
00248 typedef typename ArrayT::StorType StorT;
00249
00250 String typeStr = Element::TypeString<StorT>(0);
00251 String formatStr = "version: 2, binary: %d, type: " + typeStr +
00252 ", elemSize: %lld, nr: %lld";
00253 Int64 elemSize = ArrayT::ElemSize();
00254 Int64 nrA = list.size();
00255 char buf[200];
00256 memset(buf, 0, 200);
00257 sprintf(buf, formatStr.c_str(), binary, elemSize, nrA);
00258
00259 if (binary)
00260 {
00261 buffer->Write(buf, 200);
00262 for (Int64 n=0 ; n<nrA ; n++)
00263 {
00264 Endian(list[n], list[n]);
00265 Int64 width = list[n]->CW();
00266 Int64 height = list[n]->CH();
00267 memset(buf, 0, 200);
00268 sprintf(buf, "%lld %lld", width, height);
00269 buffer->Write(buf, 20);
00270 for (Int64 y=0 ; y<height ; y++)
00271 {
00272 StorT* srcPtr = list[n]->CPB(0, y);
00273 buffer->Write(srcPtr, sizeof(StorT) * width * elemSize);
00274 }
00275 }
00276 }
00277 else
00278 {
00279 buffer->Puts(buf);
00280 for (Int64 n=0 ; n<nrA ; n++)
00281 {
00282 Int64 width = list[n]->CW();
00283 Int64 height = list[n]->CH();
00284 buffer->NativeTypeWrite(width);
00285 buffer->NativeTypeWrite(height);
00286 for(Int64 i=0 ; i<height ; i++)
00287 {
00288 StorT* srcPtr = list[n]->CPB(0, i);
00289 for(Int64 j=0 ; j<width ; j++)
00290 for (Int64 k=0 ; k<elemSize ; k++)
00291 buffer->NativeTypeWrite(*srcPtr++);
00292 buffer->Puts("");
00293 }
00294 }
00295 }
00296 }
00297
00298 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00299 template<class ArrayT>
00300 inline void
00301 WriteRawListVar(std::vector<ArrayT*> list, String fileName, Util::Database* db,
00302 bool writeIndex, bool binary)
00303 {
00304 ILOG_VAR(Impala.Core.Array.WriteRawListVar);
00305
00306 Util::IOBuffer* buf = db->GetIOBuffer(fileName, false, false, "tmp", 0, true);
00307 if (!buf)
00308 return;
00309
00310 WriteRawListVar(list, buf, binary);
00311 delete buf;
00312
00313 if (!writeIndex)
00314 return;
00315 if (!binary)
00316 {
00317 ILOG_ERROR("Cannot write index for non-binary files");
00318 return;
00319 }
00320
00321 size_t storSize = sizeof(typename ArrayT::StorType);
00322 Int64 elemSize = ArrayT::ElemSize();
00323 std::vector<Int64> v;
00324 Int64 pos = 200;
00325 for (Int64 i=0 ; i<list.size() ; i++)
00326 {
00327 v.push_back(pos);
00328 pos += 20 + storSize * list[i]->CW() * list[i]->CH() * elemSize;
00329 }
00330 Util::DatabaseWriteNative(fileName + ".idx", db, v.begin(), v.end());
00331 }
00332 #endif // REPOSITORY_USED
00333
00334 template<class ArrayT>
00335 inline void
00336 WriteRawListVar(std::vector<ArrayT*> list, Persistency::File file,
00337 bool writeIndex, bool binary)
00338 {
00339 ILOG_VAR(Impala.Core.Array.WriteRawListVar);
00340 Util::IOBuffer* buf = file.GetWriteBuffer();
00341 if (!buf)
00342 return;
00343
00344 WriteRawListVar(list, buf, binary);
00345 delete buf;
00346
00347 if (!writeIndex)
00348 return;
00349 if (!binary)
00350 {
00351 ILOG_ERROR("Cannot write index for non-binary files");
00352 return;
00353 }
00354
00355 size_t storSize = sizeof(typename ArrayT::StorType);
00356 Int64 elemSize = ArrayT::ElemSize();
00357 std::vector<Int64> v;
00358 Int64 pos = 200;
00359 for (Int64 i=0 ; i<list.size() ; i++)
00360 {
00361 v.push_back(pos);
00362 pos += 20 + storSize * list[i]->CW() * list[i]->CH() * elemSize;
00363 }
00364 Persistency::File idxFile(file.GetPath() + ".idx", file.GetFileSystem(),
00365 file.IsWritable());
00366 idxFile.WriteNative(v.begin(), v.end());
00367 }
00368
00369 }
00370 }
00371 }
00372
00373 #endif