00001 #ifndef Impala_Util_IOBufferFile_h
00002 #define Impala_Util_IOBufferFile_h
00003
00004 #include <string>
00005
00006 #include "Basis/NativeTypeFormats.h"
00007 #include "Basis/String.h"
00008 #include "Util/IOBuffer.h"
00009
00010 namespace Impala
00011 {
00012 namespace Util
00013 {
00014
00015
00016 class IOBufferFile : public IOBuffer
00017 {
00018 public:
00019
00020 static bool
00021 FileExists(CString path)
00022 {
00023 FILE* fp = Open(path, "r");
00024 if (fp)
00025 {
00026 fclose(fp);
00027 return true;
00028 }
00029 return false;
00030 }
00031
00032 IOBufferFile(CString filename, bool readMode, bool useMemory)
00033 {
00034 if (!readMode && useMemory)
00035 {
00036 ILOG_ERROR("useMemory not supported for write mode");
00037 return;
00038 }
00039
00040 mReadMode = readMode;
00041 mUseMemory = useMemory;
00042
00043 mFile = Open(filename, (mReadMode ? "rb" : "w+b"));
00044 if (!mFile)
00045 {
00046 ILOG_ERROR("Failed to open " << filename);
00047 return;
00048 }
00049
00050 if (mReadMode)
00051 {
00052 FSeek(mFile, 0, SEEK_END);
00053 PositionType size = FTell(mFile);
00054 FSeek(mFile, 0, SEEK_SET);
00055 if (mUseMemory)
00056 {
00057 unsigned char* buf = new unsigned char[size];
00058 fread(buf, 1, size, mFile);
00059 SetBuffer(buf, size);
00060 fclose(mFile);
00061 mFile = 0;
00062 }
00063 else
00064 {
00065 SetSize(size);
00066 }
00067 }
00068 }
00069
00070 virtual
00071 ~IOBufferFile()
00072 {
00073 CheckDataChannelWrite();
00074 if (mFile)
00075 {
00076 fclose(mFile);
00077 }
00078 }
00079
00080 virtual bool
00081 Valid()
00082 {
00083 if (mReadMode && mUseMemory)
00084 return (GetBuffer() != 0);
00085 return (mFile != 0);
00086 }
00087
00088 virtual void
00089 SetPosition(PositionType position)
00090 {
00091 if (GetPosition() == position)
00092 return;
00093 if (!mUseMemory)
00094 FSeek(mFile, position, SEEK_SET);
00095 IOBuffer::SetPosition(position);
00096 }
00097
00098 virtual Int64
00099 Read(void* buf, Int64 bytesToRead)
00100 {
00101 Int64 available = Available();
00102 if (available < bytesToRead)
00103 bytesToRead = available;
00104 if (mUseMemory)
00105 {
00106 memcpy(buf, GetBuffer() + GetPosition(), bytesToRead);
00107 }
00108 else
00109 {
00110 FSeek(mFile, GetPosition(), SEEK_SET);
00111 Int64 nRead = fread(buf, 1, bytesToRead, mFile);
00112 }
00113 SetPosition(GetPosition() + bytesToRead);
00114 return bytesToRead;
00115 }
00116
00117 virtual String
00118 ReadLine()
00119 {
00120 if (mUseMemory)
00121 return IOBuffer::ReadLine();
00122
00123
00124 Int64 available = Available();
00125 Int64 maxLineSize = 1048576;
00126 if (available < maxLineSize)
00127 maxLineSize = available + 1;
00128
00129
00130 static char buf[1024 * 1024];
00131 FSeek(mFile, GetPosition(), SEEK_SET);
00132 fgets(buf, maxLineSize, mFile);
00133 SetPosition(FTell(mFile));
00134 for (int i=0 ; i<maxLineSize ; i++)
00135 {
00136 if ((buf[i] == '\r') || (buf[i] == '\n'))
00137 {
00138 buf[i] = '\0';
00139 break;
00140 }
00141 }
00142 String res(buf);
00143
00144 return res;
00145 }
00146
00147 virtual Int64
00148 Gets(char* buf, Int64 bytesToRead)
00149 {
00150 if (mUseMemory)
00151 return IOBuffer::Gets(buf, bytesToRead);
00152
00153 Int64 available = Available();
00154 if (available < bytesToRead)
00155 bytesToRead = available;
00156 FSeek(mFile, GetPosition(), SEEK_SET);
00157 fgets(buf, bytesToRead, mFile);
00158 SetPosition(FTell(mFile));
00159 return bytesToRead;
00160 }
00161
00162 virtual void
00163 NativeTypeRead(Int8* ptr)
00164 {
00165 DoNativeTypeRead(ptr);
00166 }
00167
00168 virtual void
00169 NativeTypeRead(UInt8* ptr)
00170 {
00171 DoNativeTypeRead(ptr);
00172 }
00173
00174 virtual void
00175 NativeTypeRead(Int16* ptr)
00176 {
00177 DoNativeTypeRead(ptr);
00178 }
00179
00180 virtual void
00181 NativeTypeRead(UInt16* ptr)
00182 {
00183 DoNativeTypeRead(ptr);
00184 }
00185
00186 virtual void
00187 NativeTypeRead(Int32* ptr)
00188 {
00189 DoNativeTypeRead(ptr);
00190 }
00191
00192 virtual void
00193 NativeTypeRead(UInt32* ptr)
00194 {
00195 DoNativeTypeRead(ptr);
00196 }
00197
00198 virtual void
00199 NativeTypeRead(Int64* ptr)
00200 {
00201 DoNativeTypeRead(ptr);
00202 }
00203
00204 virtual void
00205 NativeTypeRead(UInt64* ptr)
00206 {
00207 DoNativeTypeRead(ptr);
00208 }
00209
00210 virtual void
00211 NativeTypeRead(Real32* ptr)
00212 {
00213 DoNativeTypeRead(ptr);
00214 }
00215
00216 virtual void
00217 NativeTypeRead(Real64* ptr)
00218 {
00219 DoNativeTypeRead(ptr);
00220 }
00221
00222 virtual Int64
00223 Write(const void* buf, Int64 bytesToWrite)
00224 {
00225 size_t bytesWritten = fwrite(buf, 1, bytesToWrite, mFile);
00226 if (bytesWritten != bytesToWrite)
00227 ILOG_ERROR("Bytes written to file (" << bytesWritten << ") " <<
00228 "differs from bytes to write (" << bytesToWrite << ")");
00229 SetPositionAndSize(FTell(mFile));
00230 return bytesWritten;
00231 }
00232
00233 virtual Int64
00234 Puts(const char* buf)
00235 {
00236 fprintf(mFile, "%s\n", buf);
00237 SetPositionAndSize(FTell(mFile));
00238 return 1;
00239 }
00240
00241 virtual void
00242 NativeTypeWrite(Int8 val)
00243 {
00244 DoNativeTypeWrite(val);
00245 }
00246
00247 virtual void
00248 NativeTypeWrite(UInt8 val)
00249 {
00250 DoNativeTypeWrite(val);
00251 }
00252
00253 virtual void
00254 NativeTypeWrite(Int16 val)
00255 {
00256 DoNativeTypeWrite(val);
00257 }
00258
00259 virtual void
00260 NativeTypeWrite(UInt16 val)
00261 {
00262 DoNativeTypeWrite(val);
00263 }
00264
00265 virtual void
00266 NativeTypeWrite(Int32 val)
00267 {
00268 DoNativeTypeWrite(val);
00269 }
00270
00271 virtual void
00272 NativeTypeWrite(UInt32 val)
00273 {
00274 DoNativeTypeWrite(val);
00275 }
00276
00277 virtual void
00278 NativeTypeWrite(Int64 val)
00279 {
00280 DoNativeTypeWrite(val);
00281 }
00282
00283 virtual void
00284 NativeTypeWrite(UInt64 val)
00285 {
00286 DoNativeTypeWrite(val);
00287 }
00288
00289 virtual void
00290 NativeTypeWrite(Real32 val)
00291 {
00292 DoNativeTypeWrite(val);
00293 }
00294
00295 virtual void
00296 NativeTypeWrite(Real64 val)
00297 {
00298 DoNativeTypeWrite(val);
00299 }
00300
00301 private:
00302
00303 static FILE*
00304 Open(CString path, CString mode)
00305 {
00306 String thePath = path;
00307 #ifdef WIN32
00308 thePath = StringReplaceAll(thePath, "/", "\\");
00309 thePath = StringReplaceAll(thePath, "\\\\", "\\", true, 2);
00310 thePath = StringReplaceAll(thePath, "\\.\\", "\\");
00311 bool isAbs = (thePath.at(0) == '\\') || (thePath.find(":") != String::npos);
00312 int pathSize = thePath.size();
00313
00314 static const long maxSimplePath = _MAX_PATH;
00315 if ((pathSize > maxSimplePath) || !isAbs)
00316 {
00317 if (!isAbs)
00318 {
00319 if (pathSize > maxSimplePath)
00320 {
00321 ILOG_ERROR("Relative path over " << maxSimplePath <<
00322 " characters long: " << thePath);
00323 return NULL;
00324 }
00325 static const int relPathWarnLength = maxSimplePath / 2;
00326 if (pathSize > relPathWarnLength)
00327 ILOG_WARN("Relative path over " << relPathWarnLength <<
00328 " characters long; it may not be possible to" <<
00329 " resolve the absolute path for: " << thePath);
00330
00331
00332
00333
00334 String pathHead = thePath;
00335 String pathTail = "";
00336 int firstSepPos = thePath.find("\\");
00337 if (firstSepPos != String::npos)
00338 {
00339 int secondSepPos = thePath.find("\\", firstSepPos + 1);
00340 if (secondSepPos != String::npos)
00341 {
00342 pathHead = thePath.substr(0, secondSepPos);
00343 pathTail = thePath.substr(secondSepPos);
00344 }
00345 }
00346 char absPath[maxSimplePath + 1];
00347 if (!_fullpath(absPath, pathHead.c_str(), maxSimplePath + 1))
00348 {
00349 ILOG_ERROR("Failed to compose absolute path for " <<
00350 thePath << " (" << strerror(errno) << ")");
00351 return NULL;
00352 }
00353 thePath = String(absPath) + pathTail;
00354 }
00355
00356 static const String longPathPrefix = "\\\\?\\";
00357 String longPath = longPathPrefix;
00358 if (thePath.substr(0, 2) == "\\\\")
00359 longPath += "UNC" + thePath.substr(1);
00360 else
00361 longPath += thePath;
00362
00363
00364
00365 static const int maxLongPath = 32 * 1024;
00366 wchar_t longPathWC[maxLongPath + 1];
00367 for (int i = 0; i < longPath.size(); i++)
00368 longPathWC[i] = (wchar_t) longPath.at(i);
00369 longPathWC[longPath.size()] = L'\0';
00370 wchar_t modeWC[10];
00371 for (int i = 0; i < mode.size(); i++)
00372 modeWC[i] = (wchar_t) mode.at(i);
00373 modeWC[mode.size()] = L'\0';
00374 FILE* fp = _wfopen(longPathWC, modeWC);
00375 return fp;
00376 }
00377 #endif
00378 return fopen(thePath.c_str(), mode.c_str());
00379 }
00380
00381 template <class NativeType>
00382 void
00383 DoNativeTypeRead(NativeType* ptr)
00384 {
00385 if (mUseMemory)
00386 return IOBuffer::DoNativeTypeRead(ptr);
00387
00388 String fs = NativeTypeFormat<NativeType>(0);
00389 FSeek(mFile, GetPosition(), SEEK_SET);
00390 fscanf(mFile, fs.c_str(), ptr);
00391 SetPosition(FTell(mFile));
00392 }
00393
00394 template <class NativeType>
00395 void
00396 DoNativeTypeWrite(NativeType val)
00397 {
00398 String fs = NativeTypeFormat<NativeType>(0) + " ";
00399 if (mUseMemory)
00400 {
00401 char* start = (char*) (GetBuffer() + GetPosition());
00402 sprintf(start, fs.c_str(), val);
00403 Int64 i = 0;
00404 while (start[i] != ' ')
00405 i++;
00406 SetPosition(GetPosition() + i + 1);
00407 }
00408 else
00409 {
00410 fprintf(mFile, fs.c_str(), val);
00411 SetPositionAndSize(FTell(mFile));
00412 }
00413 }
00414
00415 PositionType
00416 FTell(FILE* fp)
00417 {
00418 #ifdef WIN32
00419 #if _MSC_VER <= 1310
00420 return ftell(fp);
00421 #else
00422 return _ftelli64(fp);
00423 #endif
00424 #else
00425 return ftello(fp);
00426 #endif
00427 }
00428
00429 int
00430 FSeek(FILE* fp, PositionType offset, int origin)
00431 {
00432 #ifdef WIN32
00433 #if _MSC_VER <= 1310
00434 return fseek(fp, offset, origin);
00435 #else
00436 return _fseeki64(fp, offset, origin);
00437 #endif
00438 #else
00439 return fseeko(fp, offset, origin);
00440 #endif
00441 }
00442
00443 bool mReadMode;
00444 bool mUseMemory;
00445 FILE* mFile;
00446
00447 ILOG_VAR_DEC;
00448 };
00449
00450 ILOG_VAR_INIT(IOBufferFile, Impala.Util);
00451
00452 }
00453 }
00454
00455 #endif