00001 #ifndef Impala_Core_Stream_RgbDataSrcInfo_h
00002 #define Impala_Core_Stream_RgbDataSrcInfo_h
00003
00004 #include "Basis/FileName.h"
00005 #include "Basis/ILog.h"
00006 #include "Basis/Timer.h"
00007 #include "Util/StringParser.h"
00008 #include "Util/Database.h"
00009 #include "Util/IOBuffer.h"
00010 #include "Core/Stream/RgbDataSrc.h"
00011 #include "Core/Table/TableTem.h"
00012 #include "Core/Table/Write.h"
00013 #include "Core/Table/Read.h"
00014 #include "Core/Column/ColumnTem.h"
00015 #include "Core/Column/FixedStringColumn.h"
00016 #include "Persistency/RgbDataSrcLocator.h"
00017 #include "Persistency/RepositoryInFileSystem.h"
00018
00019 namespace Impala
00020 {
00021 namespace Core
00022 {
00023 namespace Stream
00024 {
00025
00026
00027 class RgbDataSrcInfo : public RgbDataSrc
00028 {
00029
00030 typedef Table::TableTem<Column::ColumnTem<Int32>,
00031 Column::ColumnTem<Int32>,
00032 Column::ColumnTem<UInt64>,
00033 Column::FixedStringColumn > IndexTableType;
00034
00035 typedef Column::FixedStringColumn::ColElemType FixedString;
00036
00037 public:
00038
00039 static const String DUMMY_FRAME_HASH;
00040 static const String MISSING_FRAME_HASH;
00041
00042 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00043 RgbDataSrcInfo(int src, CString srcName, Util::Database* db,
00044 CString infoName = "", bool newImpl = false)
00045 : RgbDataSrc(src, srcName), mDb(db)
00046 {
00047 #else // REPOSITORY_USED
00048 RgbDataSrcInfo(int src, const Persistency::RgbDataSrcLocator& loc,
00049 CString infoName = "", bool newImpl = false)
00050 : RgbDataSrc(src, loc.GetName())
00051 {
00052 String srcName = loc.GetName();
00053 mLoc = loc;
00054 #endif // REPOSITORY_USED
00055 mNewImpl = newImpl;
00056
00057
00058 if (infoName.empty())
00059 {
00060 #ifdef USE_IFILE
00061 int posAfterProtocol = srcName.find(":") + 1;
00062 mInfoPath = srcName.substr(posAfterProtocol) + ".info";
00063 #else
00064 mInfoPath = srcName + ".info";
00065 #endif
00066 }
00067 else
00068 mInfoPath = infoName;
00069
00070 InitInfo();
00071
00072
00073
00074
00075
00076 if ((src == 18) || (src == 19))
00077 ReadInfo();
00078 if (src == 19)
00079 DumpIndex();
00080 }
00081
00082
00083
00084
00085
00086
00087
00088
00089 virtual
00090 ~RgbDataSrcInfo()
00091 {
00092 if (mIndex)
00093 delete mIndex;
00094 }
00095
00096 virtual bool
00097 ReadFrameData() = 0;
00098
00099 virtual bool
00100 CurIsIFrame()
00101 {
00102 int f = FrameNr();
00103 return GetSeekableFrame(f) == f;
00104 }
00105
00106 virtual bool
00107 Valid() const
00108 {
00109 return true;
00110 }
00111
00112 Int64
00113 DumpHashValue(char* &buffer, const int maxChars, int bytesWritten,
00114 CString hash) const
00115 {
00116 const int hashLen = hash.size();
00117 if (bytesWritten + hashLen + 1 > maxChars)
00118 {
00119 ILOG_ERROR("Buffer too small (" << maxChars <<
00120 ") to write all index hashes");
00121 return -1;
00122 }
00123 sprintf(buffer, "%s\t", hash.c_str());
00124 buffer += (hashLen + 1);
00125 int written = bytesWritten + (hashLen + 1);
00126 return written;
00127 }
00128
00129 Int64
00130 DumpIndexHashes(char* buffer, const int maxChars) const
00131 {
00132 const int nrOfIndexedFrames = mIndex->Size();
00133 const int maxFrameNr = mIndex->Get1(nrOfIndexedFrames - 1);
00134
00135 int indx = 0;
00136 Int64 bytesWritten = 0;
00137 int frameNr = 0;
00138 while (indx < nrOfIndexedFrames)
00139 {
00140 String hash;
00141 const int frameNrFromIndexEntry = mIndex->Get1(indx);
00142 if (frameNr < frameNrFromIndexEntry)
00143 {
00144 ILOG_WARN("Index contains no entry for frame #" << frameNr);
00145 hash = MISSING_FRAME_HASH;
00146 }
00147 else
00148 {
00149 if (frameNr > frameNrFromIndexEntry)
00150 {
00151 ILOG_ERROR("[DumpIndexHashes] Index contains inconsistency for frame #" <<
00152 frameNrFromIndexEntry << " at table position " << indx <<
00153 " (zero based)");
00154 }
00155 const String hash32(mIndex->Get4(indx).GetData(), HASH_SIZE);
00156 hash = String(hash32.c_str());
00157 indx++;
00158 }
00159 bytesWritten = DumpHashValue(buffer, maxChars, bytesWritten, hash);
00160 if (bytesWritten < 0)
00161 return -1;
00162 frameNr++;
00163 }
00164 return bytesWritten;
00165 }
00166
00167 protected:
00168
00169 String
00170 ReplaceWhite(String result)
00171 {
00172 String replaceWhat=" ";
00173 String replaceWithWhat="_";
00174 while(1)
00175 {
00176 const int pos = result.find(replaceWhat);
00177 if (pos==-1)
00178 break;
00179 result.replace(pos,replaceWhat.size(),replaceWithWhat);
00180 }
00181 if(result=="")
00182 {
00183 ILOG_DEBUG("String was empty, resetting to _");
00184 result="_";
00185 }
00186 return result;
00187
00188 }
00189
00190 String
00191 GetInfoFilePath(bool toWrite, bool silent=true) const
00192 {
00193 return mInfoPath;
00194 }
00195
00196 Util::IOBuffer*
00197 GetInfoIOBuffer(bool toWrite, bool silent=true) const
00198 {
00199 String fname = GetInfoFilePath(toWrite, silent);
00200 if (fname.empty())
00201 {
00202 if (!silent)
00203 ILOG_ERROR("Unable to determine info file name!");
00204 return 0;
00205 }
00206
00207
00208 String tmpFile = (toWrite) ? "tmp" : "";
00209 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00210 Util::IOBuffer* buf = mDb->GetIOBuffer(fname, !toWrite, false, tmpFile);
00211 #else // REPOSITORY_USED
00212 typedef Persistency::RepositoryInFileSystem RepFS;
00213 Persistency::File file = RepFS::GetInstance().GetFile(mLoc, "", fname,
00214 toWrite, false);
00215 Util::IOBuffer* buf = (toWrite) ? file.GetWriteBuffer()
00216 : file.GetReadBuffer();
00217 #endif // REPOSITORY_USED
00218
00219 if(!buf)
00220 {
00221 ILOG_DEBUG("Invalid File! "<<mInfoPath);
00222 return 0;
00223 }
00224
00225 if(!buf->Valid())
00226 {
00227 ILOG_DEBUG("Invalid File! "<<mInfoPath);
00228 delete buf;
00229 return 0;
00230 }
00231
00232 return buf;
00233 }
00234
00235 void InitInfo()
00236 {
00237 mInfoVersion = "2.2";
00238
00239
00240 mIndexInPlace = true;
00241
00242 mIndex = new IndexTableType(Column::ColumnTem<Int32>(1),
00243 Column::ColumnTem<Int32>(1),
00244 Column::ColumnTem<UInt64>(1),
00245 Column::FixedStringColumn(HASH_SIZE, 1));
00246 mIndex->SetColName(1, "frame_nr");
00247 mIndex->SetColName(2, "key_frame_nr");
00248 mIndex->SetColName(3, "key_frame_pos");
00249 mIndex->SetColName(4, "frame_hash");
00250
00251 mFormatStr = String(
00252 "INFO VERSION : ") + mInfoVersion + "\n"
00253 + "==================================\n"
00254 + "File name : %s\n"
00255 + "\n"
00256 + "Format name : %s\n"
00257 + "Format name (long) : %s\n"
00258 + "Bit rate (bps) : %d \n"
00259 + "Stream count : %d \n"
00260 + "\n"
00261 + "Codec name : %s\n"
00262 + "Codec tag : %s\n"
00263 + "Time base : %d / %d \n"
00264 + "Duration (fractions) : %lld \n"
00265 + "Duration (seconds) : %lld \n"
00266 + "Duration (human) : %lldh %lldm %llds \n"
00267 + "\n"
00268 + "Frame width : %d \n"
00269 + "Frame height : %d \n"
00270 + "Aspect ratio : %d / %d \n"
00271 + "\n"
00272 + "Frame count : %d \n"
00273 + "Frame rate (fps) : %d / %d \n"
00274 + "GOP size : %d \n"
00275 + "Leading bad frames : %d \n"
00276 + "Bad frame ID's : %s \n"
00277 + "\n"
00278 + "Index exists : %s \n";
00279 }
00280
00281 bool ReadInfo(bool silent = false)
00282 {
00283 ILOG_DEBUG("Reading info file");
00284 Timer timer;
00285 Util::IOBuffer* infoBuf = GetInfoIOBuffer(false, silent);
00286 if (!infoBuf)
00287 {
00288 ILOG_WARN("Reading canceled");
00289 return false;
00290 }
00291
00292 char buf[BUFFER_SIZE];
00293 infoBuf->Read(buf, BUFFER_SIZE);
00294 char srcName[100];
00295 char formatName[30];
00296 char formatLName[100];
00297 char codecName[30];
00298 char codecTag[5];
00299 char indexExists[5];
00300
00301 int frameCount;
00302 char badFramesList[10000];
00303 int numberOfLeadingBadFrames;
00304 int res = sscanf(buf, mFormatStr.c_str(),
00305 srcName,
00306 formatName,
00307 formatLName,
00308 &mBitRate,
00309 &mStreamCount,
00310 codecName,
00311 codecTag,
00312 &mTimeBaseNum, &mTimeBaseDen,
00313 &mVideoDuration,
00314 &mDuration,
00315 &mDurationHour, &mDurationMin, &mDurationSec,
00316 &mFrameWidth,
00317 &mFrameHeight,
00318 &mAspectRatioNum, &mAspectRatioDen,
00319 &frameCount,
00320 &mFrameRateNum, &mFrameRateDen,
00321 &mGopSize,
00322 &numberOfLeadingBadFrames,
00323 badFramesList,
00324 indexExists);
00325 if (res != 25)
00326 {
00327 ILOG_ERROR("RGB data source info format not compatible; " <<
00328 "expected version is: " << mInfoVersion);
00329 return false;
00330 }
00331
00332
00333
00334
00335
00336
00337
00338
00339
00340
00341 mLastFrame = frameCount - 1;
00342 mFormatName = formatName;
00343 mFormatLName = formatLName;
00344 mCodecName = codecName;
00345 mCodecTag = codecTag;
00346
00347 SetLeadingBadFrames(numberOfLeadingBadFrames);
00348 mBadFrameNrs.clear();
00349 const String badFramesListString(badFramesList);
00350 if (badFramesListString != "none")
00351 {
00352 Util::StringParser parser(badFramesListString);
00353 while (!parser.TheEnd())
00354 {
00355 const int frameNr = parser.GetInt(';', false, true);
00356 MarkBadFrame(frameNr);
00357 }
00358 }
00359
00360 if (String(indexExists) == "yes")
00361 mIndexExists = true;
00362 else
00363 mIndexExists = false;
00364
00365
00366 {
00367 StringList badFramesString;
00368 for (int i = 0; i < mBadFrameNrs.size(); i++)
00369 badFramesString << MakeString(mBadFrameNrs[i]);
00370
00371 sprintf(buf, mFormatStr.c_str(),
00372 srcName,
00373 formatName,
00374 formatLName,
00375 mBitRate,
00376 mStreamCount,
00377 codecName,
00378 codecTag,
00379 mTimeBaseNum, mTimeBaseDen,
00380 mVideoDuration,
00381 mDuration,
00382 mDurationHour, mDurationMin, mDurationSec,
00383 mFrameWidth,
00384 mFrameHeight,
00385 mAspectRatioNum, mAspectRatioDen,
00386 mLastFrame + 1,
00387 mFrameRateNum, mFrameRateDen,
00388 mGopSize,
00389 mBlankFrames,
00390 badFramesString.ToString(';').c_str(),
00391 indexExists);
00392 ILOG_DEBUG("\n\n" << buf);
00393 }
00394
00395 if (mIndexExists)
00396 {
00397 ILOG_DEBUG("Reading the index attached to info file");
00398 ILOG_DEBUG(" start read " << timer.SplitTime());
00399 Read(mIndex, infoBuf);
00400 ILOG_DEBUG(" done actual read " << timer.SplitTime());
00401 ILOG_DEBUG(mIndex->Size() << " index entries read");
00402 }
00403
00404 delete infoBuf;
00405 return true;
00406 }
00407
00408 void DumpIndex()
00409 {
00410 ILOG_DEBUG("Dumping index entries:");
00411 for (int i = 0; i < mIndex->Size(); i++)
00412 ILOG_DEBUG(i << " : " << mIndex->Get1(i) << " "
00413 << mIndex->Get2(i) << " " << mIndex->Get3(i));
00414 }
00415
00416 void WriteInfo()
00417 {
00418 ILOG_DEBUG("Writing info file");
00419
00420
00421 Util::IOBuffer* infoBuf = GetInfoIOBuffer(true);
00422
00423 if (!infoBuf)
00424 {
00425 ILOG_WARN("Writing canceled: if file already exists, "
00426 << "use the --override switch; "
00427 << "otherwise make sure the directory is writable.");
00428 return;
00429 }
00430
00431
00432 char buf[BUFFER_SIZE];
00433 memset(buf, 0, BUFFER_SIZE);
00434 StringList badFramesStringList;
00435 for (int i = 0; i < mBadFrameNrs.size(); i++)
00436 badFramesStringList << MakeString(mBadFrameNrs[i]);
00437 String badFramesString = badFramesStringList.ToString(';');
00438 if (badFramesString.empty())
00439 badFramesString = "none";
00440 sprintf(buf, mFormatStr.c_str(),
00441 ReplaceWhite(FileNameTail(mSrcName)).c_str(),
00442 ReplaceWhite(mFormatName).c_str(),
00443 ReplaceWhite(mFormatLName).c_str(),
00444 mBitRate,
00445 mStreamCount,
00446 ReplaceWhite(mCodecName).c_str(),
00447 ReplaceWhite(mCodecTag).c_str(),
00448 mTimeBaseNum, mTimeBaseDen,
00449 mVideoDuration,
00450 mDuration,
00451 mDurationHour, mDurationMin, mDurationSec,
00452 mFrameWidth,
00453 mFrameHeight,
00454 mAspectRatioNum, mAspectRatioDen,
00455 mLastFrame + 1,
00456 mFrameRateNum, mFrameRateDen,
00457 mGopSize,
00458 mBlankFrames,
00459 badFramesString.c_str(),
00460 (mIndexExists ? "yes" : "no") );
00461 infoBuf->Write(buf, BUFFER_SIZE);
00462
00463 if (mIndexExists)
00464 {
00465 ILOG_DEBUG("Index exists; writing table");
00466 Write(mIndex, infoBuf, true);
00467 ILOG_DEBUG(mIndex->Size() << " index entries written");
00468 }
00469
00470 delete infoBuf;
00471 }
00472
00473 void
00474 AddIndex(int fr, int key, UInt64 pos, CString hash)
00475 {
00476 if (fr != (mIndex->Size() + (mNewImpl ? 0 : mBlankFrames)))
00477 ILOG_WARN("Frame numbers mismatch " << fr << " != " << mIndex->Size());
00478 FixedString fStr(HASH_SIZE, (char*) hash.c_str(), false);
00479 mIndex->Add(fr, key, pos, fStr);
00480 }
00481
00482 int GetSeekableFrame(int fr) const
00483 {
00484 return mIndex->Get2(fr);
00485 }
00486
00487 void SetKey(int fr, int key)
00488 {
00489 mIndex->Set2(fr, key);
00490 }
00491
00492 UInt64 GetPos(int fr) const
00493 {
00494 const int indexPosition = fr;
00495
00496 if (mIndexExists)
00497 {
00498 if (indexPosition < mIndex->Size())
00499 {
00500 if (FrameHash(indexPosition) == DUMMY_FRAME_HASH)
00501 {
00502 ILOG_WARN("No position for invalid frame: " << fr <<
00503 "; returning next frame's position instead");
00504 return GetPos(fr + 1);
00505 }
00506 else
00507 return mIndex->Get3(indexPosition);
00508 }
00509 else
00510 ILOG_ERROR("Index position out of bounds: " << indexPosition);
00511 }
00512 else
00513 {
00514 ILOG_WARN("No external index available; " <<
00515 "cannot lookup seek position for frame: " << fr);
00516 if (!mNewImpl)
00517 return 0;
00518 }
00519
00520 return -1;
00521 }
00522
00523
00524 int GetFrameFromHash(String hash) const
00525 {
00526 for (int i = mBlankFrames; i < mIndex->Size(); i++)
00527 if (FrameHash(i) == hash)
00528 return i;
00529
00530 return -1;
00531 }
00532
00533 void SetPos(int fr, UInt64 pos)
00534 {
00535 mIndex->Set3(fr, pos);
00536 }
00537
00538 virtual String
00539 FrameHash() const
00540 {
00541 if (HasIndex())
00542 return FrameHash(FrameNr());
00543 else
00544 return RgbDataSrc::FrameHash();
00545 }
00546
00547 virtual String
00548 FrameHash(int fr) const
00549 {
00550 return String(mIndex->Get4(fr).GetData(), HASH_SIZE);
00551 }
00552
00553
00554 virtual const int
00555 BadFrames() const
00556 {
00557 return mBadFrameNrs.size();
00558 }
00559
00560
00561 protected:
00562 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00563 Util::Database* mDb;
00564 #endif // REPOSITORY_USED
00565 IndexTableType* mIndex;
00566
00567
00568 virtual bool
00569 FrameValid(int frameNr) const
00570 {
00571 if (!RgbDataSrc::FrameValid(frameNr))
00572 return false;
00573
00574 for (int i = 0; i < mBadFrameNrs.size(); i++)
00575 if (mBadFrameNrs[i] == frameNr)
00576 return false;
00577
00578 return true;
00579 }
00580
00581 bool
00582 MarkBadFrame(int frameNr)
00583 {
00584 for (int i = 0; i < mBadFrameNrs.size(); i++)
00585 if (mBadFrameNrs[i] == frameNr)
00586 return false;
00587
00588 mBadFrameNrs.push_back(frameNr);
00589 return true;
00590 }
00591
00592
00593 private:
00594
00595 const static int BUFFER_SIZE = 100000;
00596
00597 String mFormatStr;
00598 String mInfoPath;
00599 String mInfoVersion;
00600 bool mIndexInPlace;
00601
00602 std::vector<int> mBadFrameNrs;
00603 Persistency::RgbDataSrcLocator mLoc;
00604
00605 ILOG_VAR_DEC;
00606
00607 };
00608
00609 ILOG_VAR_INIT(RgbDataSrcInfo, Impala.Core.Stream);
00610
00611
00612 const String RgbDataSrcInfo::DUMMY_FRAME_HASH = "DUMMY_FRAME_HASH________________";
00613 const String RgbDataSrcInfo::MISSING_FRAME_HASH = "MISSING_FRAME_HASH______________";
00614
00615 }
00616 }
00617 }
00618
00619 #endif