00001 #ifndef Impala_Core_Table_AnnotationTable_h
00002 #define Impala_Core_Table_AnnotationTable_h
00003
00004 #include "Util/QuickSort.h"
00005 #include "Util/SimpleMap.h"
00006 #include "Core/Table/SelectQuids.h"
00007 #include "Core/Table/Read.h"
00008 #include "Core/Table/Criterion.h"
00009 #include "Core/Table/CollectQuidObjects.h"
00010 #include "Core/Table/Select.h"
00011 #include "Core/Column/Find.h"
00012 #include "Core/Column/FindInAscending.h"
00013 #include "Util/QuasiRandomSequenceIterator.h"
00014
00015 namespace Impala
00016 {
00017 namespace Core
00018 {
00019 namespace Table
00020 {
00021
00022
00023 typedef TableTem<Column::ColumnTem<Quid>,
00024 Column::ColumnTem<Real64> >
00025 AnnotationTableBaseType;
00026
00027
00032 class AnnotationTable : public AnnotationTableBaseType
00033 {
00034 public:
00035
00036 typedef Column::ColumnTem<Real64> ColumnReal64;
00037 typedef Column::ColumnTem<Quid> ColumnQuid;
00038
00039 AnnotationTable() :
00040 AnnotationTableBaseType(ColumnQuid(0), ColumnReal64(0))
00041 {
00042 mLabel = "";
00043 mIsAscending = false;
00044 SetInfo("AnnotationTable");
00045 SetColName(1, "annotee");
00046 SetColName(2, "qualifier");
00047 }
00048
00049 AnnotationTable(String label, int tableSize) :
00050 AnnotationTableBaseType(ColumnQuid(tableSize), ColumnReal64(tableSize))
00051 {
00052 mLabel = label;
00053 mIsAscending = false;
00054 SetInfo("AnnotationTable : " + label);
00055 SetColName(1, "annotee");
00056 SetColName(2, "qualifier " + label);
00057 }
00058
00059 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00060 static AnnotationTable*
00061 MakeFromFile(String label, String fileName, Util::Database* db)
00062 {
00063 AnnotationTable* res = new AnnotationTable(label, 0);
00064 Read(res, fileName, db);
00065
00066 return res;
00067 }
00068 #endif // REPOSITORY_USED
00069
00070 String
00071 GetLabel() const
00072 {
00073 return mLabel;
00074 }
00075
00076 void
00077 SetLabel(String label)
00078 {
00079 mLabel = label;
00080 }
00081
00082 int
00083 GetQuidClass() const
00084 {
00085 if (Size() < 1)
00086 return QUID_CLASS_UNKNOWN;
00087 return QuidClass(Get1(0));
00088 }
00089
00090 bool
00091 ContainsFrames() const
00092 {
00093 return GetQuidClass() == QUID_CLASS_FRAME;
00094 }
00095
00096 bool
00097 ContainsShots() const
00098 {
00099 return GetQuidClass() == QUID_CLASS_SHOT;
00100 }
00101
00102 int
00103 GetNrPositive() const
00104 {
00105 int res = 0;
00106 for (int i=0 ; i<Size() ; i++)
00107 if (IsPositive(i))
00108 res++;
00109 return res;
00110 }
00111
00112 int
00113 GetNrNegative() const
00114 {
00115 int res = 0;
00116 for (int i=0 ; i<Size() ; i++)
00117 if (IsNegative(i))
00118 res++;
00119 return res;
00120 }
00121
00122 QuidTable*
00123 GetPositive() const
00124 {
00125 CriterionElement2Equals<AnnotationTableBaseType> c(1.);
00126 return SelectQuids(this, c);
00127 }
00128
00129 QuidTable*
00130 GetNegative() const
00131 {
00132 CriterionElement2Equals<AnnotationTableBaseType> c(-1.);
00133 return SelectQuids(this, c);
00134 }
00135
00136 QuidTable*
00137 GetAnnotatedQuids() const
00138 {
00139 CriterionElement2NotEquals<AnnotationTableBaseType> c(0.);
00140 return SelectQuids(this, c);
00141 }
00142
00143 int
00144 GetIndex(Quid annotee) const
00145 {
00146 int index = Size();
00147 if (mQuidMap.Get(annotee, index))
00148 return index;
00149 if (mIsAscending)
00150 return Column::FindInAscending(GetColumn1(), annotee, 0, Size());
00151 return Column::Find(GetColumn1(), annotee, 0, Size());
00152 }
00153
00154 bool
00155 IsAnnotated(Quid annotee) const
00156 {
00157 return (GetIndex(annotee) != Size());
00158 }
00159
00160 Real64
00161 GetQualification(Quid annotee) const
00162 {
00163 int index = GetIndex(annotee);
00164 if (index != Size())
00165 return Get2(index);
00166 return -2.0;
00167 }
00168
00169 String
00170 GetQualificationString(Quid annotee) const
00171 {
00172 int index = GetIndex(annotee);
00173 if (index == Size())
00174 return "UNK";
00175 if (IsPositive(index))
00176 return "POS";
00177 if (IsNegative(index))
00178 return "NEG";
00179 if (IsSkip(index))
00180 return "SKIP";
00181 return "ERROR";
00182 }
00183
00184 bool
00185 IsPositive(int idx) const
00186 {
00187 return Get2(idx) == 1.0;
00188 }
00189
00190 bool
00191 IsPositive(Quid annotee) const
00192 {
00193 int index = GetIndex(annotee);
00194 if (index != Size())
00195 return IsPositive(index);
00196 return false;
00197 }
00198
00199 void
00200 SetPositive(int idx)
00201 {
00202 Set2(idx, 1.0);
00203 }
00204
00205 bool
00206 IsNegative(int idx) const
00207 {
00208 return Get2(idx) == -1.0;
00209 }
00210
00211 bool
00212 IsNegative(Quid annotee) const
00213 {
00214 int index = GetIndex(annotee);
00215 if (index != Size())
00216 return IsNegative(index);
00217 return false;
00218 }
00219
00220 void
00221 SetNegative(int idx)
00222 {
00223 Set2(idx, -1.0);
00224 }
00225
00226 bool
00227 IsSkip(int idx) const
00228 {
00229 return Get2(idx) == 0.0;
00230 }
00231
00232 bool
00233 IsSkip(Quid annotee) const
00234 {
00235 int index = GetIndex(annotee);
00236 if (index != Size())
00237 return IsSkip(index);
00238 return false;
00239 }
00240
00241 void
00242 SetSkip(int idx)
00243 {
00244 Set2(idx, 0.0);
00245 }
00246
00247 void
00248 AddPositive(Quid annotee)
00249 {
00250 Add(annotee, 1.0);
00251 }
00252
00253 void
00254 AddNegative(Quid annotee)
00255 {
00256 Add(annotee, -1.0);
00257 }
00258
00259 void
00260 AddSkip(Quid annotee)
00261 {
00262 Add(annotee, 0.0);
00263 }
00264
00265
00266 void
00267 SelectQuidObjectMaxId(int maxId)
00268 {
00269 if (maxId == -1)
00270 return;
00271
00272 CriterionQuidObjectMaxId<AnnotationTable> crit(maxId);
00273 Select(this, this, crit, true);
00274 }
00275
00276
00277 void
00278 SelectQuidObjectMaxPositive(int number)
00279 {
00280 if (number == -1)
00281 return;
00282
00283 CriterionQuidObjectMaxNumberElement2<AnnotationTable> crit(1.0, number);
00284 Select(this, this, crit, true);
00285 }
00286
00287
00288 void
00289 SelectQuidObjectMaxNegative(int number)
00290 {
00291 if (number == -1)
00292 return;
00293
00294 CriterionQuidObjectMaxNumberElement2<AnnotationTable> crit(-1.0, number);
00295 Select(this, this, crit, true);
00296 }
00297
00298
00299 void
00300 SelectQuidObjectMaxSkip(int number)
00301 {
00302 if (number == -1)
00303 return;
00304
00305 CriterionQuidObjectMaxNumberElement2<AnnotationTable> crit(0.0, number);
00306 Select(this, this, crit, true);
00307 }
00308
00309 void
00310 UpdateQuidMap()
00311 {
00312 mIsAscending = true;
00313 for (int i=0 ; i<Size()-1 ; i++)
00314 {
00315 if (Get1(i+1) < Get1(i))
00316 {
00317 mIsAscending = false;
00318 break;
00319 }
00320 }
00321 if (!mIsAscending)
00322 {
00323 for (int i=0 ; i<Size() ; i++)
00324 mQuidMap.Add(Get1(i), i);
00325 }
00326 }
00327
00328
00329 void
00330 Merge(AnnotationTable* arg)
00331 {
00332 for (int i=0 ; i<arg->Size() ; i++)
00333 {
00334 Quid annotee = arg->Get1(i);
00335 int index = GetIndex(annotee);
00336 if (index == Size())
00337 {
00338 Add(annotee, arg->Get2(i));
00339 }
00340 else
00341 {
00342 Real64 qualification = Max(Get2(index), arg->Get2(i));
00343 Set2(index, qualification);
00344 }
00345 }
00346 }
00347
00348 void
00349 Correct(AnnotationTable* corrections)
00350 {
00351 for (int i=0 ; i<corrections->Size() ; i++)
00352 {
00353 Quid annotee = corrections->Get1(i);
00354 int index = GetIndex(annotee);
00355 if (index == Size())
00356 {
00357 Add(annotee, corrections->Get2(i));
00358 }
00359 else
00360 {
00361 Set2(index, corrections->Get2(i));
00362 }
00363 }
00364 }
00365
00366 int
00367 Diff(AnnotationTable* arg) const
00368 {
00369 if (Size() != arg->Size())
00370 {
00371 ILOG_ERROR("Diff: Size differs: " << Size() << " vs " <<
00372 arg->Size());
00373 return 1;
00374 }
00375 int nDiff = 0;
00376 for (int i=0 ; i<Size() ; i++)
00377 {
00378 if (Get1(i) != arg->Get1(i))
00379 {
00380 ILOG_DEBUG("Quid " << i << " differs " << Get1(i) <<
00381 " vs " << arg->Get1(i));
00382 nDiff++;
00383 }
00384 else if (::fabs(Get2(i) -arg->Get2(i)) > 0.00001)
00385 {
00386 ILOG_DEBUG("Anno " << i << " differs " << Get2(i) <<
00387 " vs " << arg->Get2(i));
00388 nDiff++;
00389 }
00390 }
00391 if (nDiff > 0)
00392 ILOG_ERROR("Found " << nDiff << " differences");
00393 return nDiff;
00394 }
00395
00396 virtual void
00397 Dump(int from = 0, int to = -1)
00398 {
00399 if (to == -1 || to > Size())
00400 to = Size();
00401 if (to < from)
00402 to = from;
00403 if (to <= from)
00404 return ;
00405
00406 std::cout << "Dumping table from " << from << " to " << to
00407 << " (table size=" << Size() << ", capacity=" << Capacity()
00408 << ")" << std::endl;
00409 for (int i=0 ; i<2 ; i++)
00410 std::cout << i+1 << "=" << GetColName(i+1) << ", ";
00411 std::cout << std::endl;
00412 for (int i=from ; i<to ; i++)
00413 {
00414 std::cout << QuidObj(Get1(i)) << ", " << Get2(i) << std::endl;
00415 }
00416 std::cout << std::endl;
00417 }
00418
00419 virtual void
00420 Dump(Database::RawDataSet* dataSet, int from = 0, int to = -1)
00421 {
00422 if (dataSet == 0)
00423 return Dump(from, to);
00424
00425 if (to == -1 || to > Size())
00426 to = Size();
00427 if (to < from)
00428 to = from;
00429 if (to <= from)
00430 return ;
00431
00432 std::cout << "Dumping table from " << from << " to " << to
00433 << " (table size=" << Size() << ", capacity=" << Capacity()
00434 << ")" << std::endl;
00435 for (int i=0 ; i<2 ; i++)
00436 std::cout << i+1 << "=" << GetColName(i+1) << ", ";
00437 std::cout << std::endl;
00438 for (int i=from ; i<to ; i++)
00439 {
00440 Quid quid = Get1(i);
00441 std::cout << dataSet->QuidToString(quid, true) << ", " << Get2(i)
00442 << std::endl;
00443 }
00444 std::cout << std::endl;
00445 }
00446
00447 void
00448 DumpSummary()
00449 {
00450 int pos=0, neg=0, skip=0;
00451 for (int i=0 ; i<Size() ; ++i)
00452 {
00453 if (IsPositive(i))
00454 ++pos;
00455 else if (IsNegative(i))
00456 ++neg;
00457 else
00458 ++skip;
00459 }
00460 std::cout << GetLabel() << " : " << pos << " positive, " << neg
00461 << " negative, " << skip << " skip, " << Size() << " total"
00462 << std::endl;
00463 }
00464
00465 void
00466 DumpSummaryObject()
00467 {
00468 std::vector<int> objs = CollectQuidObjects(this);
00469 for (int i=0 ; i<objs.size() ; i++)
00470 {
00471 std::cout << " obj=" << objs[i] << " : ";
00472 std::vector<int> oneObj;
00473 oneObj.push_back(objs[i]);
00474 CriterionQuidObjectInSet<AnnotationTable> crit(oneObj);
00475 AnnotationTable* tmp = Select(this, crit);
00476 tmp->DumpSummary();
00477 delete tmp;
00478 }
00479 }
00480
00481
00486 std::vector<QuidTable*>
00487 MakeEpisodeFolds(int foldCount, int repetition)
00488 {
00489 ILOG_DEBUG("entering MakeEpisodeFolds");
00490
00491 int quidclass = GetQuidClass();
00492 if(quidclass != QUID_CLASS_SHOT &&
00493 quidclass != QUID_CLASS_KEYFRAME &&
00494 quidclass != QUID_CLASS_FRAME)
00495 {
00496 ILOG_WARNING_HEADNODE("annotees of wrong quidclass, cannot do episode " <<
00497 "constrained");
00498 return MakeRandomFolds(foldCount, repetition);
00499 }
00500
00501 if(foldCount <= 1)
00502 {
00503 ILOG_ERROR("we need at least 2 folds");
00504 return std::vector<QuidTable*>(1, GetAnnotatedQuids());
00505 }
00506
00507 QuidTable* positive = GetPositive();
00508 QuidTable* negative = GetNegative();
00509 ILOG_DEBUG_NODE("make episode folds: #annotations = " <<
00510 positive->Size() + negative->Size());
00511
00512
00513 std::vector<int> positiveVideos;
00514 std::vector<int> negativeVideos;
00515 GroupPerVideo(positive, negative, positiveVideos, negativeVideos);
00516 delete positive;
00517 delete negative;
00518
00519 std::vector<QuidTable*> folds;
00520 for(int i=0 ; i<foldCount ; i++)
00521 folds.push_back(new QuidTable(0));
00522
00523
00524 int lastFold=0;
00525 Util::QuasiRandomSequenceIterator pos(positiveVideos.size(),repetition);
00526 for(int i=0 ; i<positiveVideos.size() ; ++i)
00527 {
00528 CriterionQuidObjectEquals<AnnotationTable>
00529 c(positiveVideos[*pos]);
00530 SelectOneColumn(folds[lastFold], this, 1, c, false);
00531 ++pos;
00532 ++lastFold;
00533 if(lastFold >= foldCount)
00534 lastFold = 0;
00535 }
00536
00537
00538 Util::QuasiRandomSequenceIterator neg(negativeVideos.size(),repetition);
00539 for(int i=0 ; i<negativeVideos.size() ; ++i)
00540 {
00541 CriterionQuidObjectEquals<AnnotationTable>
00542 c(negativeVideos[*neg]);
00543 SelectOneColumn(folds[lastFold], this, 1, c, false);
00544 ++neg;
00545 ++lastFold;
00546 if(lastFold >= foldCount)
00547 lastFold = 0;
00548 }
00549
00550 ILOG_DEBUG("leaving MakeFolds");
00551 return folds;
00552 }
00553
00554 std::vector<QuidTable*>
00555 MakeRandomFolds(int foldCount, int repetition)
00556 {
00557 if(foldCount <= 1)
00558 {
00559 ILOG_ERROR("we need at least 2 folds");
00560 return std::vector<QuidTable*>(1, GetAnnotatedQuids());
00561 }
00562
00563 QuidTable* positive = GetPositive();
00564 QuidTable* negative = GetNegative();
00565 ILOG_DEBUG_NODE("make random folds: #annotations = " <<
00566 positive->Size() + negative->Size());
00567
00568 std::vector<QuidTable*> folds;
00569 for(int i=0 ; i<foldCount ; ++i)
00570 folds.push_back(new QuidTable(0));
00571
00572 int lastFold=0;
00573 Util::QuasiRandomSequenceIterator pos(positive->Size(), repetition);
00574 for(int i=0 ; i<positive->Size() ; ++i)
00575 {
00576 folds[lastFold]->Add(positive->Get1(*pos));
00577 ++pos;
00578 ++lastFold;
00579 if(lastFold >= foldCount)
00580 lastFold = 0;
00581 }
00582 for (int i=0 ; i<foldCount ; i++)
00583 {
00584 ILOG_DEBUG("fold "<< i <<" has "<< folds[i]->Size() <<" positives");
00585 if (folds[i]->Size() == 0)
00586 ILOG_ERROR("fold "<< i <<" has NO positives");
00587 }
00588
00589 Util::QuasiRandomSequenceIterator neg(negative->Size(), repetition);
00590 for(int i=0 ; i<negative->Size() ; ++i)
00591 {
00592 folds[lastFold]->Add(negative->Get1(*neg));
00593 ++neg;
00594 ++lastFold;
00595 if(lastFold >= foldCount)
00596 lastFold = 0;
00597 }
00598
00599 delete positive;
00600 delete negative;
00601 return folds;
00602 }
00603
00610 void Sort()
00611 {
00612 Util::QuickSortDescCo
00613 (GetColumn2()->GetData(), GetColumn1()->GetData(), 0, Size()-1);
00614
00615 int to;
00616 for(int i=0 ; i<Size() ; ++i)
00617 {
00618 to = i;
00619 if(Get2(i) != 1)
00620 break;
00621 }
00622 Util::QuickSort(GetColumn1()->GetData(), 0, to-1);
00623
00624 int from;
00625 for(int i=to ; i<Size() ; ++i)
00626 {
00627 from = i;
00628 if(Get2(i) == -1)
00629 break;
00630 }
00631 Util::QuickSort(GetColumn1()->GetData(), from, Size()-1);
00632 }
00633
00634 private:
00635
00636 String mLabel;
00637 bool mIsAscending;
00638 Util::SimpleMap<Quid, int> mQuidMap;
00639
00640
00641
00642
00643 void GroupPerVideo(const QuidTable* positive,
00644 const QuidTable* negative,
00645 std::vector<int>& positiveVideos,
00646 std::vector<int>& negativeVideos)
00647 {
00648 Quid lastQuid=0;
00649
00650 for(int i=0 ; i<positive->Size() ; i++)
00651 {
00652 Quid q = positive->Get1(i);
00653
00654
00655 if(!QuidObjectEqual(lastQuid,q))
00656 {
00657 lastQuid = q;
00658 int videoID = QuidObject(q);
00659 if(find(positiveVideos.begin(), positiveVideos.end(), videoID)
00660 == positiveVideos.end())
00661 {
00662 positiveVideos.push_back(videoID);
00663 }
00664 }
00665 }
00666
00667 lastQuid=0;
00668
00669
00670 for(int i=0 ; i<negative->Size() ; i++)
00671 {
00672 Quid q = negative->Get1(i);
00673 if(!QuidObjectEqual(lastQuid,q))
00674 {
00675 lastQuid = q;
00676 int videoID = QuidObject(q);
00677 if(find(positiveVideos.begin(), positiveVideos.end(), videoID)
00678 == positiveVideos.end())
00679 {
00680 if(find(negativeVideos.begin(), negativeVideos.end(),
00681 videoID) == negativeVideos.end())
00682 {
00683 negativeVideos.push_back(videoID);
00684 }
00685 }
00686 }
00687 }
00688 }
00689
00690
00691
00692 ILOG_VAR_DEC;
00693 };
00694
00695 ILOG_VAR_INIT(AnnotationTable, Impala.Core.Table);
00696
00697 }
00698 }
00699 }
00700
00701 #endif