00001 #ifndef Impala_Core_VideoSet_IndexAnnotation_h
00002 #define Impala_Core_VideoSet_IndexAnnotation_h
00003
00004 #include "Persistency/AnnotationTableSetRepository.h"
00005 #include "Persistency/Mpeg7DocRepository.h"
00006 #include "Persistency/QuidTableRepository.h"
00007 #include "Core/Table/AnnotationTableSet.h"
00008 #include "Core/Table/Sort.h"
00009 #include "Core/VideoSet/Reporter.h"
00010 #include "Core/VideoSet/Mpeg7Doc.h"
00011
00012 namespace Impala
00013 {
00014 namespace Core
00015 {
00016 namespace VideoSet
00017 {
00018
00019
00020 class IndexAnnotation : public Listener
00021 {
00022 public:
00023
00024 IndexAnnotation(Reporter* reporter, CmdOptions& options)
00025 {
00026 mReporter = reporter;
00027 if (options.GetNrArg() < 3)
00028 {
00029 ILOG_ERROR("Missing argument");
00030 return;
00031 }
00032 mConceptSet = options.GetArg(2);
00033 mConceptExtra = "";
00034 if (options.GetNrArg() > 3)
00035 mConceptExtra = options.GetArg(3);
00036 mNrExtra = 0;
00037 if (options.GetNrArg() > 4)
00038 mNrExtra = atol(options.GetArg(4));
00039 mKeyframes = 0;
00040 mSegmentation = 0;
00041 mRkfMask = options.GetBool("rkfMask");
00042 }
00043
00044 virtual void
00045 HandleNewWalk(VideoSet* vs)
00046 {
00047 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00048 mAnnoShot = Table::AnnotationTableSet::MakeFromFile(vs, mConceptSet,
00049 false, 0);
00050 mAnnoFrame = Table::AnnotationTableSet::MakeFromFile(vs, mConceptSet,
00051 false, 0);
00052 #else // REPOSITORY_USED
00053 Persistency::KeywordListLocator loc(vs->GetLocator(), mConceptSet);
00054 Table::KeywordList keyList =
00055 *(Persistency::KeywordListRepository().Get(loc));
00056 mAnnoShot = new Table::AnnotationTableSet(keyList, 0);
00057 mAnnoFrame = new Table::AnnotationTableSet(keyList, 0);
00058 #endif // REPOSITORY_USED
00059 if (! (mAnnoShot && mAnnoFrame))
00060 {
00061 ILOG_ERROR("Unable to read conceptSet");
00062 }
00063 mAnnoExtra = 0;
00064 mQuidExtra = 0;
00065 if (!mConceptExtra.empty())
00066 {
00067 ILOG_INFO("Reading extra : " << mConceptExtra);
00068 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00069 mAnnoExtra = Table::AnnotationTableSet::MakeFromFile(vs,
00070 mConceptExtra,
00071 false, 0);
00072 #else // REPOSITORY_USED
00073 loc.SetConceptSet(mConceptExtra);
00074 keyList = *(Persistency::KeywordListRepository().Get(loc));
00075 mAnnoExtra = new Table::AnnotationTableSet(keyList, 0);
00076 #endif // REPOSITORY_USED
00077 if (!mAnnoExtra)
00078 {
00079 ILOG_ERROR("Unable to read conceptExtra");
00080 }
00081 mQuidExtra = new Table::QuidTable(100);
00082 }
00083 int nr = mAnnoShot->Size();
00084 mTotalNrAnno.resize(nr);
00085 mTotalNrSingleFrame.resize(nr);
00086 mTotalNrFrames.resize(nr);
00087 mTotalNrAnnoPos.resize(nr);
00088 mTotalNrSingleFramePos.resize(nr);
00089 mTotalNrFramesPos.resize(nr);
00090 }
00091
00092 virtual void
00093 HandleNewWalk(VideoSet* vs, Keyframes* keyframes, bool* keyframeMask)
00094 {
00095 mKeyframes = keyframes;
00096 }
00097
00098 virtual void
00099 HandleNewWalk(VideoSet* vs, Segmentation* segmentation)
00100 {
00101 mSegmentation = segmentation;
00102 }
00103
00104 virtual void
00105 HandleNewFile(VideoSet* vs, int fileId, Stream::RgbDataSrc* src)
00106 {
00107 if (mAnnoExtra && (!(mKeyframes && mSegmentation)))
00108 {
00109 ILOG_ERROR("Need keyframes and segmentation for extra annotations");
00110 delete mAnnoExtra;
00111 mAnnoExtra = 0;
00112 delete mQuidExtra;
00113 mQuidExtra = 0;
00114 }
00115 int frameRateNum = (src) ? src->FrameRateNum() : 0;
00116 int frameRateDen = (src) ? src->FrameRateDen() : 0;
00117
00118 for (int c=0 ; c<mAnnoShot->Size() ; c++)
00119 {
00120 String concept = mAnnoShot->GetLabel(c);
00121 String container = mConceptSet + "/" + vs->GetContainerFile(fileId);
00122 typedef Persistency::Mpeg7DocLocator MP7Loc;
00123 MP7Loc loc(vs->GetLocator(), MP7Loc::MPEG7_ANNOTATION, container,
00124 concept + ".xml");
00125 if (!Persistency::Mpeg7DocRepository().Exists(loc))
00126 continue;
00127
00128 Mpeg7Doc* mp7 = Persistency::Mpeg7DocRepository().Get(loc);
00129 for (int s=0 ; s<mp7->NrShots() ; s++)
00130 {
00131 mTotalNrAnno[c]++;
00132 int len = mp7->EndFrame(s) - mp7->StartFrame(s);
00133 if (len <= 1)
00134 mTotalNrSingleFrame[c]++;
00135 else
00136 mTotalNrFrames[c] += len;
00137 if (mp7->IsPositiveAnno(s))
00138 {
00139 mTotalNrAnnoPos[c]++;
00140 if (len <= 1)
00141 mTotalNrSingleFramePos[c]++;
00142 else
00143 mTotalNrFramesPos[c] += len;
00144 }
00145 }
00146 if (mKeyframes)
00147 DoKeyframes(vs, fileId, c, mp7);
00148 if (mSegmentation)
00149 DoShots(vs, fileId, c, mp7);
00150 delete mp7;
00151 }
00152 }
00153
00154 virtual void
00155 HandleDoneWalk(VideoSet* vs)
00156 {
00157 Persistency::AnnotationTableSetLocator loc(vs->GetLocator(), 0,
00158 mConceptSet);
00159 for (int c=0 ; c<mAnnoShot->Size() ; c++)
00160 {
00161 String concept = mAnnoShot->GetLabel(c);
00162 int nrShot = mTotalNrAnno[c] - mTotalNrSingleFrame[c];
00163 double avg = (double) mTotalNrFrames[c] / nrShot;
00164 if (nrShot == 0)
00165 avg = 1;
00166 ILOG_INFO("Indexed " << mTotalNrAnno[c] << " annotations for "
00167 << concept);
00168 ILOG_INFO("total of " << mTotalNrSingleFrame[c] << " frames and "
00169 << nrShot << " shots with avg length of " << avg);
00170 nrShot = mTotalNrAnnoPos[c] - mTotalNrSingleFramePos[c];
00171 avg = (double) mTotalNrFramesPos[c] / nrShot;
00172 if (nrShot == 0)
00173 avg = 1;
00174 ILOG_INFO("postives: total of " << mTotalNrSingleFramePos[c] <<
00175 " frames and " << nrShot << " shots with avg length of "
00176 << avg);
00177 }
00178 if (mSegmentation)
00179 {
00180 ILOG_INFO("Number of shots " << mSegmentation->GetNrShots());
00181 ILOG_INFO("Saving shot annotations");
00182 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00183 mAnnoShot->SaveTables(vs, mConceptSet, true);
00184 #else // REPOSITORY_USED
00185 Persistency::AnnotationTableSetRepository().Add(loc, mAnnoShot);
00186 #endif // REPOSITORY_USED
00187 }
00188 if (mKeyframes)
00189 {
00190 ILOG_INFO("Number of keyframes " << mKeyframes->GetNrKeyframes());
00191 ILOG_INFO("Saving keyframe annotations");
00192 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00193 mAnnoFrame->SaveTables(vs, mConceptSet, true);
00194 #else // REPOSITORY_USED
00195 Persistency::AnnotationTableSetRepository().Add(loc, mAnnoFrame);
00196 #endif // REPOSITORY_USED
00197 }
00198 if (mAnnoExtra)
00199 {
00200 ILOG_INFO("Saving extra annotations");
00201 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00202 mAnnoExtra->SaveTables(vs, mConceptExtra, true);
00203 #else // REPOSITORY_USED
00204 loc.SetConceptSet(mConceptExtra);
00205 Persistency::AnnotationTableSetRepository().Add(loc, mAnnoExtra);
00206 #endif // REPOSITORY_USED
00207 ILOG_INFO("Sorting and saving quids");
00208 Table::Sort(mQuidExtra, 1, true);
00209 String qName = FileNameBase(mConceptExtra) + "_quids.tab";
00210 #ifndef REPOSITORY_USED // Here comes the deprecated stuff
00211 String fName = vs->GetFilePathAnnotation(qName, true, false);
00212 if (!fName.empty())
00213 Table::Write(mQuidExtra, fName, vs->GetDatabase(), true);
00214 #else // REPOSITORY_USED
00215 Persistency::QuidTableLocator qLoc(vs->GetLocator(), "Annotations",
00216 qName);
00217 Persistency::QuidTableRepository().Add(qLoc, mQuidExtra);
00218 #endif // REPOSITORY_USED
00219 }
00220 }
00221
00222 private:
00223
00224 void
00225 AddExtra(VideoSet* vs, int fileId, int concept, Mpeg7Doc* mp7, int key,
00226 int idx)
00227 {
00228 Table::AnnotationTable* frameExtra = mAnnoExtra->GetTable(concept);
00229 int shot = mKeyframes->GetShotId(key);
00230 int begin = Max(mSegmentation->GetStart(shot), mp7->StartFrame(idx));
00231 int end = Max(mSegmentation->GetEnd(shot), mp7->EndFrame(idx));
00232 int step = (end - begin) / (mNrExtra + 2);
00233 int frame = begin + step;
00234 while (frame <= end - step)
00235 {
00236 Quid q = MakeQuidFrame(vs->GetSetId(), fileId, frame);
00237 if (!frameExtra->IsAnnotated(q))
00238 {
00239 frameExtra->AddPositive(q);
00240 if (!mQuidExtra->Contains(q))
00241 mQuidExtra->Add(q);
00242 }
00243 frame += step;
00244 }
00245 }
00246
00247 void
00248 DoKeyframes(VideoSet* vs, int fileId, int concept, Mpeg7Doc* mp7)
00249 {
00250 Table::AnnotationTable* frameTable = mAnnoFrame->GetTable(concept);
00251 Table::AnnotationTable* frameExtra = 0;
00252 if (mAnnoExtra)
00253 frameExtra = mAnnoExtra->GetTable(concept);
00254 int startKey = mKeyframes->GetFirstKeyframeVideo(fileId);
00255 int endKey = startKey + mKeyframes->GetNrKeyframesVideo(fileId);
00256 for (int k=startKey ; k<endKey ; k++)
00257 {
00258 if (mRkfMask && (!mKeyframes->IsRKF(k)))
00259 continue;
00260 int frame = mKeyframes->GetFrameNr(k);
00261 int idx = mp7->FindShot(frame);
00262 if (idx != -1)
00263 {
00264 Quid q = MakeQuidFrame(vs->GetSetId(), fileId, frame);
00265 if (mQuidExtra && (!mQuidExtra->Contains(q)))
00266 mQuidExtra->Add(q);
00267 if (mp7->IsPositiveAnno(idx))
00268 {
00269 frameTable->AddPositive(q);
00270 if (frameExtra)
00271 {
00272 frameExtra->AddPositive(q);
00273 int len = mp7->EndFrame(idx) - mp7->StartFrame(idx);
00274 if (len > 1)
00275 {
00276 AddExtra(vs, fileId, concept, mp7, k, idx);
00277 }
00278 }
00279 }
00280 else if (mp7->IsNegativeAnno(idx))
00281 {
00282 frameTable->AddNegative(q);
00283 if (frameExtra)
00284 frameExtra->AddNegative(q);
00285 }
00286 else
00287 {
00288 frameTable->AddSkip(q);
00289 if (frameExtra)
00290 frameExtra->AddSkip(q);
00291 }
00292 }
00293 }
00294 }
00295
00296 void
00297 DoShots(VideoSet* vs, int fileId, int concept, Mpeg7Doc* mp7)
00298 {
00299 Table::AnnotationTable* shotTable = mAnnoShot->GetTable(concept);
00300 int startShot = mSegmentation->GetFirstShotVideo(fileId);
00301 int endShot = startShot + mSegmentation->GetNrShotsVideo(fileId);
00302 for (int s=startShot ; s<endShot ; s++)
00303 {
00304 int startFrame = mSegmentation->GetStart(s);
00305 int endFrame = mSegmentation->GetEnd(s);
00306 std::vector<int> idx = mp7->FindShots(startFrame, endFrame);
00307
00308 if (idx.size() > 0)
00309 {
00310 Quid q = MakeQuidShot(vs->GetSetId(), fileId, s - startShot);
00311
00312 bool hasPos = false;
00313 bool hasNeg = false;
00314 for (int i=0 ; i<idx.size() ; i++)
00315 {
00316 if (mp7->IsPositiveAnno(idx[i]))
00317 hasPos = true;
00318 else if (mp7->IsNegativeAnno(idx[i]))
00319 hasNeg = true;
00320 }
00321
00322 if (hasPos)
00323 shotTable->AddPositive(q);
00324 else if (hasNeg)
00325 shotTable->AddNegative(q);
00326 else
00327 shotTable->AddSkip(q);
00328 }
00329 }
00330 }
00331
00332 Reporter* mReporter;
00333 String mConceptSet;
00334 String mConceptExtra;
00335 int mNrExtra;
00336 Segmentation* mSegmentation;
00337 Keyframes* mKeyframes;
00338 bool mRkfMask;
00339 std::vector<String> mConcepts;
00340 Table::AnnotationTableSet* mAnnoShot;
00341 Table::AnnotationTableSet* mAnnoFrame;
00342 Table::AnnotationTableSet* mAnnoExtra;
00343 Table::QuidTable* mQuidExtra;
00344
00345 std::vector<int> mTotalNrAnno;
00346 std::vector<int> mTotalNrSingleFrame;
00347 std::vector<int> mTotalNrFrames;
00348 std::vector<int> mTotalNrAnnoPos;
00349 std::vector<int> mTotalNrSingleFramePos;
00350 std::vector<int> mTotalNrFramesPos;
00351
00352 ILOG_VAR_DEC;
00353
00354 };
00355
00356 ILOG_VAR_INIT(IndexAnnotation, Impala.Core.VideoSet);
00357
00358 }
00359 }
00360 }
00361
00362 #endif