Impala Documentation

00001 #ifndef Impala_Core_IDash_VideoSetWrapper_h
00002 #define Impala_Core_IDash_VideoSetWrapper_h
00003 
00004 #include "Core/IDash/QuerySetTable.h"
00005 #include "Core/IDash/VideoTable.h"
00006 #include "Core/IDash/XmlVideo.h"
00007 #include "Core/IDash/XmlShot.h"
00008 #include "Core/IDash/XmlCase.h"
00009 #include "Core/IDash/XmlQuerySet.h"
00010 #include "Core/IDash/XmlVideoList.h"
00011 #include "Core/IDash/XmlAnnotationSet.h"
00012 #include "Core/VideoSet/MakeVideoSet.h"
00013 #include "Core/VideoSet/Segmentation.h"
00014 #include "Core/VideoSet/Keyframes.h"
00015 #include "Core/Array/ArrayListDelete.h"
00016 #include "Core/Array/ReadImage.h"
00017 #include "Core/Array/Scale.h"
00018 #include "Core/Array/WritePng.h"
00019 #include "Core/Table/SimilarityTableSet.h"
00020 
00021 namespace Impala
00022 {
00023 namespace Core
00024 {
00025 namespace IDash
00026 {
00027 
00028 
00029 class VideoSetWrapper
00030 {
00031 public:
00032 
00033     typedef Table::SimilarityTableSet SimilarityTableSet;
00034     typedef Table::SimilarityTableSet::SimTableType SimTableType;
00035     typedef Table::SimilarityTableSet::RankTableType RankTableType;
00036 
00037     VideoSetWrapper(String setName)
00038     {
00039         mVidSet = Core::VideoSet::MakeVideoSet(setName);
00040         bool first = (mVidSet->NrFiles() == 0);
00041         String segName = (first) ? "" : "segmentation";
00042         mSegm = new VideoSet::Segmentation(mVidSet, segName);
00043         String keyName = (first) ? "" : "keyframes";
00044         mKeys = new VideoSet::Keyframes(mVidSet, keyName);
00045         String vidTableName = (first) ? "" : "videotable";
00046         mVidTable = new VideoTable(mVidSet, vidTableName);
00047         mQuerySetTable = new QuerySetTable(mVidSet, "querysettable");
00048     }
00049 
00050     ~VideoSetWrapper()
00051     {
00052         delete mQuerySetTable;
00053         delete mVidTable;
00054         delete mKeys;
00055         delete mSegm;
00056         delete mVidSet;
00057     }
00058 
00059     VideoSet::VideoSet*
00060     GetVideoSet()
00061     {
00062         return mVidSet;
00063     }
00064 
00065     Quid
00066     Add(const XmlVideo& xmlVideo, int singleShot)
00067     {
00068         int firstShotToAdd = (singleShot >= 0) ? singleShot : 0;
00069         int lastShotToAdd = (singleShot >= 0) ? singleShot+1 : xmlVideo.GetShots();
00070         String fName = "case_" + xmlVideo.GetCaseId() + "_video_"
00071             + xmlVideo.GetId() + "_file_" + xmlVideo.GetFileName();
00072         ILOG_INFO("Adding video " << fName << " with "
00073                   << lastShotToAdd - firstShotToAdd << " shots");
00074         if (mVidSet->GetFileId(fName) != -1)
00075         {
00076             ILOG_ERROR("Video " << fName << " already present in set");
00077             return 0;
00078         }
00079         if (! xmlVideo.IsProcessed())
00080         {
00081             ILOG_ERROR("Video " << fName << " is not yet processed");
00082             return 0;
00083         }
00084         if (xmlVideo.GetShots() <= 0)
00085         {
00086             ILOG_ERROR("Video " << fName << " does not have shots");
00087             return 0;
00088         }
00089 
00090         std::vector<int> frames;
00091         std::vector<Array::Array2dScalarUInt8*> imList;
00092         for (int i=firstShotToAdd ; i<lastShotToAdd ; i++)
00093         {
00094             String href = xmlVideo.GetShotLinkXml(i);
00095             XmlShot xmlShot(href);
00096             if (!xmlShot.Valid())
00097             {
00098                 ILOG_ERROR("Shot " << i << " of video " << fName <<
00099                            " has invalid XML");
00100                 Array::ArrayListDelete(&imList);
00101                 return 0;
00102             }
00103             frames.push_back(xmlShot.GetFrame());
00104 
00105             href = xmlVideo.GetShotLinkImage(i);
00106             Link::Curl::Memory mem = Link::Curl::Get(href);
00107             if (mem.size == 0)
00108             {
00109                 ILOG_ERROR("Shot " << i << " of video " << fName <<
00110                            " is missing");
00111                 Array::ArrayListDelete(&imList);
00112                 return 0;
00113             }
00114             Array::Array2dScalarUInt8* data =
00115                 Array::MakeFromData<Array::Array2dScalarUInt8>((UInt8*) mem.data,
00116                                                                mem.size, 1);
00117             imList.push_back(data);
00118             delete mem.data;
00119         }
00120         frames.push_back(xmlVideo.GetFrames()-1);
00121         CheckFrameSizes(imList);
00122 
00123         // data is OK, now really add it
00124         mVidSet->AddFile(fName, -1);
00125         int vidId = mVidSet->GetFileId(fName);
00126         mVidTable->Add(vidId, xmlVideo.GetId(), xmlVideo.GetCaseId(),
00127                        xmlVideo.GetLink());
00128         for (int i=firstShotToAdd ; i<lastShotToAdd ; i++)
00129         {
00130             String sName = "shot_" + MakeString(vidId) + "_" + MakeString(i);
00131             int idx = i - firstShotToAdd;
00132             mSegm->Add(vidId, frames[idx], frames[idx+1]-1, sName); // not ok single
00133             int shotId = mSegm->Size() - 1;
00134             String kName = "keyfr_" + MakeString(vidId) + "_" + MakeString(i);
00135             mKeys->Add(vidId, shotId, frames[idx], kName);
00136         }
00137 
00138         mVidSet->UpdateVideoSet();
00139         mSegm->Update("segmentation");
00140         mKeys->Update("keyframes");
00141         mKeys->WriteImageSets(true);
00142         mVidTable->Update("videotable");
00143 
00144         String arName = mVidSet->GetFilePathFrames(vidId, "images.raw",
00145                                                    true, false);
00146         Array::WriteRawListVar(imList, arName, mVidSet->GetDatabase(), true,
00147                                true);
00148         Array::ArrayListDelete(&imList);
00149         return mVidSet->GetQuidFrame(vidId, frames[0]);
00150     }
00151 
00152     void
00153     Add(const XmlVideo& xmlVideo, const XmlQuerySet& xmlQuerySet)
00154     {
00155         mQuerySetTable->Add(xmlQuerySet.GetId(), xmlQuerySet.GetVideosLink(),
00156                             xmlQuerySet.GetLink(), xmlQuerySet.GetCaseId());
00157         mQuerySetTable->Update("querysettable");
00158         Add(xmlVideo, -1);
00159     }
00160 
00161     void
00162     Add(const XmlCase& xmlCase)
00163     {
00164         XmlVideoList xmlVideoList(xmlCase.GetVideosLink());
00165         for (int i=0 ; i<xmlVideoList.GetNrVideos() ; i++)
00166         {
00167             XmlVideo xmlVideo(xmlVideoList.GetVideo(i));
00168             Add(xmlVideo, -1);
00169         }
00170     }
00171 
00172     void
00173     Add(const XmlQuerySet& xmlQuerySet)
00174     {
00175         mQuerySetTable->Add(xmlQuerySet.GetId(), xmlQuerySet.GetVideosLink(),
00176                             xmlQuerySet.GetLink(), xmlQuerySet.GetCaseId());
00177         mQuerySetTable->Update("querysettable");
00178 
00179         XmlVideoList xmlVideoList(xmlQuerySet.GetVideosLink());
00180         for (int i=0 ; i<xmlVideoList.GetNrVideos() ; i++)
00181         {
00182             XmlVideo xmlVideo(xmlVideoList.GetVideo(i));
00183             Add(xmlVideo, -1);
00184         }
00185     }
00186 
00187     void
00188     Add(const XmlAnnotationSet& xmlAnnotationSet)
00189     {
00190         for (int i=0 ; i<xmlAnnotationSet.GetNrMoments() ; i++)
00191         {
00192             XmlShot xmlShot(xmlAnnotationSet.GetMoment(i));
00193             XmlVideo xmlVideo(xmlShot.GetVideo());
00194             Add(xmlVideo, xmlShot.GetShotIndex());
00195         }
00196     }
00197 
00198     String
00199     GetAsRankingList(SimilarityTableSet* simSet, String baseUri, String caseId,
00200                      String qSetId, String vidId, String conceptName,
00201                      int maxNumber, double probabilityThreshold)
00202     {
00203         RankTableType* rankTable = simSet->GetRankTable(conceptName);
00204         if (!rankTable)
00205         {
00206             ILOG_ERROR("No ranking for " << conceptName);
00207             return "ERROR: 400: No ranking for " + conceptName;
00208         }
00209         SimTableType* simTable = simSet->GetSimTable(conceptName);
00210 
00211         String s = "<?xml version=\"1.0\"?>";
00212         s += "<ranking xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xmlns=\"http://www.i-dash.eu/ActionData\">";
00213         s += "  <queryURL xlink:type=\"simple\" xlink:href=\"" + baseUri
00214             + "cases/" + caseId + "/concepts/" + conceptName + "\" />";
00215         s += "  <rankingItems>";
00216 
00217         int nrAdded = 0;
00218         for (int i=0 ; i<rankTable->Size() && nrAdded<maxNumber; i++)
00219         {
00220             Quid q = rankTable->Get1(i);
00221             int idx = simSet->FindQuid(q);
00222             double sim = simTable->Get1(idx);
00223             if (sim < probabilityThreshold)
00224                 break;
00225             if (IsInTargetSet(q, baseUri, caseId, qSetId, vidId))
00226             {
00227                 String item = GetAsRankingItem(q, sim, conceptName);
00228                 s += item;
00229                 nrAdded++;
00230             }
00231         }
00232         s += "  </rankingItems>";
00233         s += "</ranking>";
00234         return s;
00235     }
00236 
00237 private:
00238 
00239     void
00240     CheckFrameSizes(std::vector<Array::Array2dScalarUInt8*>& imList)
00241     {
00242         int bufSize = 1000000;
00243         char* buf = new char[bufSize];
00244         for (int i=0 ; i<imList.size() ; i++)
00245         {
00246             Array::Array2dScalarUInt8* data = imList[i];
00247             Array::Array2dVec3UInt8* im = 0;
00248             Array::ReadImageFromMemory(im, (char*) data->CPB(), data->CW());
00249             const int maxWidth = 512;
00250             if (im->CW() > maxWidth)
00251             {
00252                 ILOG_INFO("imSize=" << im->CW() << "x" << im->CH() <<
00253                           ", adjusting");
00254                 double factor = 0.5;
00255                 while (factor * im->CW() > maxWidth)
00256                     factor *= 0.5;
00257                 Array::Array2dVec3UInt8* sc = 0;
00258                 Array::Scale(sc, im, factor, factor, Geometry::NEAREST, true);
00259 
00260                 size_t nrBytes = 0;
00261                 Array::WritePngToMemory(sc, buf, bufSize, &nrBytes);
00262                 Array::Array2dScalarUInt8* bufIm =
00263                     Array::MakeFromData<Array::Array2dScalarUInt8>((UInt8*) buf,
00264                                                                    nrBytes, 1);
00265                 delete data;
00266                 imList[i] = bufIm;
00267                 delete sc;
00268             }
00269             delete im;
00270         }
00271         delete buf;
00272     }
00273 
00274     String
00275     GetAsRankingItem(Quid quid, Real64 sim, CString conceptName)
00276     {
00277         int vidId = QuidObject(quid);
00278         int shotIdx = mSegm->GetShotId(quid) - mSegm->GetFirstShotVideo(vidId);
00279 
00280         String url = mVidTable->GetUrl(vidId) + "/shots/" + MakeString(shotIdx);
00281         String s = "    <rankingItem>";
00282         s += "    <entity xlink:href=\"" + url
00283             + "\" xlink:type=\"simple\" entityType=\"Moment\" />";
00284         s += "    <probability>" + MakeString(sim) + "</probability>";
00285         s += "    <tag>" + conceptName + "</tag>";
00286         s += "    </rankingItem>";
00287         return s;
00288     }
00289 
00290     bool
00291     IsInTargetSet(Quid quid, String baseUri, String caseId, String qSetId,
00292                   String vdsVidId)
00293     {
00294         int vidId = QuidObject(quid);
00295         if (qSetId.empty())
00296         {
00297             if (mVidTable->GetCaseId(vidId) != caseId)
00298                 return false;
00299             if (vdsVidId.empty())
00300                 return true;
00301             return (mVidTable->GetVdsVidId(vidId) == vdsVidId);
00302         }
00303         if (! ((qSetId == mLastQSetId) && (caseId == mLastCaseId)))
00304         {
00305             String ref = mQuerySetTable->GetUrl(qSetId, caseId);
00306             XmlQuerySet qSet(ref);
00307             if (!qSet.Valid())
00308                 return false;
00309             XmlVideoList vList(qSet.GetVideosLink());
00310             mQSetList = vList.GetVideos();
00311             mLastQSetId = qSetId;
00312             mLastCaseId = caseId;
00313         }
00314         String vidUrl = mVidTable->GetUrl(vidId);
00315         return (find(mQSetList.begin(), mQSetList.end(), vidUrl) !=
00316                      mQSetList.end());
00317     }
00318 
00319     VideoSet::VideoSet*     mVidSet;
00320     VideoSet::Segmentation* mSegm;
00321     VideoSet::Keyframes*    mKeys;
00322     VideoTable*             mVidTable;
00323     QuerySetTable*          mQuerySetTable;
00324 
00325     String                  mLastQSetId;
00326     String                  mLastCaseId;
00327     std::vector<String>     mQSetList;
00328 
00329     ILOG_VAR_DEC;
00330 
00331 };
00332 
00333 ILOG_VAR_INIT(VideoSetWrapper, Impala.Core.IDash);
00334 
00335 } // namespace IDash
00336 } // namespace Core
00337 } // namespace Impala
00338 
00339 #endif
VideoSetWrapper.h
