|| || || || || || || ||

---

TableDataSourceCSV.h

Go to the documentation of this file.

#ifndef MediaTable_TableDataSourceCSV_h
#define MediaTable_TableDataSourceCSV_h

#include "WritableTableDataSource.h"

#ifdef MEDIATABLE_BASIC
#include "RemoteRetriever.h"
#endif

#include <iostream>
#include <fstream>
#include <string>

namespace Impala {
namespace Application {
namespace MediaTable {

class TableDataSourceCSV : public WritableTableDataSource
{
public:
    TableDataSourceCSV(std::string filename):
        WritableTableDataSource()
    {
        Init();
        OpenCSV(filename);
    }

    ~TableDataSourceCSV()
    {
    }

    void OpenCSV(std::string filename) {
        std::ifstream is(filename.c_str());

                /* From: http://stackoverflow.com/questions/1120140/csv-parser-in-c
                 * Can also use boost:
                std::vector<std::string> vec;
                using namespace boost;
                tokenizer<escaped_list_separator<char> > tk(
                   line, escaped_list_separator<char>('\\', ',', '\"'));
                for (tokenizer<escaped_list_separator<char> >::iterator i(tk.begin());
                   i!=tk.end();++i)
                {
                   vec.push_back(*i);
                }*/

        std::vector<int> columnType;
        std::vector<int> newColumnType;
        std::vector<std::vector<std::string> > data;
        if(!is.good()) return;

                std::string line, cell;
        while(is.good()) {
                        //ILOG_DEBUG("is good");
                std::vector<std::string> row;
                std::getline(is, line);
                if(line.length() == 0) continue;
                std::stringstream lineStream(line);
                        //ILOG_INFO("LINE: " << line);
                        newColumnType = columnType;
                while(std::getline(lineStream,cell,',')) {
                        if(cell.length() > 1 &&
                                        (cell.at(0) == 0x27 || cell.at(0) == 0x22))
                        {
                                        // Fix CR/LF issue on Unixes
                                        if(cell.at(cell.length()-1) == '\r')
                                                cell = cell.substr(0, cell.length()-1);
                                while(cell.at(cell.length()-1) != 0x27 &&
                                          cell.at(cell.length()-1) != 0x22)
                                {
                                        std::string ncell;
                                        if(!std::getline(lineStream,ncell,','))
                                        {
                                                // Error!
                                                // CSV spec states that a new line is also allowed in a field.
                                        ILOG_DEBUG("Line " << data.size() + 1 << " has bad quoting.");
                                        row.clear();
                                                break;
                                        } else
                                                cell += ncell;
                                }
                                cell = cell.substr(1, cell.length()-2);
                        }
                        //printf(cell.c_str());
                        //printf(",");
                        row.push_back(cell);
                        if(data.size() == 0) {
                                newColumnType.push_back(TYPE_INT);
                        } else {
                                char result[100];
                                        if(row.size() > newColumnType.size())
                                        {
                                                // skipping overflow row
//                                              ILOG_DEBUG("Skipping overflow row");
                                                continue;
                                        }
                                if(newColumnType[row.size()-1] == TYPE_INT) {
                                        sprintf(result, "%d", atoi(cell.c_str()));
                                        if(std::string(result) != cell)
                                                newColumnType[row.size()-1] = TYPE_DOUBLE;
                                }
                                if(newColumnType[row.size()-1] == TYPE_DOUBLE) {
                                        sprintf(result, "%f", atof(cell.c_str()));
                                        if(strncmp(cell.c_str(), result, cell.length()))
                                                {
                                                        /*ILOG_DEBUG("On row " << data.size()+1 << " and column " << 
                                                                           data[0][row.size()-1] << ": " << 
                                                                           result << " != " << cell);*/
                                                newColumnType[row.size()-1] = TYPE_IMAGE;
                                                }
                                }
                                if(newColumnType[row.size()-1] == TYPE_IMAGE) {
                                                ILOG_DEBUG(cell.substr(0, 7));
                                                ILOG_DEBUG(cell.substr(cell.length()-4));
                                        if ((cell.length() < 13) || 
                                                    (cell.substr(0, 7) != "http://") ||
                                                   ((cell.substr(cell.length()-4) != ".jpg") && 
                                                        (cell.substr(cell.length()-5) != ".jpeg")))
                                                {
                                                        /*ILOG_DEBUG("On row " << data.size()+1 << " and column " << 
                                                         data[0][row.size()-1] << ": " << 
                                                         result << " != " << cell);*/
                                                newColumnType[row.size()-1] = TYPE_TEXT;
                                                }
                                }
                                }
                }
                        if(!row.size()) continue;
                if(data.size() && (row.size() != columnType.size())) {
                        ILOG_DEBUG("Line " << data.size() + 1 << " has wrong number of cells (" <<
                                                   row.size() << " should be " << columnType.size() << ")");
                                
                } else {
                        //printf("\n");
                        data.push_back(row);
                                // Only store columnType for valid rows
                                columnType = newColumnType;
                }
        }
        is.close();

                ILOG_DEBUG("phase 2");

#ifdef MEDIATABLE_BASIC
        int gotcha = 0;
        bool isFlickr = false;
        for(int col=0; col < columnType.size(); col++) {
                if(data[0][col] == "id") gotcha++;
                if(data[0][col] == "farm") gotcha++;
                if(data[0][col] == "server") gotcha++;
                if(data[0][col] == "secret") gotcha++;
        }
        if(gotcha == 4) {
                isFlickr = true;
        }
                mRemoteRetriever = RemoteRetriever::GetInstance();
        std::string id, farm, server, secret;
        std::string sizeSuffix = "t";
#endif
                ILOG_DEBUG("phase 3");

        for(int col=0; col < columnType.size(); col++) {
#ifdef MEDIATABLE_BASIC
                if(isFlickr)
                {
                        if(data[0][col] == "id")
                                        columnType[col] = TYPE_TEXT;
                        if(data[0][col] == "farm" | data[0][col] == "server")
                                continue;
                        if(data[0][col] == "secret")
                        {
                        AddStaticColumn("thumb", TYPE_IMAGE);
                        //AddStaticColumn("photo", TYPE_IMAGE);
                        continue;
                        }
                }
#endif
                AddStaticColumn(data[0][col], columnType[col]);
                ILOG_DEBUG("Added column " << data[0][col] << " of type " << columnType[col]);
        }

        for(int row=1; row < data.size(); row++) {
                for(int col=0; col < columnType.size(); col++) {
#ifdef MEDIATABLE_BASIC
                                if(isFlickr)
                                {
                                        if(data[0][col] == "id")
                                                id = data[row][col];
                                        if(data[0][col] == "farm")
                                        {
                                                farm = data[row][col];
                                                continue;
                                        }
                                        if(data[0][col] == "server")
                                        {
                                                server = data[row][col];
                                                continue;
                                        }
                                        if(data[0][col] == "secret")
                                        {
                                                secret = data[row][col];
                                        std::string imageUrl = "http://farm";
                                        imageUrl += farm + ".static.flickr.com/" + server + "/";
                                        imageUrl += id + "_" + secret;
                                                //AddTextData("photo", imageUrl + ".jpg");
                                        if(sizeSuffix.length() > 0) imageUrl += "_" + sizeSuffix;
                                                AddTextData("thumb", imageUrl + ".jpg");
                                                continue;
                                        }
                                }
#endif
                        if(columnType[col] == TYPE_INT)         AddIntData(data[0][col], atoi(data[row][col].c_str()));
                        if(columnType[col] == TYPE_DOUBLE)      AddDoubleData(data[0][col], atof(data[row][col].c_str()));
                        if(columnType[col] == TYPE_IMAGE)       AddTextData(data[0][col], data[row][col]);
                        if(columnType[col] == TYPE_TEXT)        AddTextData(data[0][col], data[row][col]);
                }
        }
    }

#ifdef MEDIATABLE_BASIC
    Array2dVec3UInt8*
    GetImageDataByID(String column, int row)
    {
        std::string imageUrl = GetTextDataByID(column, row);
        Array2dVec3UInt8* ar = 0;
        ar = mRemoteRetriever->RetrieveImageData(imageUrl);
        return ar;
    }

    RemoteRetriever* mRemoteRetriever;

#endif

private:

    void Init()
    {
    }

    ILOG_VAR_DEC;
};

ILOG_VAR_INIT(TableDataSourceCSV, Application.MediaTable);

} // namespace MediaTable
} // namespace Application
} // namespace Impala

#endif // TableDataSourceCSV_h

---