NICE_VisLearning/progs/evaluateCompleteBoWPipeline.cpp

/**
* @file evaluateCompleteBoWPipeline.cpp
* @brief A complete BoW pipeline: feature extraction, codebook creation, vector quantization, classifier training, evaluation on separate test set
* @author Alexander Freytag
* @date 10-05-2013
*/

//STL
#include <iostream>
#include <limits>

//core -- basic stuff
#include <core/basics/Config.h>
#include <core/basics/ResourceStatistics.h>
#include <core/basics/Timer.h>
#include <core/image/Convert.h>
#include <core/vector/VectorT.h>

//vislearning -- basic stuff
#include <vislearning/baselib/Globals.h>
#include <vislearning/baselib/ICETools.h>
#include <vislearning/cbaselib/MultiDataset.h>
#include <vislearning/cbaselib/Example.h>
#include <vislearning/cbaselib/ClassificationResult.h>
#include <vislearning/cbaselib/ClassificationResults.h>
//
// vislearning -- classifier
#include <vislearning/classifier/classifierbase/VecClassifier.h>
#include <vislearning/classifier/genericClassifierSelection.h>
//
// vislearning -- BoW codebooks
#include "vislearning/features/simplefeatures/CodebookPrototypes.h"
#include "vislearning/features/simplefeatures/BoWFeatureConverter.h"
// 
// vislearning -- local features
// #include <vislearning/features/localfeatures/LFonHSG.h>
// #include <vislearning/features/localfeatures/LFColorSande.h>
// #include <vislearning/features/localfeatures/LFColorWeijer.h>
// #include <vislearning/features/localfeatures/LFReadCache.h>
// #include <vislearning/features/localfeatures/LFWriteCache.h>
#include <vislearning/features/localfeatures/GenericLFSelection.h>
//
// vislearning -- clustering methods
#include <vislearning/math/cluster/GenericClusterAlgorithmSelection.h>
// #include <vislearning/math/cluster/ClusterAlgorithm.h>
// #include <vislearning/math/cluster/RandomClustering.h>
// #include <vislearning/math/cluster/KMeans.h>
// #include <vislearning/math/cluster/KMedian.h>
// #include <vislearning/math/cluster/GMM.h>
//

using namespace std;
using namespace NICE;
using namespace OBJREC;


// LocalFeatureRepresentation * setFeatureExtractor( const Config * _conf )
// {  
//   LocalFeatureRepresentation * featureExtractor;
//    
//     //feature stuff
//   // which OpponentSIFT implementation to use {NICE, VANDESANDE}
//   std::string opSiftImpl;  
//   opSiftImpl = _conf->gS ( "Descriptor", "implementation", "VANDESANDE" );
//   // read features?
//   bool readfeat;
//   readfeat = _conf->gB ( "Descriptor", "read", true );
//   // write features?
//   bool writefeat;  
//   writefeat = _conf->gB ( "Descriptor", "write", true );   
//   
//   // Welche Opponentsift Implementierung soll genutzt werden ?
//   LocalFeatureRepresentation *cSIFT = NULL;
//   LocalFeatureRepresentation *writeFeats = NULL;
//   LocalFeatureRepresentation *readFeats = NULL;
//   featureExtractor = NULL;
//   if ( opSiftImpl == "NICE" )
//   {
//      cSIFT = new OBJREC::LFonHSG ( _conf, "HSG" );
//   }
//   else if ( opSiftImpl == "VANDESANDE" )
//   {
//     cSIFT = new OBJREC::LFColorSande ( _conf, "LFColorSande" );
//   }
//   else
//   {
//     fthrow ( Exception, "feattype: %s not yet supported" << opSiftImpl );
//   }
// 
//   featureExtractor = cSIFT;
//   
//   if ( writefeat )
//   {
//     // write the features to a file, if there isn't any to read
//     writeFeats = new LFWriteCache ( _conf, cSIFT );
//     featureExtractor = writeFeats;
//   }
// 
//   if ( readfeat )
//   {
//     // read the features from a file
//     if ( writefeat )
//     {
//       readFeats = new LFReadCache ( _conf, writeFeats, -1 );
//     }
//     else
//     {
//       readFeats = new LFReadCache ( _conf, cSIFT, -1 );
//     }
//     featureExtractor = readFeats; 
//   }  
//   
//   //only set feature stuff to NULL, deletion of the underlying object is done in the destructor
//   if ( cSIFT != NULL )
//     cSIFT = NULL;
//   if ( writeFeats != NULL )
//     writeFeats = NULL;
//   if ( readFeats != NULL )
//     readFeats  = NULL ;   
//   
//   return featureExtractor;
// }
// 
// OBJREC::ClusterAlgorithm * setClusterAlgo( const Config * _conf )
// { 
//  std::string section ( "clusteringStuff" );
//   // define the initial number of clusters our codebook shall contain
//   int noClusters = _conf->gI(section, "noClusters", 10);
//   
//   // define the clustering algorithm to be used
//   std::string clusterAlgoString = _conf->gS(section, "clusterAlgo", "kmeans");  
//   
//   OBJREC::ClusterAlgorithm * clusterAlgo;
//   
//   if (clusterAlgoString.compare("kmeans") == 0)
//   {
//     clusterAlgo = new OBJREC::KMeans(noClusters);
//   }
//   else if (clusterAlgoString.compare("kmedian") == 0)
//   {
//     clusterAlgo = new OBJREC::KMedian(noClusters);
//   }  
//   else if (clusterAlgoString.compare("GMM") == 0) 
//   {
//     clusterAlgo = new OBJREC::GMM( _conf, noClusters );
//   }
//   else if ( clusterAlgoString.compare("RandomClustering") == 0 )   
//   {
//     clusterAlgo = new OBJREC::RandomClustering( _conf, section );
//   }
//   else
//   {
//     std::cerr << "Unknown cluster algorithm selected, use random clustering instead" << std::endl;
//     clusterAlgo = new OBJREC::RandomClustering( _conf, section );
//   }    
//   
//   return clusterAlgo;
// }


/**
 a complete BoW pipeline
 
 possibly, we can make use of objrec/progs/testClassifier.cpp
*/
int main( int argc, char **argv )
{
  std::set_terminate( __gnu_cxx::__verbose_terminate_handler );

  Config * conf = new Config ( argc, argv );
  
  const bool writeClassificationResults = conf->gB( "main", "writeClassificationResults", true );
  const std::string resultsfile = conf->gS( "main", "resultsfile", "/tmp/results.txt" );
  
  ResourceStatistics rs;
  
  // ========================================================================
  //                            TRAINING STEP
  // ========================================================================

  MultiDataset md( conf );
  const LabeledSet *trainFiles = md["train"];  
   
  //**********************************************
  //
  //     FEATURE EXTRACTION FOR TRAINING IMAGES
  //
  //**********************************************  
  
  std::cerr << "FEATURE EXTRACTION FOR TRAINING IMAGES" << std::endl;
  
  OBJREC::LocalFeatureRepresentation * featureExtractor = OBJREC::GenericLFSelection::selectLocalFeatureRep ( conf, "features", OBJREC::GenericLFSelection::TRAINING );
//   LocalFeatureRepresentation * featureExtractor = setFeatureExtractor( conf );
  
  //collect features in a single data structure
  NICE::VVector featuresFromAllTrainingImages;  
  featuresFromAllTrainingImages.clear();
  
  //determine how many training images we actually use to easily allocate the correct amount of memory afterwards
  int numberOfTrainingImages ( 0 );
  for(LabeledSet::const_iterator classIt = trainFiles->begin() ; classIt != trainFiles->end() ; classIt++)
  {
    numberOfTrainingImages += classIt->second.size();
    std::cerr << "number of examples for this class: " << classIt->second.size() << std::endl;
  }
  
  
  //okay, this is redundant - but I see no way to do it more easy right now...
  std::vector<NICE::VVector> featuresOfImages ( numberOfTrainingImages );
  //this again is somehow redundant, but we need the labels lateron for easy access - change this to a better solution :)
  NICE::VectorT<int> labelsTrain ( numberOfTrainingImages, 0 );
  
  //TODO replace the nasty makro by a suitable for-loop to make it omp-ready (parallelization)
  int imgCnt ( 0 );
   
  // the corresponding nasty makro: LOOP_ALL_S( *trainFiles )
  for(LabeledSet::const_iterator classIt = trainFiles->begin() ; classIt != trainFiles->end() ; classIt++)
  {
    for ( std::vector<ImageInfo *>::const_iterator imgIt = classIt->second.begin(); 
          imgIt != classIt->second.end(); 
          imgIt++, imgCnt++
        ) 
    {
      // the corresponding nasty makro: EACH_INFO( classno, info );
      int classno ( classIt->first );
      const ImageInfo imgInfo = *(*imgIt);      
      
      std::string filename = imgInfo.img();      
      
      NICE::ColorImage img( filename );
  
      //compute features
      
      //variables to store feature information
      NICE::VVector features;
      NICE::VVector positions;
  
      Globals::setCurrentImgFN ( filename );
      featureExtractor->extractFeatures ( img, features, positions );  
              
      //normalization :)
      for ( NICE::VVector::iterator i = features.begin();
            i != features.end();
            i++)
      {              
        i->normalizeL1();
      }  
            
      //collect them all in a larger data structure
      featuresFromAllTrainingImages.append( features );
      
      //and store it as well in the data struct that additionally keeps the information which features belong to which image
      //TODO this can be made more clever!
//       featuresOfImages.push_back( features );
      featuresOfImages[imgCnt] = features;
      labelsTrain[imgCnt] = classno;
    }
  }
  
  
  
  //**********************************************
  //
  //             CODEBOOK CREATION
  //
  //**********************************************    
  
  std::cerr << "CODEBOOK CREATION" << std::endl;
//   OBJREC::ClusterAlgorithm * clusterAlgo = setClusterAlgo( conf );
  
  OBJREC::ClusterAlgorithm * clusterAlgo = OBJREC::GenericClusterAlgorithmSelection::selectClusterAlgo ( conf );
   
  NICE::VVector prototypes;
  
  std::vector<double> weights;
  std::vector<int> assignments;
  
  std::cerr << "call cluster of cluster algo " << std::endl;
  clusterAlgo->cluster( featuresFromAllTrainingImages, prototypes, weights, assignments );
  
  std::cerr << "create new codebook with the computed prototypes" << std::endl;
  OBJREC::CodebookPrototypes * codebook = new OBJREC::CodebookPrototypes ( prototypes );
  
  
  //**********************************************
  //
  //             VECTOR QUANTIZATION OF
  //           FEATURES OF TRAINING IMAGES
  //
  //**********************************************  
  
  OBJREC::BoWFeatureConverter * bowConverter = new OBJREC::BoWFeatureConverter ( conf, codebook );  
  
  OBJREC::LabeledSetVector trainSet;
  
  NICE::VVector histograms ( featuresOfImages.size() /* number of vectors*/, 0 /* dimension of vectors*/ ); //the internal vectors will be resized within calcHistogram
  NICE::VVector::iterator histogramIt = histograms.begin();
  NICE::VectorT<int>::const_iterator labelsIt = labelsTrain.begin();
  
  for (std::vector<NICE::VVector>::const_iterator imgIt = featuresOfImages.begin(); imgIt != featuresOfImages.end(); imgIt++, histogramIt++, labelsIt++)
  {
    bowConverter->calcHistogram ( *imgIt, *histogramIt );
    bowConverter->normalizeHistogram ( *histogramIt );
    
    //NOTE perhaps we should use add_reference here
    trainSet.add( *labelsIt, *histogramIt );
  }
  
  
  //**********************************************
  //
  //             CLASSIFIER TRAINING
  //
  //**********************************************  
  
  std::string classifierType = conf->gS( "main", "classifierType", "GPHIK" );
  OBJREC::VecClassifier * classifier = OBJREC::GenericClassifierSelection::selectVecClassifier( conf, classifierType );
  
  //TODO integrate GP-HIK-NICE into vislearning and add it into genericClassifierSelection
  
  //this method adds the training data to the temporary knowledge of our classifier
  classifier->teach( trainSet );
  //now the actual training step starts (e.g., parameter estimation, ... )
  classifier->finishTeaching();
  
  
  // ========================================================================
  //                            TEST STEP
  // ========================================================================
  
  const LabeledSet *testFiles = md["test"];

  delete featureExtractor;
  featureExtractor = OBJREC::GenericLFSelection::selectLocalFeatureRep ( conf, "features", OBJREC::GenericLFSelection::TESTING );  
  
  NICE::Matrix confusionMat ( trainFiles->size() /* number of classes in training */, testFiles->size() /* number of classes for testing*/, 0.0 );
  NICE::Timer t;
  
  ClassificationResults results;
  
  LOOP_ALL_S( *testFiles )
  {
      EACH_INFO( classno, info );
      std::string filename = info.img();
  
      //**********************************************
      //
      //     FEATURE EXTRACTION FOR TEST IMAGES
      //
      //**********************************************  
      
      NICE::ColorImage img( filename );
  
      //compute features
      
      //variables to store feature information
      NICE::VVector features;
      NICE::VVector positions;
  
      Globals::setCurrentImgFN ( filename );
      featureExtractor->extractFeatures ( img, features, positions );  
        
      //normalization :)
      for ( NICE::VVector::iterator i = features.begin();
            i != features.end();
            i++)
      {              
        i->normalizeL1();
      }        
      
      //**********************************************
      //
      //             VECTOR QUANTIZATION OF
      //           FEATURES OF TEST IMAGES
      //
      //********************************************** 
      
      NICE::Vector histogramOfCurrentImg;
      bowConverter->calcHistogram ( features, histogramOfCurrentImg );
      bowConverter->normalizeHistogram ( histogramOfCurrentImg );
      
      //**********************************************
      //
      //             CLASSIFIER EVALUATION
      //
      //**********************************************   
            
      uint classno_groundtruth = classno;
      
      t.start();
      ClassificationResult r = classifier->classify ( histogramOfCurrentImg );
      t.stop();
      uint classno_estimated = r.classno;
      r.classno_groundtruth = classno_groundtruth;

      //if we like to store the classification results for external post processing, uncomment this
      if ( writeClassificationResults )
      {
        results.push_back( r );
      }      
      
      confusionMat( classno_estimated, classno_groundtruth ) += 1;
  }
  
  confusionMat.normalizeColumnsL1();
  std::cerr << confusionMat << std::endl;

  std::cerr << "average recognition rate: " << confusionMat.trace()/confusionMat.rows() << std::endl;  
  
  if ( writeClassificationResults )
  {
    double avgRecogResults  ( results.getAverageRecognitionRate () );    
    std::cerr << "average recognition rate according to classificationResults: " << avgRecogResults << std::endl;
    results.writeWEKA ( resultsfile, 0 );
  }     
  
 
  
   return 0;
}