NICE_VisLearning/progs/evaluateCompleteBoWPipeline.cpp

/**
* @file evaluateCompleteBoWPipeline.cpp
* @brief A complete BoW pipeline: feature extraction, codebook creation, vector quantization, classifier training, evaluation on separate test set
* @author Alexander Freytag
* @date 10-05-2013
*/

//STL
#include <iostream>
#include <limits>

//core -- basic stuff
#include <core/basics/Config.h>
#include <core/basics/ResourceStatistics.h>
#include <core/basics/Timer.h>
#include <core/image/Convert.h>
#include <core/vector/VectorT.h>

//vislearning -- basic stuff
#include <vislearning/baselib/Globals.h>
#include <vislearning/baselib/ICETools.h>
#include <vislearning/cbaselib/MultiDataset.h>
#include <vislearning/cbaselib/Example.h>
#include <vislearning/cbaselib/ClassificationResult.h>
#include <vislearning/cbaselib/ClassificationResults.h>
//
// vislearning -- classifier
#include <vislearning/classifier/classifierbase/VecClassifier.h>
#include <vislearning/classifier/genericClassifierSelection.h>
//
// vislearning -- BoW codebooks
#include "vislearning/features/simplefeatures/CodebookPrototypes.h"
#include "vislearning/features/simplefeatures/BoWFeatureConverter.h"
// 
// vislearning -- local features
#include <vislearning/features/localfeatures/GenericLFSelection.h>
//
// vislearning -- clustering methods
#include <vislearning/math/cluster/GenericClusterAlgorithmSelection.h>
//

using namespace std;
using namespace NICE;
using namespace OBJREC;

/**
 a complete BoW pipeline
 
 possibly, we can make use of objrec/progs/testClassifier.cpp
*/
int main( int argc, char **argv )
{
  std::set_terminate( __gnu_cxx::__verbose_terminate_handler );

  Config * conf = new Config ( argc, argv );
  
  const bool writeClassificationResults = conf->gB( "main", "writeClassificationResults", true );
  const std::string resultsfile = conf->gS( "main", "resultsfile", "/tmp/results.txt" );
  
  ResourceStatistics rs;
  
  // ========================================================================
  //                            TRAINING STEP
  // ========================================================================

  MultiDataset md( conf );
  const LabeledSet *trainFiles = md["train"];  
   
  //**********************************************
  //
  //     FEATURE EXTRACTION FOR TRAINING IMAGES
  //
  //**********************************************  
  
  std::cerr << "FEATURE EXTRACTION FOR TRAINING IMAGES" << std::endl;
  
  OBJREC::LocalFeatureRepresentation * featureExtractor = OBJREC::GenericLFSelection::selectLocalFeatureRep ( conf, "features", OBJREC::GenericLFSelection::TRAINING );
  
  //collect features in a single data structure
  NICE::VVector featuresFromAllTrainingImages;  
  featuresFromAllTrainingImages.clear();
  
  //determine how many training images we actually use to easily allocate the correct amount of memory afterwards
  int numberOfTrainingImages ( 0 );
  for(LabeledSet::const_iterator classIt = trainFiles->begin() ; classIt != trainFiles->end() ; classIt++)
  {
    numberOfTrainingImages += classIt->second.size();
    std::cerr << "number of examples for this class: " << classIt->second.size() << std::endl;
  }
  
  
  //okay, this is redundant - but I see no way to do it more easy right now...
  std::vector<NICE::VVector> featuresOfImages ( numberOfTrainingImages );
  //this again is somehow redundant, but we need the labels lateron for easy access - change this to a better solution :)
  NICE::VectorT<int> labelsTrain ( numberOfTrainingImages, 0 );
  
  //TODO replace the nasty makro by a suitable for-loop to make it omp-ready (parallelization)
  int imgCnt ( 0 );
   
  // the corresponding nasty makro: LOOP_ALL_S( *trainFiles )
  for(LabeledSet::const_iterator classIt = trainFiles->begin() ; classIt != trainFiles->end() ; classIt++)
  {
    for ( std::vector<ImageInfo *>::const_iterator imgIt = classIt->second.begin(); 
          imgIt != classIt->second.end(); 
          imgIt++, imgCnt++
        ) 
    {
      // the corresponding nasty makro: EACH_INFO( classno, info );
      int classno ( classIt->first );
      const ImageInfo imgInfo = *(*imgIt);      
      
      std::string filename = imgInfo.img();      
      
      NICE::ColorImage img( filename );
  
      //compute features
      
      //variables to store feature information
      NICE::VVector features;
      NICE::VVector positions;
  
      Globals::setCurrentImgFN ( filename );
      featureExtractor->extractFeatures ( img, features, positions );  
              
      //normalization :)
      for ( NICE::VVector::iterator i = features.begin();
            i != features.end();
            i++)
      {              
        i->normalizeL1();
      }  
            
      //collect them all in a larger data structure
      featuresFromAllTrainingImages.append( features );
      
      //and store it as well in the data struct that additionally keeps the information which features belong to which image
      //TODO this can be made more clever!
//       featuresOfImages.push_back( features );
      featuresOfImages[imgCnt] = features;
      labelsTrain[imgCnt] = classno;
    }
  }
  
  
  
  //**********************************************
  //
  //             CODEBOOK CREATION
  //
  //**********************************************    
  
  std::cerr << "CODEBOOK CREATION" << std::endl;
  
  OBJREC::ClusterAlgorithm * clusterAlgo = OBJREC::GenericClusterAlgorithmSelection::selectClusterAlgo ( conf );
   
  NICE::VVector prototypes;
  
  std::vector<double> weights;
  std::vector<int> assignments;
  
  std::cerr << "call cluster of cluster algo " << std::endl;
  clusterAlgo->cluster( featuresFromAllTrainingImages, prototypes, weights, assignments );
  
  std::cerr << "create new codebook with the computed prototypes" << std::endl;
  OBJREC::CodebookPrototypes * codebook = new OBJREC::CodebookPrototypes ( prototypes );
  
  
  //**********************************************
  //
  //             VECTOR QUANTIZATION OF
  //           FEATURES OF TRAINING IMAGES
  //
  //**********************************************  
  
  OBJREC::BoWFeatureConverter * bowConverter = new OBJREC::BoWFeatureConverter ( conf, codebook );  
  
  OBJREC::LabeledSetVector trainSet;
  
  NICE::VVector histograms ( featuresOfImages.size() /* number of vectors*/, 0 /* dimension of vectors*/ ); //the internal vectors will be resized within calcHistogram
  NICE::VVector::iterator histogramIt = histograms.begin();
  NICE::VectorT<int>::const_iterator labelsIt = labelsTrain.begin();
  
  for (std::vector<NICE::VVector>::const_iterator imgIt = featuresOfImages.begin(); imgIt != featuresOfImages.end(); imgIt++, histogramIt++, labelsIt++)
  {
    bowConverter->calcHistogram ( *imgIt, *histogramIt );
    bowConverter->normalizeHistogram ( *histogramIt );
    
    //NOTE perhaps we should use add_reference here
    trainSet.add( *labelsIt, *histogramIt );
  }
  
  
  //**********************************************
  //
  //             CLASSIFIER TRAINING
  //
  //**********************************************  
  
  std::string classifierType = conf->gS( "main", "classifierType", "GPHIK" );
  OBJREC::VecClassifier * classifier = OBJREC::GenericClassifierSelection::selectVecClassifier( conf, classifierType );
  
  //TODO integrate GP-HIK-NICE into vislearning and add it into genericClassifierSelection
  
  //this method adds the training data to the temporary knowledge of our classifier
  classifier->teach( trainSet );
  //now the actual training step starts (e.g., parameter estimation, ... )
  classifier->finishTeaching();
  
  
  // ========================================================================
  //                            TEST STEP
  // ========================================================================
  
  const LabeledSet *testFiles = md["test"];

  delete featureExtractor;
  featureExtractor = OBJREC::GenericLFSelection::selectLocalFeatureRep ( conf, "features", OBJREC::GenericLFSelection::TESTING );  
  
  NICE::Matrix confusionMat ( trainFiles->size() /* number of classes in training */, testFiles->size() /* number of classes for testing*/, 0.0 );
  NICE::Timer t;
  
  ClassificationResults results;
  
  LOOP_ALL_S( *testFiles )
  {
      EACH_INFO( classno, info );
      std::string filename = info.img();
  
      //**********************************************
      //
      //     FEATURE EXTRACTION FOR TEST IMAGES
      //
      //**********************************************  
      
      NICE::ColorImage img( filename );
  
      //compute features
      
      //variables to store feature information
      NICE::VVector features;
      NICE::VVector positions;
  
      Globals::setCurrentImgFN ( filename );
      featureExtractor->extractFeatures ( img, features, positions );  
        
      //normalization :)
      for ( NICE::VVector::iterator i = features.begin();
            i != features.end();
            i++)
      {              
        i->normalizeL1();
      }        
      
      //**********************************************
      //
      //             VECTOR QUANTIZATION OF
      //           FEATURES OF TEST IMAGES
      //
      //********************************************** 
      
      NICE::Vector histogramOfCurrentImg;
      bowConverter->calcHistogram ( features, histogramOfCurrentImg );
      bowConverter->normalizeHistogram ( histogramOfCurrentImg );
      
      //**********************************************
      //
      //             CLASSIFIER EVALUATION
      //
      //**********************************************   
            
      uint classno_groundtruth = classno;
      
      t.start();
      ClassificationResult r = classifier->classify ( histogramOfCurrentImg );
      t.stop();
      uint classno_estimated = r.classno;
      r.classno_groundtruth = classno_groundtruth;

      //if we like to store the classification results for external post processing, uncomment this
      if ( writeClassificationResults )
      {
        results.push_back( r );
      }      
      
      confusionMat( classno_estimated, classno_groundtruth ) += 1;
  }
  
  confusionMat.normalizeColumnsL1();
  std::cerr << confusionMat << std::endl;

  std::cerr << "average recognition rate: " << confusionMat.trace()/confusionMat.rows() << std::endl;  
  
  if ( writeClassificationResults )
  {
    double avgRecogResults  ( results.getAverageRecognitionRate () );    
    std::cerr << "average recognition rate according to classificationResults: " << avgRecogResults << std::endl;
    results.writeWEKA ( resultsfile, 0 );
  }     
  
 
  
   return 0;
}