NICE_VisLearning/featureLearning/progs/testFeatureLearning.cpp

/**
* @file testFeatureLearning.cpp
* @brief test the feature learning routines to incrementally increase / adapt the codebook currently used
* @author Alexander Freytag
* @date 11-04-2013
*/

//STL
#include <iostream>
#include <limits>

//core
#include <core/basics/Config.h>
#include <core/basics/ResourceStatistics.h>
#include <core/image/Convert.h>
#include <core/vector/VectorT.h>

//vislearning
#include <vislearning/baselib/Globals.h>
#include <vislearning/baselib/ICETools.h>
#include <vislearning/cbaselib/MultiDataset.h>
#include <vislearning/cbaselib/Example.h>
// 
#include "vislearning/featureLearning/FeatureLearningGeneric.h"
#include "vislearning/featureLearning/FeatureLearningClusterBased.h"
#include "vislearning/featureLearning/FeatureLearningRegionBased.h"
//
#include "vislearning/noveltyDetection/NDCodebookLevelImagePooling.h"


using namespace std;
using namespace NICE;
using namespace OBJREC;


/**
 test feature learning routines
*/
int main( int argc, char **argv )
{
  std::set_terminate( __gnu_cxx::__verbose_terminate_handler );

  Config * conf = new Config ( argc, argv );
  
  bool showTrainingImages= conf->gB( "featureLearning", "showTrainingImages", false );
  bool showTestImages= conf->gB( "featureLearning", "showTestImages", false );
  
  ResourceStatistics rs;
  std::string resultdir;
  resultdir = conf->gS( "featureLearning", "resultdir", "/tmp/");
  
  
  //**********************************************
  //
  //      READ INITIAL TRAINING SET TO COMPUTE
  //             AN INITIAL CODEBOOK
  //
  //**********************************************
  
  std::cerr << " READ INITIAL TRAINING SET TO COMPUTE AN INITIAL CODEBOOK" << std::endl;
  
  MultiDataset md( conf );
  const LabeledSet *trainFiles = md["train"];
  
  //**********************************************
  //
  //      SET UP THE FEATURE LEARNING ALGO
  //
  //********************************************** 
  
  OBJREC::FeatureLearningGeneric * featureLearning;
  
  std::string featureLearningMethod = conf->gS( "featureLearning", "featureLearningMethod", "clusterBased" );
  
  if (featureLearningMethod.compare("clusterBased") == 0)
  {
    featureLearning = new OBJREC::FeatureLearningClusterBased( conf, &md );
  }
  else if (featureLearningMethod.compare("regionBased") == 0) 
  {
    featureLearning = new OBJREC::FeatureLearningRegionBased( conf, &md );
  }
  else
  {
    std::cerr << "Unknown feature learning algorithm selected, use cluster based instead" << std::endl;
    featureLearning = new OBJREC::FeatureLearningClusterBased( conf, &md );
  }    
  
  //**********************************************
  //
  //      SET UP THE NOVELTY DECTION ALGO
  //
  //**********************************************   
  
  OBJREC::NDCodebookLevelImagePooling * novDetector;
  novDetector = new OBJREC::NDCodebookLevelImagePooling( conf, &md, "featureLearning" );
    
  //evaluate how well the training images are covered with our initial codebook
  //that is, compute these nice "novelty maps" per feature
  
  //NOTE we skip this currently
  LOOP_ALL_S( *trainFiles )
  {
      EACH_INFO( classno, info );
      std::string filename = info.img();
//       
//       featureLearning->evaluateCurrentCodebook( filename , true /* beforeComputingNewFeatures */);       
      
      NICE::ImageT< int > imgClusterAssignments;
      imgClusterAssignments = featureLearning->evaluateCurrentCodebookByAssignments(filename , false /* beforeComputingNewFeatures */, false /* _binaryShowLatestPrototype*/ );
      
      std::cerr << "now do image To pseude color and show the initial cluster assignments" << std::endl;
      NICE::ColorImage imgClusterAssignmentsRGB (imgClusterAssignments.width(), imgClusterAssignments.height() );
      imageToPseudoColor( imgClusterAssignments, imgClusterAssignmentsRGB );      
      showImage(imgClusterAssignmentsRGB, "cluster Assignments" ) ;
  }
  
  //**********************************************
  //
  //        EVALUATE INITIAL CLUSTER
  //
  //       COMPUTE A NICE CONFUSION MATRIX
  //          FOR OUR INITIAL CODEBOOK
  //
  //**********************************************  
  NICE::Matrix confusionMatInitial;
  featureLearning->evaluateCurrentCodebookByConfusionMatrix( confusionMatInitial );
  std::cerr << "initial Confusion matrix: " << std::endl << confusionMatInitial << std::endl;
  
  //set the initially computed codebook to the novelty detection mechanism
  //TODO this should be done, but currently we do not care about
//   novDetector->setCodebook( featureLearning->getCurrentCodebook() );
  

  //**********************************************
  //
  //        FOR-LOOP OVER UNSEEN IMAGES
  //
  //       EXTRACT FEATURES, CLUSTER THEM, TAKE
  //       MOST "VALUABLE" CLUSTERS AS NEW 
  //    REPRESENTATIVES IN AN INCREASED CODEBOK
  //
  //**********************************************
  const LabeledSet *testFiles = md["test"];
    
  std::cerr << "start looping over all files" << std::endl;
  int imageCnt ( 0 );
  LOOP_ALL_S( *testFiles )
  {
      EACH_INFO( classno, info );
      std::string filename = info.img();
      
      NICE::ColorImage orig( filename );
      
      NICE::FloatImage noveltyImageBefore;
      noveltyImageBefore = featureLearning->evaluateCurrentCodebookByDistance( filename , true /* beforeComputingNewFeatures */ );
      
      //**********************************************
      //
      //             IS THIS IMAGE NOVEL?
      //
      //          IF NOT, GO TO THE NEXT ONE
      //
      //**********************************************       
      
      bool b_isImageNovel ( novDetector->evaluateNoveltyOfImage( noveltyImageBefore ) );
      
      if ( ! b_isImageNovel )
      {
        std::cerr << " --- NOT NOVEL --- " << std::endl << std::endl;
        continue;
      }
      else
      {
        std::cerr << " --- NOVEL --- " << std::endl;
      }
      
      //**********************************************
      //
      //       LEARN NEW FEATURE FOR A NOVEL IMAGE
      //
      //**********************************************       
      
      featureLearning->learnNewFeatures( filename );
      
      //and update the codebook pointer within our novelty detection algorithm
      //TODO this should be done, but currently we do not care about
//       novDetector->setCodebook( featureLearning->getCurrentCodebook() );      
      
      //**********************************************
      //
      //       EVALUATE HOW WELL THE CURRENT IMAGE
      //            CAN BE EXPLAINED AFTER 
      //           COMPUTING A NEW FEATURE
      //
      //       SHOULD WE REPEAT THIS UNTIL THE IMAGE
      //       IS NOT CLASSIFIED AS "NOVEL" ANYMORE?
      //
      //**********************************************       
      
      NICE::FloatImage noveltyImageAfter;
      noveltyImageAfter = featureLearning->evaluateCurrentCodebookByDistance( filename , false /* beforeComputingNewFeatures */ );
      
      NICE::FloatImage noveltyImageDifference ( noveltyImageAfter.width(), noveltyImageAfter.height());
      for ( uint y = 0 ; y < ( uint ) noveltyImageAfter.height() ; y++ )
      {
        for ( uint x = 0 ; x < ( uint ) noveltyImageAfter.width(); x++ )
        {
          noveltyImageDifference(x,y) = fabs ( noveltyImageBefore(x,y) - noveltyImageAfter(x,y) );
        }
      }
      
      std::cerr << "min diff: " << noveltyImageDifference.min() << " and max diff: " << noveltyImageDifference.max() << std::endl;
      
      NICE::ColorImage noveltyImageDifferenceRGB (noveltyImageAfter.width(), noveltyImageAfter.height() );
      imageToPseudoColor( noveltyImageDifference, noveltyImageDifferenceRGB );      
      
      std::vector< std::string > list2;
      StringTools::split ( filename, '/', list2 );      

      std::string destination ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_4_differenceOfNovelties.ppm");
      noveltyImageDifferenceRGB.writePPM( destination );      

      NICE::ImageT< int > imgClusterAssignmentsBinary;
      imgClusterAssignmentsBinary = featureLearning->evaluateCurrentCodebookByAssignments(filename , true, true /* _binaryShowLatestPrototype*/ );      
      
      NICE::ImageT< int > imgClusterAssignments;
      imgClusterAssignments = featureLearning->evaluateCurrentCodebookByAssignments(filename , true, false /* _binaryShowLatestPrototype*/ );

      
      NICE::ColorImage imgClusterAssignmentsBinaryRGB (imgClusterAssignmentsBinary.width(), imgClusterAssignmentsBinary.height() );
      imageToPseudoColor( imgClusterAssignmentsBinary, imgClusterAssignmentsBinaryRGB );   
      
      NICE::ColorImage imgClusterAssignmentsRGB (imgClusterAssignments.width(), imgClusterAssignments.height() );
      imageToPseudoColor( imgClusterAssignments, imgClusterAssignmentsRGB );           
      
      if ( false )
      {
        showImage(imgClusterAssignmentsBinaryRGB, "cluster Assignments Binary (latest prototype)" ) ;      
        showImage(imgClusterAssignmentsRGB, "cluster Assignments" ) ;
      }
      else
      {
          std::string destination ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_5_clusterAssignments.ppm");
          std::string destinationBinary ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_6_clusterAssignmentsBinary.ppm");
          
          imgClusterAssignmentsRGB.writePPM( destination );
          imgClusterAssignmentsBinaryRGB.writePPM( destinationBinary );
      }      
      
      
    

      
      imageCnt++;
  } //Loop over all test images
  
  
  //don't waste memory
  std::cerr << "don't waste memory - cleaning up" << std::endl;
//   if (trainFiles != NULL)
//     delete trainFiles;
  if (featureLearning != NULL)
    delete featureLearning;
  if (novDetector != NULL)
    delete novDetector;
  if (conf != NULL)
    delete conf;
  
   return 0;
}