NICE_VisLearning/featureLearning/progs/testFeatureLearning.cpp

/**
* @file testFeatureLearning.cpp
* @brief test the feature learning routines to incrementally increase / adapt the codebook currently used
* @author Alexander Freytag
* @date 11-04-2013
*/

//STL
#include <iostream>
#include <limits>

//core
#include <core/basics/Config.h>
#include <core/basics/ResourceStatistics.h>
#include <core/image/Convert.h>
#include <core/vector/VectorT.h>

//vislearning
#include <vislearning/baselib/Globals.h>
#include <vislearning/baselib/ICETools.h>
#include <vislearning/cbaselib/MultiDataset.h>
#include <vislearning/cbaselib/Example.h>
// 
#include "vislearning/featureLearning/FeatureLearningGeneric.h"
#include "vislearning/featureLearning/FeatureLearningClusterBased.h"
#include "vislearning/featureLearning/FeatureLearningRegionBased.h"
//
#include "vislearning/noveltyDetection/NDCodebookLevelImagePooling.h"


using namespace std;
using namespace NICE;
using namespace OBJREC;


/**
 test feature learning routines
*/
int main( int argc, char **argv )
{
#ifndef __clang__
#ifndef __llvm__
  std::set_terminate( __gnu_cxx::__verbose_terminate_handler );
#endif
#endif

  Config * conf = new Config ( argc, argv );
  
  bool showTrainingImages = conf->gB( "featureLearning", "showTrainingImages", false );
  bool showTestImages = conf->gB( "featureLearning", "showTestImages", false );
  bool showResults = conf->gB( "featureLearning", "showResults", false );
  
  ResourceStatistics rs;
  std::string resultdir;
  resultdir = conf->gS( "featureLearning", "resultdir", "/tmp/");
  
  
  NICE::ImageT<int> noveltyScale ( 20, 100 );
  for (int j = 0; j < 100; j++)
  {
    for (int i = 0; i < 20; i++)
      noveltyScale(i,j) = 99-j;
  }
  
  NICE::ColorImage noveltyScaleRGB (noveltyScale.width(), noveltyScale.height() );
  imageToPseudoColor( noveltyScale, noveltyScaleRGB );
  
  std::string destinationNoveltyScale ( resultdir + "_" + "_noveltyScale.ppm");
  noveltyScaleRGB.writePPM( destinationNoveltyScale );   
  
  
  
  //**********************************************
  //
  //      READ INITIAL TRAINING SET TO COMPUTE
  //             AN INITIAL CODEBOOK
  //
  //**********************************************
  
  std::cerr << " READ INITIAL TRAINING SET TO COMPUTE AN INITIAL CODEBOOK" << std::endl;
  
  MultiDataset md( conf );
  const LabeledSet *trainFiles = md["train"];
  
  //**********************************************
  //
  //      SET UP THE FEATURE LEARNING ALGO
  //
  //********************************************** 
  
  OBJREC::FeatureLearningGeneric * featureLearning;
  
  std::string featureLearningMethod = conf->gS( "featureLearning", "featureLearningMethod", "clusterBased" );
  
  if (featureLearningMethod.compare("clusterBased") == 0)
  {
    featureLearning = new OBJREC::FeatureLearningClusterBased( conf, &md );
  }
  else if (featureLearningMethod.compare("regionBased") == 0) 
  {
    featureLearning = new OBJREC::FeatureLearningRegionBased( conf, &md );
  }
  else
  {
    std::cerr << "Unknown feature learning algorithm selected, use cluster based instead" << std::endl;
    featureLearning = new OBJREC::FeatureLearningClusterBased( conf, &md );
  }    
  
  //**********************************************
  //
  //      SET UP THE NOVELTY DECTION ALGO
  //
  //**********************************************   
  
  OBJREC::NDCodebookLevelImagePooling * novDetector;
  novDetector = new OBJREC::NDCodebookLevelImagePooling( conf, &md, "featureLearning" );
    
  //evaluate how well the training images are covered with our initial codebook
  //that is, compute these nice "novelty maps" per feature
  
  //NOTE we skip this currently
  LOOP_ALL_S( *trainFiles )
  {
      EACH_INFO( classno, info );
      std::string filename = info.img();
//       
//       featureLearning->evaluateCurrentCodebook( filename , true /* beforeComputingNewFeatures */);       
      
      NICE::ImageT< int > imgClusterAssignments;
      imgClusterAssignments = featureLearning->evaluateCurrentCodebookByAssignments(filename , false /* beforeComputingNewFeatures */, false /* _binaryShowLatestPrototype*/ );
      
      std::cerr << "now do image To pseude color and show the initial cluster assignments" << std::endl;
      NICE::ColorImage imgClusterAssignmentsRGB (imgClusterAssignments.width(), imgClusterAssignments.height() );
      imageToPseudoColor( imgClusterAssignments, imgClusterAssignmentsRGB );      
      
      if ( showResults )
        showImage(imgClusterAssignmentsRGB, "cluster Assignments" ) ;
      else
      {
        std::vector< std::string > list2;
        StringTools::split ( filename, '/', list2 );        
        std::string destination ( resultdir + NICE::intToString(0) + "_" + list2.back() + "_00_initialClusterAssignments.ppm");
        imgClusterAssignmentsRGB.writePPM( destination );        
      }
  }
  
  //**********************************************
  //
  //        EVALUATE INITIAL CLUSTER
  //
  //       COMPUTE A NICE CONFUSION MATRIX
  //          FOR OUR INITIAL CODEBOOK
  //
  //**********************************************  
  NICE::Matrix confusionMatInitial;
  featureLearning->evaluateCurrentCodebookByConfusionMatrix( confusionMatInitial );
  std::cerr << "initial Confusion matrix: " << std::endl << confusionMatInitial << std::endl;
  
  //set the initially computed codebook to the novelty detection mechanism
  //TODO this should be done, but currently we do not care about
//   novDetector->setCodebook( featureLearning->getCurrentCodebook() );
  

  //**********************************************
  //
  //        FOR-LOOP OVER UNSEEN IMAGES
  //
  //       EXTRACT FEATURES, CLUSTER THEM, TAKE
  //       MOST "VALUABLE" CLUSTERS AS NEW 
  //    REPRESENTATIVES IN AN INCREASED CODEBOK
  //
  //**********************************************
  const LabeledSet *testFiles = md["test"];
    
  std::cerr << "start looping over all files" << std::endl;
  int imageCnt ( 0 );
  LOOP_ALL_S( *testFiles )
  {
      EACH_INFO( classno, info );
      std::string filename = info.img();
      
      NICE::ColorImage orig( filename );
      
      NICE::ImageT< int > imgClusterAssignments;
      imgClusterAssignments = featureLearning->evaluateCurrentCodebookByAssignments(filename , false /* beforeComputingNewFeatures */, false /* _binaryShowLatestPrototype*/ );      
      
      NICE::ColorImage imgClusterAssignmentsRGB (imgClusterAssignments.width(), imgClusterAssignments.height() );
      imageToPseudoColor( imgClusterAssignments, imgClusterAssignmentsRGB );       
      
      NICE::FloatImage noveltyImageBefore;
      noveltyImageBefore = featureLearning->evaluateCurrentCodebookByDistance( filename , true /* beforeComputingNewFeatures */ );
      
      NICE::ColorImage noveltyImageBeforeRGB (noveltyImageBefore.width(), noveltyImageBefore.height() );
      imageToPseudoColor( noveltyImageBefore, noveltyImageBeforeRGB );        
      
      if ( showResults )
        showImage(imgClusterAssignmentsRGB, "cluster Assignments" ) ;
      else
      {
        std::vector< std::string > list2;
        StringTools::split ( filename, '/', list2 );        
        std::string destination ( resultdir + NICE::intToString(0) + "_" + list2.back() + "_00_initialClusterAssignments.ppm");
        imgClusterAssignmentsRGB.writePPM( destination );        
        
        std::string destinationNovelty ( resultdir + NICE::intToString(0) + "_" + list2.back() + "_01_initialClusterDistances.ppm");
        noveltyImageBeforeRGB.writePPM( destinationNovelty );          
      }      
      
      //**********************************************
      //
      //             IS THIS IMAGE NOVEL?
      //
      //          IF NOT, GO TO THE NEXT ONE
      //
      //**********************************************       
      
      bool b_isImageNovel ( novDetector->evaluateNoveltyOfImage( noveltyImageBefore ) );
      
      if ( ! b_isImageNovel )
      {
        std::cerr << " --- NOT NOVEL --- " << std::endl << std::endl;
        continue;
      }
      else
      {
        std::cerr << " --- NOVEL --- " << std::endl;
      }
      
      while ( b_isImageNovel ) 
      {
              
        //**********************************************
        //
        //       LEARN NEW FEATURE FOR A NOVEL IMAGE
        //
        //**********************************************       
        
        featureLearning->learnNewFeatures( filename );
        
        //and update the codebook pointer within our novelty detection algorithm
        //TODO this should be done, but currently we do not care about
  //       novDetector->setCodebook( featureLearning->getCurrentCodebook() );      
        
        //**********************************************
        //
        //       EVALUATE HOW WELL THE CURRENT IMAGE
        //            CAN BE EXPLAINED AFTER 
        //           COMPUTING A NEW FEATURE
        //
        //       SHOULD WE REPEAT THIS UNTIL THE IMAGE
        //       IS NOT CLASSIFIED AS "NOVEL" ANYMORE?
        //
        //**********************************************       
        
        NICE::FloatImage noveltyImageAfter;
        noveltyImageAfter = featureLearning->evaluateCurrentCodebookByDistance( filename , false /* beforeComputingNewFeatures */ );
        
        NICE::FloatImage noveltyImageDifference ( noveltyImageAfter.width(), noveltyImageAfter.height());
        for ( uint y = 0 ; y < ( uint ) noveltyImageAfter.height() ; y++ )
        {
          for ( uint x = 0 ; x < ( uint ) noveltyImageAfter.width(); x++ )
          {
            noveltyImageDifference(x,y) = fabs ( noveltyImageBefore(x,y) - noveltyImageAfter(x,y) );
          }
        }
        
        std::cerr << "min diff: " << noveltyImageDifference.min() << " and max diff: " << noveltyImageDifference.max() << std::endl;
        
        NICE::ColorImage noveltyImageDifferenceRGB (noveltyImageAfter.width(), noveltyImageAfter.height() );
        imageToPseudoColor( noveltyImageDifference, noveltyImageDifferenceRGB );      
        
        std::vector< std::string > list2;
        StringTools::split ( filename, '/', list2 );      

        if ( showResults )
        {
          showImage( noveltyImageDifferenceRGB, "Difference of novelty images" );
        }
        else
        {          
          std::string destination ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_02_initialLoopClusterAssignments.ppm");  
          imgClusterAssignmentsRGB.writePPM( destination );    

          std::string destinationNoveltyDifference ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_03_noveltyDifferences.ppm");  
          noveltyImageDifferenceRGB.writePPM( destinationNoveltyDifference );                        
        }

        NICE::ImageT< int > imgClusterAssignmentsBinary;
        imgClusterAssignmentsBinary = featureLearning->evaluateCurrentCodebookByAssignments(filename , true, true /* _binaryShowLatestPrototype*/ );      
        
        NICE::ImageT< int > imgClusterAssignments;
        imgClusterAssignments = featureLearning->evaluateCurrentCodebookByAssignments(filename , true, false /* _binaryShowLatestPrototype*/ );

        
        NICE::ColorImage imgClusterAssignmentsBinaryRGB (imgClusterAssignmentsBinary.width(), imgClusterAssignmentsBinary.height() );
        imageToPseudoColor( imgClusterAssignmentsBinary, imgClusterAssignmentsBinaryRGB );   
        
        NICE::ColorImage imgClusterAssignmentsRGB (imgClusterAssignments.width(), imgClusterAssignments.height() );
        imageToPseudoColor( imgClusterAssignments, imgClusterAssignmentsRGB );           
        
        if ( showResults )
        {
          showImage(imgClusterAssignmentsBinaryRGB, "cluster Assignments Binary (latest prototype)" ) ;      
          showImage(imgClusterAssignmentsRGB, "cluster Assignments" ) ;
        }
        else
        {
            std::string destination ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_5_clusterAssignments.ppm");
            std::string destinationBinary ( resultdir + NICE::intToString(imageCnt) + "_" + list2.back() + "_6_clusterAssignmentsBinary.ppm");
            
            imgClusterAssignmentsRGB.writePPM( destination );
            imgClusterAssignmentsBinaryRGB.writePPM( destinationBinary );
        }      
        
        //check, whether the image will still be seen as novel or not
        // i.e., are we able to "explain" the image with the lately computed codebook entry?
        b_isImageNovel = novDetector->evaluateNoveltyOfImage( noveltyImageAfter ) ;
        
        if ( ! b_isImageNovel )
        {
          std::cerr << " --- NOT NOVEL ANYMORE --- " << std::endl << std::endl;
        }
        else
        {
          std::cerr << " --- STILL NOVEL --- " << std::endl;
          noveltyImageBefore = noveltyImageAfter;
        }        
        
        //artifial break for the moment
        break;
      }

      
      imageCnt++;
  } //Loop over all test images
  
  
  //don't waste memory
  std::cerr << "don't waste memory - cleaning up" << std::endl;
//   if (trainFiles != NULL)
//     delete trainFiles;
  if (featureLearning != NULL)
    delete featureLearning;
  if (novDetector != NULL)
    delete novDetector;
  if (conf != NULL)
    delete conf;
  
   return 0;
}