NICE_VisLearning/featureLearning/FeatureLearningRegionBased.cpp

#include "FeatureLearningRegionBased.h"

//STL
#include <iostream>

//core
#include <core/image/FilterT.h>
#include <core/image/CircleT.h>
#include <core/image/Convert.h>
#include <core/vector/VectorT.h>

#include <segmentation/GenericRegionSegmentationMethodSelection.h>

//vislearning
#include <vislearning/baselib/Globals.h>


using namespace std;
using namespace NICE;
using namespace OBJREC;

  //**********************************************
  //
  //                 PROTECTED METHODS
  //
  //********************************************** 


  //**********************************************
  //
  //                 PUBLIC METHODS
  //
  //********************************************** 


FeatureLearningRegionBased::FeatureLearningRegionBased ( const Config *_conf,
                               const MultiDataset *_md, const std::string & _section )
    : FeatureLearningPrototypes ( _conf, _md, _section )
{ 

   //save and read segmentation results from files  
  this->reuseSegmentation = conf->gB ( "FPCPixel", "reuseSegmentation", true );  
  
  // select your segmentation method here
  // currently, the following options are supported: "MarkovCluster", "GraphBased", "MeanShift", "SLIC"
  string rsMethode = conf->gS ( section, "segmentationMethod", "MeanShift" );
 
  OBJREC::RegionSegmentationMethod *tmpRegionSeg = OBJREC::GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(conf, rsMethode);    
  
  if ( reuseSegmentation )
    this->segmentationAlgo = new RSCache ( conf, tmpRegionSeg );
  else
    this->segmentationAlgo = tmpRegionSeg;
  
  this->i_gridSize = conf->gI( "LFColorSandeTest" , "grid" , 5 );
}

FeatureLearningRegionBased::~FeatureLearningRegionBased()
{
  // clean-up
}

void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filename )
{  
  NICE::ColorImage img( _filename );
  
  int xsize ( img.width() );
  int ysize ( img.height() );
   
  //variables to store feature information
  NICE::VVector newFeatures;
  NICE::VVector cfeatures;
  NICE::VVector positions;

  //compute features
  std::cerr << " EXTRACT FEATURES FROM UNSEEN IMAGE" << std::endl;
  Globals::setCurrentImgFN ( _filename );
  featureExtractor->extractFeatures ( img, newFeatures, positions );  
    
  //normalization :)
  for ( NICE::VVector::iterator i = newFeatures.begin();
        i != newFeatures.end();
        i++)
  {              
    i->normalizeL1();
  }
  
  //compute region segmentation
  std::cerr << " COMPUTE REGION SEGMENTATION" << std::endl;
  NICE::Matrix mask;
  int amountRegions = segmentationAlgo->segRegions ( img, mask );
  
  //compute novelty scores on a feature level
  std::cerr << " COMPUTE NOVELTY SCORES ON A FEATURE LEVEL" << std::endl;
  FloatImage noveltyImageGaussFiltered ( xsize, ysize );    
  this->evaluateCurrentCodebookForGivenFeatures( newFeatures, positions, noveltyImageGaussFiltered );  
  
  // compute scores for every region
  std::cerr << " COMPUTE SCORES FOR EVERY REGION" << std::endl;
  std::vector<double> regionNoveltyMeasure (amountRegions, 0.0); 
  std::vector<int> regionSize (amountRegions, 0);
  
  for ( int y = 0; y < ysize; y += i_gridSize) //y++)
  {
    for (int x = 0; x < xsize; x += i_gridSize) //x++)
    {
      int r = mask(x,y);
      regionSize[r]++;

      //count the amount of "novelty" for the corresponding region
      regionNoveltyMeasure[r] += noveltyImageGaussFiltered(x,y);
    }
  }
  
  //loop over all regions and compute averaged novelty scores
  //NOTE this might be unuseful, since lateron we combine novelty score and region size (e.g. by multiplying)
  // however, we do not want to settle the combination in adavance
  for(int r = 0; r < amountRegions; r++)
  {      
    regionNoveltyMeasure[r] /= regionSize[r];  
  }

   //a new region should cover at least 3% of the image for not being penalized
  double d_minimalImageAmountForAcceptableRegions ( 0.03 );
  int minimalImageAmountForAcceptableRegions ( round( d_minimalImageAmountForAcceptableRegions * (xsize/i_gridSize) * (ysize/i_gridSize) )  );
  
  std::cerr << "minimalImageAmountForAcceptableRegions: " << minimalImageAmountForAcceptableRegions << std::endl;
  
  //compute final scores for all regions
  NICE::Vector regionScores ( amountRegions, 0.0 );  
  std::cerr << "used region sizes for computation: " << std::endl;
  for(int r = 0; r < amountRegions; r++)
  {      
    regionScores[r] = std::min( regionSize[r], minimalImageAmountForAcceptableRegions ) * regionNoveltyMeasure[r];
    std::cerr << " " << std::min( regionSize[r], minimalImageAmountForAcceptableRegions );
  }
  std::cerr << std::endl << std::endl;
  
  int indexOfBestRegion ( regionScores.MaxIndex() );
  
  
  ////////////////////////////////////
  //
  //       VISUALIZE REGION SCORES
  //
  ////////////////////////////////////
  
  NICE::FloatImage regionScoreImage ( xsize, ysize );
  NICE::FloatImage regionNoveltyImage ( xsize, ysize ); //contains novelty score averaged over region
  NICE::FloatImage regionRelevanceImage ( xsize, ysize ); //contains higher scores for larger regions (but with upper limit)
  
  regionScoreImage.set( 0.0 );
  regionNoveltyImage.set( 0.0 );
  regionRelevanceImage.set( 0.0 );
  
    for ( int y = 0; y < ysize; y++)
    {
      for (int x = 0; x < xsize; x++)
      {
        int r = mask(x,y);

        regionNoveltyImage(x,y) = regionNoveltyMeasure[r];
        regionRelevanceImage(x,y) = regionSize[r];
        regionScoreImage(x,y) = regionScores[r];
      }
    }  
    
    std::cerr << "highest region score: " << regionScoreImage.max()<< " -- smallest region score: " << regionScoreImage.min() << std::endl;
    std::cerr << "highest region novelty score: " << regionNoveltyImage.max() << "  -- smallest region novelty score: " << regionNoveltyImage.min() << std::endl;
    
  NICE::ColorImage regionScoreImageRGB  ( xsize, ysize );
  NICE::ColorImage regionNoveltyScoreImageRGB  ( xsize, ysize );
  NICE::ColorImage regionRelevanceScoreImageRGB  ( xsize, ysize );
  //TODO properly specify the maximum value for score visualization here :)
  imageToPseudoColorWithRangeSpecification( regionScoreImage, regionScoreImageRGB, 0 /* min */, 2.2 /* max */ );  
  imageToPseudoColorWithRangeSpecification( regionNoveltyImage, regionNoveltyScoreImageRGB, 0 /* min */, 0.012 /* max */ );  
  imageToPseudoColorWithRangeSpecification( regionRelevanceImage, regionRelevanceScoreImageRGB, 0 /* min */, minimalImageAmountForAcceptableRegions /* max */ );  

    if ( b_showResults )
    {
      showImage(regionNoveltyScoreImageRGB, "Current (new) image with region NOVELTY scores"); 
      showImage(regionRelevanceScoreImageRGB, "Current (new) image with region RELEVANCE scores"); 
      showImage(regionScoreImageRGB, "Current (new) image with FINAL region scores"); 
    }
    else 
    {
      std::vector< std::string > list2;
      StringTools::split ( _filename, '/', list2 );      
     
      //write novelty score image
      std::string destinationNovelty ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_1_regionNoveltyScores.ppm");
      regionNoveltyScoreImageRGB.writePPM( destinationNovelty );
      
      //write relevance score image
      std::string destinationRelevance ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_2_regionRelevanceScores.ppm");
      regionRelevanceScoreImageRGB.writePPM( destinationRelevance );      

      //write image with final scores for region
      std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_3_regionScores.ppm");
      regionScoreImageRGB.writePPM( destination );
    }  
  
  
  //compute representative for best region
  
  NICE::Vector representative ( newFeatures.begin()->size(), 0.0 );
  
//   //FIRST TRY: average feature vectors of the "best" region
//   NICE::VVector::const_iterator posIt = positions.begin();
//   for ( NICE::VVector::const_iterator featIt = newFeatures.begin();
//         featIt != newFeatures.end();
//         featIt++, posIt++)
//   {              
//     
//     //only considere features that actually belong to the best region
//     if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion )
//       continue;
//     
//     representative += *featIt;
//   } 
//   
//   //simple mean feature vector
//   representative /= regionSize[indexOfBestRegion];
//   //normalization
//   representative.normalizeL1();
  
//   //SECOND TRY: simply take the first feature vector of the "best" region (although this one should lay on the border, and so on...)
//   NICE::VVector::const_iterator posIt = positions.begin();
//   for ( NICE::VVector::const_iterator featIt = newFeatures.begin();
//         featIt != newFeatures.end();
//         featIt++, posIt++)
//   {              
//     
//     //only considere features that actually belong to the best region
//     if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion )
//       continue;
//     
//     representative = *featIt;
//     i_posXOfNewPrototype = (*posIt)[0];
//     i_posYOfNewPrototype = (*posIt)[1];
//     //break after the first positive feature
//     break;
//   }   
  
  //THIRD TRY: simply take the feature vector of the "best" region with largest novelty score within this region
  // ... (hopefully, this is no outlier wrt to this region...)
  
  double maxNovelty ( 0.0 );
  NICE::VVector::const_iterator mostNovelFeature = newFeatures.begin() ;
  
  NICE::VVector::const_iterator posIt = positions.begin();
  for ( NICE::VVector::const_iterator featIt = newFeatures.begin();
        featIt != newFeatures.end();
        featIt++, posIt++)
  {              
    
    //only considere features that actually belong to the best region
    if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion )
      continue;
    
    //did we found a feature of the "best"region with larger novelty score then the current most novel one?
    if ( noveltyImageGaussFiltered( (*posIt)[0], (*posIt)[1] ) > maxNovelty )
    {
      maxNovelty = noveltyImageGaussFiltered( (*posIt)[0], (*posIt)[1] );
      mostNovelFeature = featIt;
      i_posXOfNewPrototype = (*posIt)[0];
      i_posYOfNewPrototype = (*posIt)[1];
    }
  }
  representative = *mostNovelFeature;
  
  std::cerr << " New representative: " << std::endl << representative << std::endl;
  
  //include the chosen information into the currently used prototypes
  prototypes.push_back( representative ); 
  
  if ( b_evaluationWhileFeatureLearning ) 
  {
    
    NICE::ColorImage imgTmp( _filename );
    
    double distToNewCluster ( std::numeric_limits<double>::max() );
    int indexOfMostSimFeat( 0 );
    double tmpDist;
    int tmpCnt ( 0 );
    
    for ( NICE::VVector::iterator i = newFeatures.begin();
          i != newFeatures.end();
          i++, tmpCnt++)
    {
      tmpDist = this->distFunction->calculate( *i, representative );
      if ( tmpDist < distToNewCluster )
      {
        distToNewCluster = tmpDist;
        indexOfMostSimFeat = tmpCnt;
      }
    }
    
    std::cerr << "** minDist to new cluster: " <<distToNewCluster << std::endl;
    
    int posX ( ( positions[indexOfMostSimFeat] ) [0]  );
    int posY ( ( positions[indexOfMostSimFeat] ) [1]  );
    
    std::cerr << "position of most similar feature --- posX: " << posX << " - posY " << posY << std::endl;
    std::cerr << "position of new prototype        --- posX: " << i_posXOfNewPrototype << " - posY " << i_posYOfNewPrototype << std::endl;
    NICE::Circle circ ( Coord( posX, posY), 10 /* radius*/, Color(200,0,255) );
    imgTmp.draw(circ); 
    
    if ( b_showResults )
      showImage(imgTmp, "Current (new) image and most similar feature for new cluster"); 
    else 
    {
      std::vector< std::string > list2;
      StringTools::split ( _filename, '/', list2 );      

      std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_2_bestNewCluster.ppm");
      imgTmp.writePPM( destination );
    }
  }  
  
  //this was a new image, so we increase our internal counter
  (this->newImageCounter)++;  
}