Feature Learning -- added region based approach

featureLearning/FeatureLearningRegionBased.cpp

@@ -0,0 +1,270 @@
+
+#include "FeatureLearningRegionBased.h"
+
+//STL
+#include <iostream>
+
+//core
+#include <core/image/FilterT.h>
+#include <core/image/CircleT.h>
+#include <core/image/Convert.h>
+#include <core/vector/VectorT.h>
+
+#include <segmentation/GenericRegionSegmentationMethodSelection.h>
+
+//vislearning
+#include <vislearning/baselib/Globals.h>
+
+
+using namespace std;
+using namespace NICE;
+using namespace OBJREC;
+
+  //**********************************************
+  //
+  //                 PROTECTED METHODS
+  //
+  //********************************************** 
+
+
+  //**********************************************
+  //
+  //                 PUBLIC METHODS
+  //
+  //********************************************** 
+
+
+FeatureLearningRegionBased::FeatureLearningRegionBased ( const Config *_conf,
+                               const MultiDataset *_md, const std::string & _section )
+    : FeatureLearningPrototypes ( _conf, _md, _section )
+{ 
+
+   //save and read segmentation results from files  
+  this->reuseSegmentation = conf->gB ( "FPCPixel", "reuseSegmentation", true );  
+  
+  // select your segmentation method here
+  // currently, the following options are supported: "MarkovCluster", "GraphBased", "MeanShift", "SLIC"
+  string rsMethode = conf->gS ( section, "segmentationMethod", "MeanShift" );
+ 
+  OBJREC::RegionSegmentationMethod *tmpRegionSeg = OBJREC::GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(conf, rsMethode);    
+  
+  if ( reuseSegmentation )
+    this->segmentationAlgo = new RSCache ( conf, tmpRegionSeg );
+  else
+    this->segmentationAlgo = tmpRegionSeg;
+  
+  this->i_gridSize = conf->gI( "LFColorSandeTest" , "grid" , 5 );
+}
+
+FeatureLearningRegionBased::~FeatureLearningRegionBased()
+{
+  // clean-up
+}
+
+void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filename )
+{  
+  NICE::ColorImage img( _filename );
+  
+  int xsize ( img.width() );
+  int ysize ( img.height() );
+   
+  //variables to store feature information
+  NICE::VVector newFeatures;
+  NICE::VVector cfeatures;
+  NICE::VVector positions;
+
+  //compute features
+  std::cerr << " EXTRACT FEATURES FROM UNSEEN IMAGE" << std::endl;
+  Globals::setCurrentImgFN ( _filename );
+  featureExtractor->extractFeatures ( img, newFeatures, positions );  
+  
+  //normalization :)
+  for ( NICE::VVector::iterator i = newFeatures.begin();
+        i != newFeatures.end();
+        i++)
+  {              
+    i->normalizeL1();
+  }
+  
+  //compute region segmentation
+  std::cerr << " COMPUTE REGION SEGMENTATION" << std::endl;
+  NICE::Matrix mask;
+  int amountRegions = segmentationAlgo->segRegions ( img, mask );
+  
+  //compute novelty scores on a feature level
+  std::cerr << " COMPUTE NOVELTY SCORES ON A FEATURE LEVEL" << std::endl;
+  FloatImage noveltyImageGaussFiltered ( xsize, ysize );    
+  this->evaluateCurrentCodebookForGivenFeatures( newFeatures, positions, noveltyImageGaussFiltered );  
+  
+  // compute scores for every region
+  std::cerr << " COMPUTE SCORES FOR EVERY REGION" << std::endl;
+  std::vector<double> regionNoveltyMeasure (amountRegions, 0.0); 
+  std::vector<int> regionSize (amountRegions, 0);
+  
+  for ( int y = 0; y < ysize; y += i_gridSize) //y++)
+  {
+    for (int x = 0; x < xsize; x += i_gridSize) //x++)
+    {
+      int r = mask(x,y);
+      regionSize[r]++;
+
+      //count the amount of "novelty" for the corresponding region
+      regionNoveltyMeasure[r] += noveltyImageGaussFiltered(x,y);
+    }
+  }
+  
+  //loop over all regions and compute averaged novelty scores
+  //NOTE this might be unuseful, since lateron we combine novelty score and region size (e.g. by multiplying)
+  // however, we do not want to settle the combination in adavance
+  for(int r = 0; r < amountRegions; r++)
+  {      
+    regionNoveltyMeasure[r] /= regionSize[r];  
+  }
+
+   //a new region should cover at least 3% of the image for not being penalized
+  double d_minimalImageAmountForAcceptableRegions ( 0.03 );
+  int minimalImageAmountForAcceptableRegions ( round( d_minimalImageAmountForAcceptableRegions * (xsize/i_gridSize) * (ysize/i_gridSize) )  );
+  
+  std::cerr << "minimalImageAmountForAcceptableRegions: " << minimalImageAmountForAcceptableRegions << std::endl;
+  
+  //compute final scores for all regions
+  NICE::Vector regionScores ( amountRegions, 0.0 );  
+  std::cerr << "used region sizes for computation: " << std::endl;
+  for(int r = 0; r < amountRegions; r++)
+  {      
+    regionScores[r] = std::min( regionSize[r], minimalImageAmountForAcceptableRegions ) * regionNoveltyMeasure[r];
+    std::cerr << " " << std::min( regionSize[r], minimalImageAmountForAcceptableRegions );
+  }
+  std::cerr << std::endl << std::endl;
+  
+  int indexOfBestRegion ( regionScores.MaxIndex() );
+  
+  
+  ////////////////////////////////////
+  //
+  //       VISUALIZE REGION SCORES
+  //
+  ////////////////////////////////////
+  
+  NICE::FloatImage regionScoreImage ( xsize, ysize );
+  NICE::FloatImage regionNoveltyImage ( xsize, ysize );
+  
+    for ( int y = 0; y < ysize; y++)
+    {
+      for (int x = 0; x < xsize; x++)
+      {
+        int r = mask(x,y);
+
+        regionNoveltyImage(x,y) = regionNoveltyMeasure[r];
+        regionScoreImage(x,y) = regionScores[r];
+      }
+    }  
+    
+    std::cerr << "highest region score: " << regionScoreImage.max()<< " -- smallest region score: " << regionScoreImage.min() << std::endl;
+    std::cerr << "highest region novelty score: " << regionNoveltyImage.max() << "  -- smallest region novelty score: " << regionNoveltyImage.min() << std::endl;
+    
+  NICE::ColorImage regionScoreImageRGB  ( xsize, ysize );
+  //TODO properly specify the maximum value for score visualization here :)
+//   imageToPseudoColorWithRangeSpecification( regionScoreImage, regionScoreImageRGB, 0 /* min */, 0.5 /* max */ );  
+  imageToPseudoColorWithRangeSpecification( regionScoreImage, regionScoreImageRGB, 0 /* min */, 2.2 /* max */ );  
+
+    if ( b_showResults )
+      showImage(regionScoreImageRGB, "Current (new) image with region scores"); 
+    else 
+    {
+      std::vector< std::string > list2;
+      StringTools::split ( _filename, '/', list2 );      
+
+      std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_regionScores.ppm");
+      regionScoreImageRGB.writePPM( destination );
+    }  
+  
+  
+  //compute representative for best region
+  
+  NICE::Vector representative ( newFeatures.begin()->size(), 0.0 );
+  
+  //first guess: average feature vectors of the "best" region
+//   NICE::VVector::const_iterator posIt = positions.begin();
+//   for ( NICE::VVector::const_iterator featIt = newFeatures.begin();
+//         featIt != newFeatures.end();
+//         featIt++, posIt++)
+//   {              
+//     
+//     //only considere features that actually belong to the best region
+//     if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion )
+//       continue;
+//     
+//     representative += *featIt;
+//   } 
+//   
+//   //simple mean feature vector
+//   representative /= regionSize[indexOfBestRegion];
+//   //normalization
+//   representative.normalizeL1();
+  
+  //next try: simply take the first feature vector of the "best" region (although this one should lay on the border, and so on...)
+  NICE::VVector::const_iterator posIt = positions.begin();
+  for ( NICE::VVector::const_iterator featIt = newFeatures.begin();
+        featIt != newFeatures.end();
+        featIt++, posIt++)
+  {              
+    
+    //only considere features that actually belong to the best region
+    if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion )
+      continue;
+    
+    representative = *featIt;
+    //break after the first positive feature
+    break;
+  }   
+  
+  std::cerr << " New representative: " << std::endl << representative << std::endl;
+  
+  //include the chosen information into the currently used prototypes
+  prototypes.push_back( representative ); 
+  
+  if ( b_evaluationWhileFeatureLearning ) 
+  {
+    
+    NICE::ColorImage imgTmp( _filename );
+    
+    double distToNewCluster ( std::numeric_limits<double>::max() );
+    int indexOfMostSimFeat( 0 );
+    double tmpDist;
+    int tmpCnt ( 0 );
+    
+    for ( NICE::VVector::iterator i = newFeatures.begin();
+          i != newFeatures.end();
+          i++, tmpCnt++)
+    {
+      tmpDist = this->distFunction->calculate( *i, representative );
+      if ( tmpDist < distToNewCluster )
+      {
+        distToNewCluster = tmpDist;
+        indexOfMostSimFeat = tmpCnt;
+      }
+    }
+    
+    std::cerr << "** minDist to new cluster: " <<distToNewCluster << std::endl;
+    
+    int posX ( ( positions[indexOfMostSimFeat] ) [0]  );
+    int posY ( ( positions[indexOfMostSimFeat] ) [1]  );
+    NICE::Circle circ ( Coord( posX, posY), 10 /* radius*/, Color(200,0,255) );
+    imgTmp.draw(circ); 
+    
+    if ( b_showResults )
+      showImage(imgTmp, "Current (new) image and most similar feature for new cluster"); 
+    else 
+    {
+      std::vector< std::string > list2;
+      StringTools::split ( _filename, '/', list2 );      
+
+      std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_2_bestNewCluster.ppm");
+      imgTmp.writePPM( destination );
+    }
+  }  
+  
+  //this was a new image, so we increase our internal counter
+  (this->newImageCounter)++;  
+}

featureLearning/FeatureLearningRegionBased.h

@@ -0,0 +1,73 @@
+/**
+ * @file FeatureLearningRegionBased.h
+ * @brief compute new histogram entries by extracting features from (moderately large) regions which have high novelty scores
+ * @author Alexander Freytag
+ * @date 17-04-2013 (dd-mm-yyyy)
+*/
+#ifndef _INCLUDEFEATURELEARNINGREGIONBASED
+#define _INCLUDEFEATURELEARNINGREGIONBASED
+
+#include "FeatureLearningPrototypes.h"
+
+#include "segmentation/RegionSegmentationMethod.h"
+
+
+namespace OBJREC
+{
+
+  /**
+   * @class FeatureLearningRegionBased
+   * @briefcompute new histogram entries by extracting features from (moderately large) regions which have high novelty scores
+   * @author Alexander Freytag
+   * @date 17-04-2013 (dd-mm-yyyy)
+  */  
+  
+  class FeatureLearningRegionBased : public FeatureLearningPrototypes
+  {
+
+    protected:
+
+      /************************
+       * 
+       *   protected variables
+       * 
+       **************************/  
+      
+      //! low level Segmentation method
+      RegionSegmentationMethod *segmentationAlgo;      
+      
+      //! boolean whether to reuse segmentation results for single images in different runs
+      bool reuseSegmentation;      
+      
+      //! determine a grid on which we extract local features
+      int i_gridSize;
+      
+      /************************
+       * 
+       *   protected methods
+       * 
+       **************************/      
+
+
+    public:
+
+      /** constructor
+        *  @param _conf needs a configfile
+        *  @param _md and a MultiDataset (contains images and other things)
+        */
+      FeatureLearningRegionBased ( const NICE::Config *_conf, const OBJREC::MultiDataset *_md , const std::string & _section = "featureLearning");
+
+      /** simple destructor */
+      virtual ~FeatureLearningRegionBased();
+      
+      /** this function has to be overloaded by all subclasses
+          @param imgWithNovelContent 
+      */
+      virtual void learnNewFeatures ( const std::string & filename );
+      
+  };
+
+
+} // namespace
+
+#endif