First class hierarchy for Feature Learning done

featureLearning/FeatureLearningClusterBased.cpp

@@ -0,0 +1,320 @@
+
+#include "FeatureLearningClusterBased.h"
+
+#include <iostream>
+
+#include <core/image/FilterT.h>
+#include <core/vector/VectorT.h>
+
+
+#include <vislearning/baselib/ICETools.h>
+// 
+#include <vislearning/features/localfeatures/LFonHSG.h>
+#include <vislearning/features/localfeatures/LFColorSande.h>
+#include <vislearning/features/localfeatures/LFColorWeijer.h>
+#include <vislearning/features/localfeatures/LFReadCache.h>
+#include <vislearning/features/localfeatures/LFWriteCache.h>
+// 
+#include <vislearning/math/cluster/KMeans.h>
+#include <vislearning/math/cluster/GMM.h>
+
+using namespace std;
+using namespace NICE;
+using namespace OBJREC;
+
+  //**********************************************
+  //
+  //                 PROTECTED METHODS
+  //
+  //********************************************** 
+
+void FeatureLearningClusterBased::extractFeaturesFromTrainingImages( const OBJREC::MultiDataset *_md, NICE::VVector & examplesTraining )
+{
+  examplesTraining.clear();
+  
+  int numberOfTrainImage ( 0 );
+  
+  const LabeledSet *trainFiles = (*_md)["train"]; 
+  
+  //run over all training images
+  LOOP_ALL_S( *trainFiles )
+  {
+      EACH_INFO( classno, info );
+      std::string filename = info.img();
+            
+      NICE::ColorImage img( filename );
+      if ( showTrainingImages )
+      {
+        showImage( img, "Input" );    
+      }
+      
+      //variables to store feature informatio
+      NICE::VVector features;
+      NICE::VVector cfeatures;
+      NICE::VVector positions;
+
+      //compute features
+      Globals::setCurrentImgFN ( filename );
+      if (featureExtractor == NULL)
+        std::cerr << "feature Extractor is NULL" << std::endl;
+      else
+        featureExtractor->extractFeatures ( img, features, positions );
+            
+      //store feature information in larger data structure
+      for ( NICE::VVector::iterator i = features.begin();
+            i != features.end();
+            i++)
+      {              
+        //normalization :)
+        i->normalizeL1();
+
+        examplesTraining.push_back(*i);
+      }
+      
+      //don't waste memory
+      features.clear();
+      positions.clear();      
+      numberOfTrainImage++;
+  }//Loop over all training images    
+}
+
+void FeatureLearningClusterBased::train ( const OBJREC::MultiDataset *_md )
+{
+ //**********************************************
+  //
+  //     EXTRACT FEATURES FROM TRAINING IMAGES
+  //
+  //**********************************************  
+  
+  std::cerr << " EXTRACT FEATURES FROM TRAINING IMAGES" << std::endl;
+  
+  NICE::VVector examplesTraining;  
+  this->extractFeaturesFromTrainingImages( _md, examplesTraining  );
+  
+  //**********************************************
+  //
+  //    CLUSTER FEATURES FROM TRAINING IMAGES
+  //
+  //    THIS GIVES US AN INITIAL CODEBOOK
+  //
+  //**********************************************  
+  std::cerr << " CLUSTER FEATURES FROM TRAINING IMAGES" << std::endl;
+  //go, go, go...  
+  prototypes.clear();
+  std::vector< double > weights;
+  std::vector< int > assignment;
+  clusterAlgo->cluster ( examplesTraining, prototypes, weights, assignment);
+  weights.clear();
+  assignment.clear();  
+}
+
+  //**********************************************
+  //
+  //                 PUBLIC METHODS
+  //
+  //********************************************** 
+
+
+FeatureLearningClusterBased::FeatureLearningClusterBased ( const Config *_conf,
+                               const MultiDataset *_md, const std::string & _section )
+    : FeatureLearningGeneric ( _conf )
+{
+  this->section = _section;
+  
+  
+    //feature stuff
+  //! which OpponentSIFT implementation to use {NICE, VANDESANDE}
+  std::string opSiftImpl;  
+  opSiftImpl = conf->gS ( "Descriptor", "implementation", "VANDESANDE" );
+  //! read features?
+  bool readfeat;
+  readfeat = conf->gB ( "Descriptor", "read", true );
+  //! write features?
+  bool writefeat;  
+  writefeat = conf->gB ( "Descriptor", "write", true ); 
+  
+  showTrainingImages = conf->gB( section, "showTrainingImages", false );  
+  
+  
+  //! define the initial number of clusters our codebook shall contain
+  initialNumberOfClusters = conf->gI(section, "initialNumberOfClusters", 10);
+  
+  //! define the clustering algorithm to be used
+  std::string clusterAlgoString = conf->gS(section, "clusterAlgo", "kmeans");
+  
+  //! define the distance function to be used
+  std::string distFunctionString = conf->gS(section, "distFunction", "euclidian");    
+  
+  
+  //**********************************************
+  //
+  //      SET UP VARIABLES AND METHODS
+  //             - FEATURE TYPE
+  //             - CLUSTERING ALGO
+  //             - DISTANCE FUNCTION
+  //             - ...
+  //
+  //**********************************************  
+  
+  std::cerr << " SET UP VARIABLES AND METHODS " << std::endl;
+  
+  // Welche Opponentsift Implementierung soll genutzt werden ?
+  LocalFeatureRepresentation *cSIFT = NULL;
+  LocalFeatureRepresentation *writeFeats = NULL;
+  LocalFeatureRepresentation *readFeats = NULL;
+  this->featureExtractor = NULL;
+  if ( opSiftImpl == "NICE" )
+  {
+    cSIFT = new OBJREC::LFonHSG ( conf, "HSGtrain" );
+  }
+  else if ( opSiftImpl == "VANDESANDE" )
+  {
+    cSIFT = new OBJREC::LFColorSande ( conf, "LFColorSandeTrain" );
+  }
+  else
+  {
+    fthrow ( Exception, "feattype: %s not yet supported" << opSiftImpl );
+  }
+
+  this->featureExtractor = cSIFT;
+  
+  if ( writefeat )
+  {
+    // write the features to a file, if there isn't any to read
+    writeFeats = new LFWriteCache ( conf, cSIFT );
+    this->featureExtractor = writeFeats;
+  }
+
+  if ( readfeat )
+  {
+    // read the features from a file
+    if ( writefeat )
+    {
+      readFeats = new LFReadCache ( conf, writeFeats, -1 );
+    }
+    else
+    {
+      readFeats = new LFReadCache ( conf, cSIFT, -1 );
+    }
+    this->featureExtractor = readFeats; 
+  }
+  
+  if (clusterAlgoString.compare("kmeans") == 0)
+  {
+    clusterAlgo = new OBJREC::KMeans(initialNumberOfClusters);
+  }
+  else if (clusterAlgoString.compare("GMM") == 0) 
+  {
+    clusterAlgo = new OBJREC::GMM(conf, initialNumberOfClusters);
+  }
+  else
+  {
+    std::cerr << "Unknown cluster algorithm selected, use k-means instead" << std::endl;
+    clusterAlgo = new OBJREC::KMeans(initialNumberOfClusters);
+  }
+  
+  if (distFunctionString.compare("euclidian") == 0)
+  {
+    distFunction = new NICE::EuclidianDistance<double>();
+  }
+  else
+  {
+    std::cerr << "Unknown vector distance selected, use euclidian instead" << std::endl;
+    distFunction = new NICE::EuclidianDistance<double>();
+  }    
+  
+  //run the training to initially compute a codebook and stuff like that
+  this->train( _md );
+  
+  //only set feature stuff to NULL, deletion of the underlying object is done in the destructor
+  if ( cSIFT != NULL )
+    cSIFT = NULL;
+  if ( writeFeats != NULL )
+    writeFeats = NULL;
+  if ( readFeats != NULL )
+    readFeats  = NULL ;    
+}
+
+FeatureLearningClusterBased::~FeatureLearningClusterBased()
+{
+  // clean-up
+  if ( clusterAlgo != NULL )
+    delete clusterAlgo;
+  if ( distFunction != NULL )
+    delete distFunction;
+  if ( featureExtractor != NULL )
+    delete featureExtractor; 
+}
+
+void FeatureLearningClusterBased::learnNewFeatures ( OBJREC::CachedExample *_ce )
+{  
+}
+
+void FeatureLearningClusterBased::evaluateCurrentCodebook ( const std::string & filename )
+{
+    NICE::ColorImage img( filename );
+    if ( showTrainingImages )
+    {
+      showImage( img, "Input" );    
+    }
+    
+    int xsize ( img.width() );
+    int ysize ( img.height() );
+    
+    //variables to store feature information
+    NICE::VVector features;
+    NICE::VVector cfeatures;
+    NICE::VVector positions;
+
+    //compute features
+    Globals::setCurrentImgFN ( filename );
+    featureExtractor->extractFeatures ( img, features, positions );
+    
+    FloatImage noveltyImage ( xsize, ysize );
+    noveltyImage.set ( 0.0 );        
+    
+    double maxDist ( 0.0 );
+    
+    NICE::VVector::const_iterator posIt = positions.begin();
+    //store feature information in larger data structure
+    for ( NICE::VVector::iterator i = features.begin();
+          i != features.end();
+          i++, posIt++)
+    {              
+      //normalization :)
+      i->normalizeL1();
+      
+      //loop over codebook representatives
+      double minDist ( std::numeric_limits<double>::max() );
+      for (NICE::VVector::const_iterator it =  prototypes.begin(); it != prototypes.end(); it++)
+      {
+        //compute distance
+        double tmpDist ( distFunction->calculate(*i,*it) );
+        if (tmpDist < minDist)
+          minDist = tmpDist;
+      }
+      
+      if (minDist > maxDist)
+        maxDist = minDist;
+
+      //take minimum distance and store in in a float image
+      
+      noveltyImage ( (*posIt)[0], (*posIt)[1]  ) = minDist;
+    }      
+    
+    //gauss-filtering for nicer visualization
+    FloatImage noveltyImageGaussFiltered ( xsize, ysize );
+    float sigma ( 3.0 );
+    FilterT<float, float, float> filter;
+    filter.filterGaussSigmaApproximate ( noveltyImage, sigma, &noveltyImageGaussFiltered );
+
+    std::cerr << "maximum distance of Training images: " << maxDist;      
+    //for suitable visualization of scores between zero (known) and one (unknown)
+//       noveltyImageGaussFiltered( 0 , 0 ) = std::max<double>(maxDist, 1.0);
+
+    
+    //convert float to RGB
+    NICE::ColorImage noveltyImageRGB ( xsize, ysize  );
+    ICETools::convertToRGB ( noveltyImageGaussFiltered, noveltyImageRGB );
+    showImage(noveltyImageRGB, "Novelty Image");  
+}

featureLearning/FeatureLearningClusterBased.h

@@ -0,0 +1,82 @@
+/**
+* @file FeatureLearningClusterBased.h
+* @brief compute new histogram entries by clustering new features and take centers of clusters which are relevant and novel
+* @author Alexander Freytag
+* @date 16-04-2013 (dd-mm-yyyy)
+
+*/
+#ifndef _INCLUDEFEATURELEARNINGCLUSTERBASED
+#define _INCLUDEFEATURELEARNINGCLUSTERBASED
+
+#include "FeatureLearningGeneric.h"
+
+#include <string>
+
+#include <core/vector/Distance.h>
+#include <core/vector/VVector.h>
+
+#include <vislearning/cbaselib/MultiDataset.h>
+#include <vislearning/features/localfeatures/GenericLocalFeatureSelection.h>
+#include <vislearning/math/cluster/ClusterAlgorithm.h>
+
+namespace OBJREC
+{
+
+  /** abstract interface for feature learning algorithms */
+  class FeatureLearningClusterBased : public FeatureLearningGeneric
+  {
+
+    protected:
+      
+      std::string section;
+      
+      bool showTrainingImages;
+      
+      //! define the initial number of clusters our codebook shall contain
+      int initialNumberOfClusters;       
+  
+      OBJREC::ClusterAlgorithm * clusterAlgo;
+      NICE::VectorDistance<double> * distFunction;
+
+      LocalFeatureRepresentation * featureExtractor;
+      
+      NICE::VVector prototypes;     
+      
+      /************************
+       * 
+       *   protected methods
+       * 
+       **************************/
+      
+      void extractFeaturesFromTrainingImages(  const OBJREC::MultiDataset *_md,  NICE::VVector & examplesTraining  );
+      
+      void train ( const OBJREC::MultiDataset *_md ); 
+      
+      
+
+
+    public:
+
+      /** constructor
+        *  @param conf needs a configfile
+        *  @param md and a MultiDataset (contains images and other things)
+        */
+      FeatureLearningClusterBased ( const NICE::Config *_conf, const OBJREC::MultiDataset *_md , const std::string & _section = "featureLearning");
+
+      /** simple destructor */
+      virtual ~FeatureLearningClusterBased();
+      
+      /** this function has to be overloaded by all subclasses
+          @param imgWithNovelContent 
+      */
+      virtual void learnNewFeatures ( OBJREC::CachedExample *_ce ) ;
+      
+      virtual void evaluateCurrentCodebook ( const std::string & filename );
+      
+
+  };
+
+
+} // namespace
+
+#endif

featureLearning/FeatureLearningGeneric.cpp

@@ -0,0 +1,44 @@
+/**
+* @file FeatureLearningGeneric.cpp
+* @brief abstract interface for feature learning algorithms
+* @author Alexander Freytag
+* @date 16-04-2013 (dd-mm-yyyy)
+
+*/
+#include <iostream>
+
+#include <vislearning/baselib/Preprocess.h>
+
+#include "FeatureLearningGeneric.h"
+
+using namespace OBJREC;
+using namespace std;
+using namespace NICE;
+
+FeatureLearningGeneric::FeatureLearningGeneric ( const Config *_conf )
+{
+  this->conf = _conf;  
+  Preprocess::Init ( _conf );
+}
+
+FeatureLearningGeneric::~FeatureLearningGeneric()
+{
+}
+
+void FeatureLearningGeneric::learnNewFeatures ( const std::string & _filename )
+{
+//   Globals::setCurrentImgFN ( filename );
+//   CachedExample *ce;
+//   if ( imagetype == IMAGETYPE_RGB )
+//   {
+//     NICE::ColorImage img = Preprocess::ReadImgAdvRGB ( filename );
+//     ce = new CachedExample ( img );
+//   } else {
+// 
+//     NICE::Image img = Preprocess::ReadImgAdv ( filename );
+//     ce = new CachedExample ( img );
+//   }
+//   fprintf ( stderr, "Starting Semantic Segmentation !\n" );
+//   learnNewFeatures ( ce );
+//   delete ce;
+}

featureLearning/FeatureLearningGeneric.h

@@ -0,0 +1,58 @@
+/**
+* @file FeatureLearningGeneric.h
+* @brief abstract interface for feature learning algorithms
+* @author Alexander Freytag
+* @date 16-04-2013 (dd-mm-yyyy)
+
+*/
+#ifndef _INCLUDEFEATURELEARNING
+#define _INCLUDEFEATURELEARNING
+
+#define ROADWORKS fthrow(NICE::Exception, "Feature Learning -- not yet implemented!");
+
+#include <string>
+#include <core/basics/Config.h>
+#include <vislearning/cbaselib/CachedExample.h>
+
+
+namespace OBJREC
+{
+
+  /** abstract interface for feature learning algorithms */
+  class FeatureLearningGeneric
+  {
+
+    protected:   
+      
+    //! Configuration File
+    const NICE::Config *conf;      
+      
+
+    public:
+
+      /** simple constructor
+          @param conf global settings
+      */
+      FeatureLearningGeneric ( const NICE::Config *_conf );
+
+      /** simple destructor */
+      virtual ~FeatureLearningGeneric();
+
+      
+      //TODO  possibly move this to protected...
+      /** this function has to be overloaded by all subclasses
+          @param imgWithNovelContent 
+      */
+      virtual void learnNewFeatures ( OBJREC::CachedExample *_ce ) = 0;  
+
+      virtual void learnNewFeatures ( const std::string & _filename );    
+      
+      virtual void evaluateCurrentCodebook ( const std::string & filename ) = 0;
+  
+
+  };
+
+
+} // namespace
+
+#endif

featureLearning/libdepend.inc

@@ -1 +1,5 @@
 $(call PKG_DEPEND_INT,core/)
+$(call PKG_DEPEND_INT,vislearning/baselib/)
+$(call PKG_DEPEND_INT,vislearning/cbaselib/)
+$(call PKG_DEPEND_INT,vislearning/features/)
+$(call PKG_DEPEND_INT,vislearning/math/)

featureLearning/progs/libdepend.inc

@@ -0,0 +1,4 @@
+$(call PKG_DEPEND_INT,vislearning/baselib/)
+$(call PKG_DEPEND_INT,vislearning/cbaselib/)
+$(call PKG_DEPEND_INT,vislearning/features/)
+$(call PKG_DEPEND_INT,vislearning/math/)

featureLearning/progs/testFeatureLearning.cpp

@@ -5,15 +5,26 @@
 * @date 11-04-2013
 */
 
+#include <iostream>
+#include <limits>
+
 #include <core/basics/Config.h>
 #include <core/basics/ResourceStatistics.h>
+#include <core/vector/VectorT.h>
 
+#include <vislearning/baselib/Globals.h>
+#include <vislearning/baselib/ICETools.h>
+#include <vislearning/cbaselib/MultiDataset.h>
+#include <vislearning/cbaselib/Example.h>
 
-using namespace OBJREC;
+#include "vislearning/featureLearning/FeatureLearningGeneric.h"
+#include "vislearning/featureLearning/FeatureLearningClusterBased.h"
 
-using namespace NICE;
 
 using namespace std;
+using namespace NICE;
+using namespace OBJREC;
+
 
 /**
  test feature learning routines
@@ -22,9 +33,70 @@ int main( int argc, char **argv )
 {
   std::set_terminate( __gnu_cxx::__verbose_terminate_handler );
 
-  Config conf( argc, argv );
+  Config * conf = new Config ( argc, argv );
+  
+  bool showTrainingImages= conf->gB( "featureLearning", "showTrainingImages", false );
+  bool showTestImages= conf->gB( "featureLearning", "showTestImages", false );
   
   ResourceStatistics rs;
+  std::string resultdir;
+  
+  
+  //**********************************************
+  //
+  //      READ INITIAL TRAINING SET TO COMPUTE
+  //             AN INITIAL CODEBOOK
+  //
+  //**********************************************
+  
+  std::cerr << " READ INITIAL TRAINING SET TO COMPUTE AN INITIAL CODEBOOK" << std::endl;
+  
+  MultiDataset md( conf );
+  const LabeledSet *trainFiles = md["train"];
+  
+  //**********************************************
+  //
+  //      SET UP THE FEATURE LEARNING ALGO
+  //
+  //********************************************** 
+  
+  OBJREC::FeatureLearningGeneric * featureLearning;
+  featureLearning = new OBJREC::FeatureLearningClusterBased( conf, &md );
+  
+  //print computed cluster centers  -- this is NOT recommended :)
+//   prototypes.store(std::cerr);
+  
+  //evaluate how well the training images are covered with our initial codebook
+  //that is, compute these nice "novelty maps" per feature
+  
+  LOOP_ALL_S( *trainFiles )
+  {
+      EACH_INFO( classno, info );
+      std::string filename = info.img();
+      
+      featureLearning->evaluateCurrentCodebook( filename );       
+  }
+
+  //**********************************************
+  //
+  //        FOR-LOOP OVER UNSEEN IMAGES
+  //
+  //       EXTRACT FEATURES, CLUSTER THEM, TAKE
+  //       MOST "VALUABLE" CLUSTERS AS NEW 
+  //    REPRESENTATIVES IN AN INCREASED CODEBOK
+  //
+  //**********************************************
+  const LabeledSet *testFiles = md["test"];
+  
+  std::cerr << "start looping over all files" << std::endl;
+  LOOP_ALL_S( *testFiles )
+  {
+      EACH_INFO( classno, info );
+      std::string file = info.img();
+      
+      NICE::ColorImage orig( file );
+      showImage( orig, "Input" );
+  } //Loop over all test images
   
    return 0;
-}
+}