adapted feature learning framework to novelty detection interface

featureLearning/FeatureLearningGeneric.cpp

@@ -40,18 +40,4 @@ FeatureLearningGeneric::~FeatureLearningGeneric()
 
 void FeatureLearningGeneric::learnNewFeatures ( const std::string & _filename )
 {
-//   Globals::setCurrentImgFN ( filename );
-//   CachedExample *ce;
-//   if ( imagetype == IMAGETYPE_RGB )
-//   {
-//     NICE::ColorImage img = Preprocess::ReadImgAdvRGB ( filename );
-//     ce = new CachedExample ( img );
-//   } else {
-// 
-//     NICE::Image img = Preprocess::ReadImgAdv ( filename );
-//     ce = new CachedExample ( img );
-//   }
-//   fprintf ( stderr, "Starting Semantic Segmentation !\n" );
-//   learnNewFeatures ( ce );
-//   delete ce;
 }

featureLearning/FeatureLearningGeneric.h

@@ -9,12 +9,19 @@
 
 #define ROADWORKS fthrow(NICE::Exception, "Feature Learning -- not yet implemented!");
 
+//STL
 #include <string>
+//
+//core
 #include <core/basics/Config.h>
 #include <core/image/ImageT.h>
+#include <core/vector/VVector.h>
+//
+//vislearning
 #include <vislearning/cbaselib/CachedExample.h>
 
 
+
 namespace OBJREC
 {
 
@@ -111,6 +118,8 @@ namespace OBJREC
       virtual NICE::ImageT<int> evaluateCurrentCodebookByAssignments ( const std::string & _filename , const bool & beforeComputingNewFeatures = true, const bool & _binaryShowLatestPrototype = false) = 0;      
       
       virtual void evaluateCurrentCodebookByConfusionMatrix( NICE::Matrix & _confusionMat ) = 0; 
+      
+      virtual NICE::VVector * getCurrentCodebook() = 0;
   
 
   };

featureLearning/FeatureLearningPrototypes.cpp

@@ -702,3 +702,8 @@ void FeatureLearningPrototypes::evaluateCurrentCodebookByConfusionMatrix( NICE::
   }
 }
 
+NICE::VVector * FeatureLearningPrototypes::getCurrentCodebook()
+{
+  return &(this->prototypes);
+}
+

featureLearning/FeatureLearningPrototypes.h

@@ -94,8 +94,9 @@ namespace OBJREC
     public:
 
       /** constructor
-        *  @param _conf needs a configfile
-        *  @param _md and a MultiDataset (contains images and other things)
+        * @param _conf needs a configfile
+        * @param _md and a MultiDataset (contains images and other things)
+        * @param _section section information for parsing config files
         */
       FeatureLearningPrototypes ( const NICE::Config *_conf, const OBJREC::MultiDataset *_md , const std::string & _section = "featureLearning");
 
@@ -113,6 +114,8 @@ namespace OBJREC
       
       virtual void evaluateCurrentCodebookByConfusionMatrix( NICE::Matrix & _confusionMat ); 
       
+      virtual NICE::VVector * getCurrentCodebook();
+      
 
   };
 

featureLearning/FeatureLearningRegionBased.cpp

@@ -147,7 +147,12 @@ void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filenam
   ////////////////////////////////////
   
   NICE::FloatImage regionScoreImage ( xsize, ysize );
-  NICE::FloatImage regionNoveltyImage ( xsize, ysize );
+  NICE::FloatImage regionNoveltyImage ( xsize, ysize ); //contains novelty score averaged over region
+  NICE::FloatImage regionRelevanceImage ( xsize, ysize ); //contains higher scores for larger regions (but with upper limit)
+  
+  regionScoreImage.set( 0.0 );
+  regionNoveltyImage.set( 0.0 );
+  regionRelevanceImage.set( 0.0 );
   
     for ( int y = 0; y < ysize; y++)
     {
@@ -156,6 +161,7 @@ void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filenam
         int r = mask(x,y);
 
         regionNoveltyImage(x,y) = regionNoveltyMeasure[r];
+        regionRelevanceImage(x,y) = regionSize[r];
         regionScoreImage(x,y) = regionScores[r];
       }
     }  
@@ -164,9 +170,12 @@ void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filenam
     std::cerr << "highest region novelty score: " << regionNoveltyImage.max() << "  -- smallest region novelty score: " << regionNoveltyImage.min() << std::endl;
     
   NICE::ColorImage regionScoreImageRGB  ( xsize, ysize );
+  NICE::ColorImage regionNoveltyScoreImageRGB  ( xsize, ysize );
+  NICE::ColorImage regionRelevanceScoreImageRGB  ( xsize, ysize );
   //TODO properly specify the maximum value for score visualization here :)
-//   imageToPseudoColorWithRangeSpecification( regionScoreImage, regionScoreImageRGB, 0 /* min */, 0.5 /* max */ );  
   imageToPseudoColorWithRangeSpecification( regionScoreImage, regionScoreImageRGB, 0 /* min */, 2.2 /* max */ );  
+  imageToPseudoColorWithRangeSpecification( regionNoveltyImage, regionNoveltyScoreImageRGB, 0 /* min */, 0.012 /* max */ );  
+  imageToPseudoColorWithRangeSpecification( regionRelevanceImage, regionRelevanceScoreImageRGB, 0 /* min */, minimalImageAmountForAcceptableRegions /* max */ );  
 
     if ( b_showResults )
       showImage(regionScoreImageRGB, "Current (new) image with region scores"); 
@@ -174,8 +183,17 @@ void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filenam
     {
       std::vector< std::string > list2;
       StringTools::split ( _filename, '/', list2 );      
+     
+      //write novelty score image
+      std::string destinationNovelty ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_1_regionNoveltyScores.ppm");
+      regionNoveltyScoreImageRGB.writePPM( destinationNovelty );
+      
+      //write relevance score image
+      std::string destinationRelevance ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_2_regionRelevanceScores.ppm");
+      regionRelevanceScoreImageRGB.writePPM( destinationRelevance );      
 
-      std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_regionScores.ppm");
+      //write image with final scores for region
+      std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_3_regionScores.ppm");
       regionScoreImageRGB.writePPM( destination );
     }  
   

featureLearning/libdepend.inc

@@ -3,4 +3,5 @@ $(call PKG_DEPEND_INT,vislearning/baselib/)
 $(call PKG_DEPEND_INT,vislearning/cbaselib/)
 $(call PKG_DEPEND_INT,vislearning/features/)
 $(call PKG_DEPEND_INT,vislearning/math/)
+$(call PKG_DEPEND_INT,vislearning/noveltyDetecion/)
 $(call PKG_DEPEND_INT,segmentation/)

featureLearning/progs/libdepend.inc

@@ -2,4 +2,5 @@ $(call PKG_DEPEND_INT,vislearning/baselib/)
 $(call PKG_DEPEND_INT,vislearning/cbaselib/)
 $(call PKG_DEPEND_INT,vislearning/features/)
 $(call PKG_DEPEND_INT,vislearning/math/)
+$(call PKG_DEPEND_INT,vislearning/noveltyDetection/)
 $(call PKG_DEPEND_INT,segmentation/)

featureLearning/progs/testFeatureLearning.cpp

@@ -24,6 +24,8 @@
 #include "vislearning/featureLearning/FeatureLearningGeneric.h"
 #include "vislearning/featureLearning/FeatureLearningClusterBased.h"
 #include "vislearning/featureLearning/FeatureLearningRegionBased.h"
+//
+#include "vislearning/noveltyDetection/NDCodebookLevelImagePooling.h"
 
 
 using namespace std;
@@ -83,6 +85,15 @@ int main( int argc, char **argv )
     std::cerr << "Unknown feature learning algorithm selected, use cluster based instead" << std::endl;
     featureLearning = new OBJREC::FeatureLearningClusterBased( conf, &md );
   }    
+  
+  //**********************************************
+  //
+  //      SET UP THE NOVELTY DECTION ALGO
+  //
+  //**********************************************   
+  
+  OBJREC::NDCodebookLevelImagePooling * novDetector;
+  novDetector = new OBJREC::NDCodebookLevelImagePooling( conf, &md, "featureLearning" );
     
   //evaluate how well the training images are covered with our initial codebook
   //that is, compute these nice "novelty maps" per feature
@@ -116,6 +127,9 @@ int main( int argc, char **argv )
   featureLearning->evaluateCurrentCodebookByConfusionMatrix( confusionMatInitial );
   std::cerr << "initial Confusion matrix: " << std::endl << confusionMatInitial << std::endl;
   
+  //set the initially computed codebook to the novelty detection mechanism
+  //TODO this should be done, but currently we do not care about
+//   novDetector->setCodebook( featureLearning->getCurrentCodebook() );
   
 
   //**********************************************
@@ -137,7 +151,6 @@ int main( int argc, char **argv )
       std::string filename = info.img();
       
       NICE::ColorImage orig( filename );
-//       showImage( orig, "Input" );
       
       NICE::FloatImage noveltyImageBefore;
       noveltyImageBefore = featureLearning->evaluateCurrentCodebookByDistance( filename , true /* beforeComputingNewFeatures */ );
@@ -150,24 +163,9 @@ int main( int argc, char **argv )
       //
       //**********************************************       
       
-      //TODO currently hard coded, and only stupid averaging of feature novelties for whole image, and hard thresholding!
-      double meanNovelty ( 0.0 );
-      for ( uint y = 0 ; y < ( uint ) noveltyImageBefore.height() ; y++ )
-      {
-        for ( uint x = 0 ; x < ( uint ) noveltyImageBefore.width(); x++ )
-        {
-          meanNovelty += noveltyImageBefore(x,y);
-        }
-      }      
-      int imageSize ( noveltyImageBefore.height() * noveltyImageBefore.width() );
-      meanNovelty /= imageSize;
-      
-//       double stupidThreshold ( 0.0022 ); // for grid size 5
-//       double stupidThreshold ( 0.01 ); // for grid size 3
-      double stupidThreshold = conf->gD ( "featureLearning" , "stupidNoveltyThreshold", 0.0064 );
+      bool b_isImageNovel ( novDetector->evaluateNoveltyOfImage( noveltyImageBefore ) );
       
-      std::cerr << "  NOVELTY SCORE FOR CURRENT IMAGE: " <<  meanNovelty << " -- threshold: " << stupidThreshold << std::endl;
-      if ( meanNovelty < stupidThreshold )
+      if ( ! b_isImageNovel )
       {
         std::cerr << " --- NOT NOVEL --- " << std::endl << std::endl;
         continue;
@@ -185,6 +183,10 @@ int main( int argc, char **argv )
       
       featureLearning->learnNewFeatures( filename );
       
+      //and update the codebook pointer within our novelty detection algorithm
+      //TODO this should be done, but currently we do not care about
+//       novDetector->setCodebook( featureLearning->getCurrentCodebook() );      
+      
       //**********************************************
       //
       //       EVALUATE HOW WELL THE CURRENT IMAGE
@@ -253,5 +255,17 @@ int main( int argc, char **argv )
       imageCnt++;
   } //Loop over all test images
   
+  
+  //don't waste memory
+  std::cerr << "don't waste memory - cleaning up" << std::endl;
+//   if (trainFiles != NULL)
+//     delete trainFiles;
+  if (featureLearning != NULL)
+    delete featureLearning;
+  if (novDetector != NULL)
+    delete novDetector;
+  if (conf != NULL)
+    delete conf;
+  
    return 0;
 }