added all currently developed AL SemSeg progs, still under development

progs/testActiveSemanticSegmentation.cpp

@@ -0,0 +1,556 @@
+// Beispielhafter Aufruf: BUILD_x86_64/progs/testActiveSemanticSegmentation -config <CONFIGFILE>
+
+/**
+* @file testActiveSemanticSegmentation.cpp
+* @brief test semantic segmentation routines with actively selecting regions for labeling
+* @author Alexander Freytag
+* @date 27-02-2013
+*/
+
+#ifdef NICE_USELIB_OPENMP
+#include <omp.h>
+#endif
+
+#include "core/basics/Config.h"
+#include "core/basics/StringTools.h"
+#include <vislearning/baselib/ICETools.h>
+
+#include <semseg/semseg/SemanticSegmentation.h>
+#include <semseg/semseg/SemSegLocal.h>
+#include <semseg/semseg/SemSegCsurka.h>
+#include <semseg/semseg/SemSegNovelty.h>
+#include <semseg/semseg/SemSegNoveltyBinary.h>
+#include <semseg/semseg/SemSegContextTree.h>
+
+#include "core/image/FilterT.h"
+
+#include <core/basics/ResourceStatistics.h>
+
+#include <fstream>
+
+using namespace OBJREC;
+
+using namespace NICE;
+
+using namespace std;
+
+void updateMatrix( const NICE::Image & img, const NICE::Image & gt,
+                   NICE::Matrix & M, const set<int> & forbidden_classes )
+{
+  double subsamplex = gt.width() / ( double )img.width();
+  double subsampley = gt.height() / ( double )img.height();
+
+  for ( int y = 0 ; y < gt.height() ; y++ )
+    for ( int x = 0 ; x < gt.width() ; x++ )
+    {
+      int xx = ( int )( x / subsamplex );
+      int yy = ( int )( y / subsampley );
+
+      if ( xx < 0 ) xx = 0;
+
+      if ( yy < 0 ) yy = 0;
+
+      if ( xx > img.width() - 1 ) xx = img.width() - 1;
+
+      if ( yy > img.height() - 1 ) yy = img.height() - 1;
+
+      int cimg = img.getPixel( xx, yy );
+
+      int gimg = gt.getPixel( x, y );
+
+      if ( forbidden_classes.find( gimg ) == forbidden_classes.end() )
+      {
+        M( gimg, cimg )++;
+      }
+    }
+}
+
+/**
+ test semantic segmentation routines
+*/
+int main( int argc, char **argv )
+{
+  std::set_terminate( __gnu_cxx::__verbose_terminate_handler );
+
+  Config conf( argc, argv );
+  
+  ResourceStatistics rs;
+  
+  bool show_result = conf.gB( "debug", "show_results", false );
+
+  bool write_results = conf.gB( "debug", "write_results", false );
+
+  bool write_results_pascal = conf.gB( "debug", "write_results_pascal", false );
+
+  std::string resultdir = conf.gS( "debug", "resultdir", "." );
+  
+  //how often do we want to iterate between sem-seg and active query?
+  int activeIterations = conf.gI("main", "activeIterations", 1 );
+    
+  if ( write_results )
+  {
+    cerr << "Writing Results to " << resultdir << endl;
+  }
+
+  MultiDataset md( &conf );
+
+  const ClassNames & classNames = md.getClassNames( "train" );
+
+  string method = conf.gS( "main", "method", "SSCsurka" );
+
+  //currently, we only allow SemSegNovelty, because it implements addNovelExamples()
+  SemanticSegmentation *semseg = NULL;
+  
+      Timer timer;
+      timer.start();
+  if ( method == "SSCsurka" )
+  {
+    semseg = new SemSegCsurka( &conf, &md );
+  }
+  else if ( method == "SSContext" )
+  {
+    semseg = new SemSegContextTree( &conf, &md );
+  }
+  else if( method == "SSNovelty" )
+  {
+    semseg = new SemSegNovelty( &conf, &md );
+  }
+  else if( method == "SSNoveltyBinary" )
+  {
+    semseg = new SemSegNoveltyBinary( &conf, &md );
+  }  
+  timer.stop();
+  std::cerr << "AL time for training: " << timer.getLast() << std::endl;
+
+  const LabeledSet *testFiles = md["test"];
+
+  NICE::Matrix M( classNames.getMaxClassno() + 1, classNames.getMaxClassno() + 1 );
+
+  M.set( 0 );
+
+  std::set<int> forbidden_classes;
+  std::string forbidden_classes_s = conf.gS( "analysis", "forbidden_classesTrain", "" );
+  classNames.getSelection( forbidden_classes_s, forbidden_classes );
+  
+  std::set<int> forbidden_classesForActiveLearning;
+  std::string forbidden_classesForActiveLearning_s = conf.gS( "analysis", "forbidden_classesForActiveLearning", "" );
+  classNames.getSelection( forbidden_classesForActiveLearning_s, forbidden_classesForActiveLearning );
+  
+
+  for (int iterationCount = 0; iterationCount < activeIterations; iterationCount++)
+  {
+      //TODO shouldn't we clean the confusion matrix at the beginning of each iteration?
+    
+    std::cerr << "SemSeg AL Iteration: " << iterationCount << std::endl;
+    semseg->setIterationCountSuffix(iterationCount);
+    
+//     ProgressBar pb( "Semantic Segmentation Analysis" );
+// 
+//     pb.show();
+
+    int fileno = 0;
+
+    std::cerr << "start looping over all files" << std::endl;
+    LOOP_ALL_S( *testFiles )
+    {
+      EACH_INFO( classno, info );
+      std::string file = info.img();
+
+      NICE::Image lm;
+      NICE::MultiChannelImageT<double> probabilities;
+
+      if ( info.hasLocalizationInfo() )
+      {
+        const LocalizationResult *l_gt = info.localization();
+
+        lm.resize( l_gt->xsize, l_gt->ysize );
+        //lm.set( 0 );
+        l_gt->calcLabeledImage( lm, classNames.getBackgroundClass() );
+      }
+
+      semseg->semanticseg( file, lm, probabilities );
+
+      fprintf( stderr, "testSemanticSegmentation: Segmentation finished !\n" );
+
+      //ground truth image, needed for updating the confusion matrix
+      //TODO check whether this is really needed, since we computed such a label image already within SemSegNovelty
+      NICE::Image lm_gt;
+
+      if ( info.hasLocalizationInfo() )
+      {
+        const LocalizationResult *l_gt = info.localization();
+
+        lm_gt.resize( l_gt->xsize, l_gt->ysize );
+        lm_gt.set( 0 );
+
+        fprintf( stderr, "testSemanticSegmentation: Generating Labeled NICE::Image (Ground-Truth)\n" );
+        l_gt->calcLabeledImage( lm_gt, classNames.getBackgroundClass() );
+      }
+// // // 
+// // //       std::string fname = StringTools::baseName( file, false );
+// // // 
+// // //       if ( write_results_pascal )
+// // //       {
+// // // 
+// // //         NICE::Image pascal_lm( lm.width(), lm.height() );
+// // //         int backgroundClass = classNames.getBackgroundClass();
+// // // 
+// // //         for ( int y = 0 ; y < lm.height(); y++ )
+// // //           for ( int x = 0 ; x < lm.width(); x++ )
+// // //           {
+// // //             int v = lm.getPixel( x, y );
+// // // 
+// // //             if ( v == backgroundClass )
+// // //               pascal_lm.setPixel( x, y, 255 );
+// // //             else
+// // //               pascal_lm.setPixel( x, y, 255 - v - 1 );
+// // //           }
+// // // 
+// // //         char filename[1024];
+// // // 
+// // //         char *format = ( char * )"pgm";
+// // //         sprintf( filename, "%s/%s.%s", resultdir.c_str(), fname.c_str(), format );
+// // // 
+// // //         pascal_lm.write( filename );
+// // //       }
+// // // 
+      if ( show_result || write_results )
+      {
+        NICE::ColorImage orig( file );
+        NICE::ColorImage rgb;
+        NICE::ColorImage rgb_gt;
+
+        classNames.labelToRGB( lm, rgb );
+
+        classNames.labelToRGB( lm_gt, rgb_gt );
+
+        if ( write_results )
+        {
+  //         char filename[1024];
+  //         char *format = ( char * )"ppm";
+  //         sprintf( filename, "%06d.%s", fileno, format );
+  //         std::string origfilename = resultdir + "/orig_" + string( filename );
+  //         cerr << "Writing to file " << origfilename << endl;
+  //         orig.write( origfilename );
+  //         rgb.write( resultdir + "/result_" + string( filename ) );
+  //         rgb_gt.write( resultdir + "/groundtruth_" + string( filename ) );
+          
+          std::stringstream out;       
+          std::vector< std::string > myList;
+          StringTools::split ( Globals::getCurrentImgFN (), '/', myList );
+          out << resultdir << "/" << myList.back();
+          cerr << "Writing to file " << resultdir << "/"<< myList.back() << endl;
+          
+          std::string noveltyMethodString = conf.gS( "SemSegNovelty",  "noveltyMethod", "gp-variance");
+          orig.write ( out.str() + "_orig.ppm" );
+          rgb.write ( out.str() + "_" + noveltyMethodString + "_result_run_" + NICE::intToString(iterationCount) + ".ppm" );
+          rgb_gt.write ( out.str() + "_groundtruth.ppm" );
+        }
+
+        if ( show_result )
+        {
+  #ifndef NOVISUAL
+          showImage( rgb, "Result" );
+          showImage( rgb_gt, "Groundtruth" );
+          showImage( orig, "Input" );
+  #endif
+        }
+      }
+
+  //#pragma omp critical
+      updateMatrix( lm, lm_gt, M, forbidden_classes );
+
+      std::cerr << M << std::endl;
+
+      fileno++;
+
+//       pb.update( testFiles->count() );
+    } //Loop over all test images
+
+//     pb.hide();
+
+    //**********************************************
+    //                  EVALUATION 
+    //   COMPUTE CONFUSION MAT AND FINAL SCORES
+    //**********************************************
+    timer.start();
+    
+    long maxMemory;
+    rs.getMaximumMemory(maxMemory);
+    cerr << "Maximum memory used: " << maxMemory << " KB" << endl;
+    
+    double overall = 0.0;
+    double sumall = 0.0;
+
+    for ( int r = 0; r < ( int )M.rows(); r++ )
+    {
+      for ( int c = 0; c < ( int )M.cols(); c++ )
+      {
+        if ( r == c )
+          overall += M( r, c );
+
+        sumall += M( r, c );
+      }
+    }
+
+    overall /= sumall;
+
+    // normalizing M using rows
+
+    for ( int r = 0 ; r < ( int )M.rows() ; r++ )
+    {
+      double sum = 0.0;
+
+      for ( int c = 0 ; c < ( int )M.cols() ; c++ )
+        sum += M( r, c );
+
+      if ( fabs( sum ) > 1e-4 )
+        for ( int c = 0 ; c < ( int )M.cols() ; c++ )
+          M( r, c ) /= sum;
+    }
+
+    std::cerr << M << std::endl;
+
+    double avg_perf = 0.0;
+    int classes_trained = 0;
+
+    for ( int r = 0 ; r < ( int )M.rows() ; r++ )
+    {
+      if (( classNames.existsClassno( r ) ) && ( forbidden_classes.find( r ) == forbidden_classes.end() ) )
+      {
+        avg_perf += M( r, r );
+        double lsum = 0.0;
+        for(int r2 = 0; r2 < ( int )M.rows(); r2++)
+        {
+          lsum += M(r,r2);
+        }
+        if(lsum != 0.0)
+        {
+          classes_trained++;
+        }
+      }
+    }
+
+    if ( write_results )
+    {
+      ofstream fout(( resultdir + "/res.txt" ).c_str(), ios::out );
+      fout <<  "overall: " << overall << endl;
+      fout << "Average Performance " << avg_perf / ( classes_trained ) << endl;
+      fout << "Lower Bound " << 1.0  / classes_trained << endl;
+
+      for ( int r = 0 ; r < ( int )M.rows() ; r++ )
+      {
+        if (( classNames.existsClassno( r ) ) && ( forbidden_classes.find( r ) == forbidden_classes.end() ) )
+        {
+          std::string classname = classNames.text( r );
+          fout << classname.c_str() << ": " << M( r, r ) << endl;
+        }
+      }
+
+      fout.close();
+    }
+
+    fprintf( stderr, "overall: %f\n", overall );
+
+    fprintf( stderr, "Average Performance %f\n", avg_perf / ( classes_trained ) );
+    //fprintf(stderr, "Lower Bound %f\n", 1.0 / classes_trained);
+
+    for ( int r = 0 ; r < ( int )M.rows() ; r++ )
+    {
+      if (( classNames.existsClassno( r ) ) && ( forbidden_classes.find( r ) == forbidden_classes.end() ) )
+      {
+        std::string classname = classNames.text( r );
+        fprintf( stderr, "%s: %f\n", classname.c_str(), M( r, r ) );
+      }
+    }
+    
+    timer.stop();
+    std::cout << "AL time for evaluation: " << timer.getLastAbsolute() << std::endl;
+    
+    //**********************************************
+    //          READ QUERY SCORE IMAGES
+    //   AND SELECT THE REGION TO BE LABELED
+    //**********************************************
+    //NOTE this is not needed anymore, since we store everything within SemSegNovelty
+    //However, it is still needed if we use the NN-classifier for the feature learning approach
+    
+//     string alSection = "SemSegNovelty";
+//     std::string noveltyMethodString = conf.gS( alSection,  "noveltyMethod", "gp-variance");
+//     std::string uncertdir = conf.gS("debug", "resultdir", "result");
+//     int testWSize = conf.gI(alSection, "test_window_size", 10);   
+//     
+//     float maxVal(0);
+//     int maxValX(0);
+//     int maxValY(0);
+//     std::vector<ImageInfo *>::const_iterator maxValInfoIt = testFiles->begin()->second.begin();
+//     
+//     
+//     for(LabeledSet::const_iterator outerIt = testFiles->begin() ; outerIt != testFiles->end() ; outerIt++)
+//     {
+//       for ( std::vector<ImageInfo *>::const_iterator imageIt = outerIt->second.begin(); imageIt != outerIt->second.end(); imageIt++ )    
+//       {
+//         const ImageInfo & (info) = *(*imageIt);
+//         
+//         std::string file = info.img();
+//         
+//         std::stringstream dest;
+//         std::vector< std::string > list2;
+//         StringTools::split ( file, '/', list2 );
+//         dest << uncertdir << "/" << list2.back();      
+//         
+//         FloatImage noveltyImage;
+//         noveltyImage.readRaw(dest.str() + "_run_" +  NICE::intToString(iterationCount) + "_" + noveltyMethodString+".rawfloat");
+//         
+//         int xsize ( noveltyImage.width() );
+//         int ysize ( noveltyImage.height() );
+//         
+//         //compute the GT-image to ensure that we only query "useful" new features, i.e., not query background or similar "forbidden" stuff
+//         NICE::Image lm_gt;
+//         if ( (*maxValInfoIt)->hasLocalizationInfo() )
+//         {
+//           const LocalizationResult *l_gt = (*maxValInfoIt)->localization();
+// 
+//           lm_gt.resize( l_gt->xsize, l_gt->ysize );
+//           lm_gt.set( 0 );
+// 
+//           l_gt->calcLabeledImage( lm_gt, classNames.getBackgroundClass() );
+//         }                
+//         
+//         for ( int y = 0; y < ysize; y += testWSize )
+//         {
+//           for ( int x = 0; x < xsize; x += testWSize)
+//           {
+//             if ( (noveltyImage ( x, y ) > maxVal) && (  forbidden_classesForActiveLearning.find ( lm_gt(x, y) ) == forbidden_classesForActiveLearning.end() ) )
+//             {
+//               maxVal =  noveltyImage ( x, y );
+//               maxValX = x;
+//               maxValY = y;
+//               maxValInfoIt = imageIt;
+//             }
+//           }
+//         }
+//         
+//       }//iterate over inner loop
+//     }//iterate over testFiles
+// 
+//     
+//       std::cerr << "maxVal: " << maxVal << " maxValX: " << maxValX << " maxValY: " << maxValY << " maxValInfo: " << (*maxValInfoIt)->img() << std::endl;
+    
+    //**********************************************
+    //          INCLUDE THE NEW INFORMATION
+    //           AND UPDATE THE CLASSIFIER
+    //**********************************************    
+      
+     timer.start();
+     semseg->addNovelExamples(); 
+     
+     timer.stop();
+     std::cout << "AL time for incremental update: " << timer.getLastAbsolute() << std::endl;
+     //alternatively, we could call the destructor of semseg, and create it again, which does the same thing 
+     // (add new features, save the classifier, re-read it after initialization)
+     //BUT this would not setup the forbidden and known classes properly!!! We should fix that!
+     
+     const Examples * novelExamples = semseg->getNovelExamples(); 
+//      std::cerr << " ==================================== " << std::endl;
+//      std::cerr << "new examples to be added: " << std::endl;
+//      for ( uint i = 0 ; i < novelExamples->size() ; i++ )
+//      {
+//         std::cerr << (*novelExamples)[i].first << " "; (*novelExamples)[i].second.store(std::cerr);
+//      }
+//      std::cerr << " ==================================== " << std::endl;
+     
+    //check which classes will be added using the features from the novel region
+    std::set<int> newClassNumbers;
+    newClassNumbers.clear(); //just to be sure  
+    for ( uint i = 0 ; i < novelExamples->size() ; i++ )
+    {
+      if (newClassNumbers.find( (*novelExamples)[i].first /* classNumber*/) == newClassNumbers.end() )
+      {
+        newClassNumbers.insert( (*novelExamples)[i].first );
+      }
+    }
+
+    //accept the new classes as valid information
+    for (std::set<int>::const_iterator clNoIt = newClassNumbers.begin(); clNoIt != newClassNumbers.end(); clNoIt++)
+    {
+      if ( forbidden_classes.find ( *clNoIt ) != forbidden_classes.end() )
+      {
+        forbidden_classes.erase(*clNoIt);
+      }
+    }       
+      
+    //NOTE Below comes the old version:
+    // it is not needed anymore, since we store everything within SemSegNovelty
+    //However, it is still needed if we use the NN-classifier for the feature learning approach      
+//     //  ----------------------------------------------------
+//     //  therefore, we first recompute the features for the whole image and
+//     //take the one which we desire
+//       
+//     //this is NOT efficient, but a nice and easy first step
+//       
+//     NICE::ColorImage img ( (*maxValInfoIt)->img() );
+//     
+//     MultiChannelImageT<double> feats;
+// 
+//     // extract features
+//     LFColorWeijer * featExtract = new LFColorWeijer ( &conf );
+//     featExtract->getFeats ( img, feats );
+//     int featdim = feats.channels();
+//     feats.addChannel(featdim);
+// 
+//     for (int c = 0; c < featdim; c++)
+//     {
+//       ImageT<double> tmp = feats[c];
+//       ImageT<double> tmp2 = feats[c+featdim];
+// 
+//       NICE::FilterT<double, double, double>::gradientStrength (tmp, tmp2);
+//     }
+//     featdim += featdim;
+// 
+//     // compute integral images
+//     for ( int c = 0; c < featdim; c++ )
+//     {
+//       feats.calcIntegral ( c );
+//     }    
+//     
+//     //  ----------------------------------------------------
+//     //now take the feature
+//     NICE::Vector newFeature(featdim);
+//     for ( int f = 0; f < featdim; f++ )
+//     {
+//       double val = feats.getIntegralValue ( maxValX - testWSize, maxValY - testWSize, maxValX + testWSize, maxValY + testWSize, f );
+//       newFeature[f] = val;
+//     }
+//     newFeature.normalizeL1();    
+//     
+//     NICE::Image lm_gt;
+//     // take the gt class number as well    
+//     if ( (*maxValInfoIt)->hasLocalizationInfo() )
+//     {
+//       const LocalizationResult *l_gt = (*maxValInfoIt)->localization();
+// 
+//       lm_gt.resize( l_gt->xsize, l_gt->ysize );
+//       lm_gt.set( 0 );
+// 
+//       l_gt->calcLabeledImage( lm_gt, classNames.getBackgroundClass() );
+//     }
+//     int classNoGT = lm_gt(maxValX, maxValY);
+//     std::cerr << "class number GT: " << classNoGT << std::endl;
+//     
+//     
+//     semseg->addNewExample(newFeature, classNoGT);
+//     
+//     //accept the new class as valid information
+//     if ( forbidden_classes.find ( classNoGT ) != forbidden_classes.end() )
+//     {
+//       forbidden_classes.erase(classNoGT);
+//     }    
+    
+    std::cerr << "iteration finished - start the next round" << std::endl;
+    
+  } //iterationCount
+
+  delete semseg;
+
+  return 0;
+}

progs/testActiveSemanticSegmentationBinary.cpp

@@ -0,0 +1,280 @@
+// Beispielhafter Aufruf: BUILD_x86_64/progs/testActiveSemanticSegmentationBinary -config <CONFIGFILE>
+
+/**
+* @file testActiveSemanticSegmentationBinary.cpp
+* @brief test semantic segmentation routines with actively selecting regions for labeling
+* @author Alexander Freytag
+* @date 27-02-2013
+*/
+
+#ifdef NICE_USELIB_OPENMP
+#include <omp.h>
+#endif
+
+#include "core/basics/Config.h"
+#include "core/basics/StringTools.h"
+#include <vislearning/baselib/ICETools.h>
+
+#include <semseg/semseg/SemanticSegmentation.h>
+#include <semseg/semseg/SemSegLocal.h>
+#include <semseg/semseg/SemSegCsurka.h>
+#include <semseg/semseg/SemSegNovelty.h>
+#include <semseg/semseg/SemSegNoveltyBinary.h>
+#include <semseg/semseg/SemSegContextTree.h>
+
+#include "core/image/FilterT.h"
+
+#include <core/basics/ResourceStatistics.h>
+
+#include <fstream>
+
+using namespace OBJREC;
+
+using namespace NICE;
+
+using namespace std;
+
+/**
+ test semantic segmentation routines
+*/
+int main( int argc, char **argv )
+{
+  std::set_terminate( __gnu_cxx::__verbose_terminate_handler );
+
+  Config conf( argc, argv );
+  
+  ResourceStatistics rs;
+  
+  bool show_result = conf.gB( "debug", "show_results", false );
+
+  bool write_results = conf.gB( "debug", "write_results", false );
+
+  bool write_results_pascal = conf.gB( "debug", "write_results_pascal", false );
+
+  std::string resultdir = conf.gS( "debug", "resultdir", "." );
+  
+  //how often do we want to iterate between sem-seg and active query?
+  int activeIterations = conf.gI("main", "activeIterations", 1 );
+    
+  if ( write_results )
+  {
+    cerr << "Writing Results to " << resultdir << endl;
+  }
+
+  MultiDataset md( &conf );
+
+  const ClassNames & classNames = md.getClassNames( "train" );
+
+  string method = conf.gS( "main", "method", "SSCsurka" );
+
+  //currently, we only allow SemSegNoveltyBinary, because it implements addNovelExamples()
+  SemSegNoveltyBinary *semseg = NULL;
+  
+  Timer timer;
+  timer.start();
+
+  semseg = new SemSegNoveltyBinary( &conf, &md );
+
+  timer.stop();
+  
+  std::cerr << "AL time for training: " << timer.getLast() << std::endl;
+
+  const LabeledSet *testFiles = md["test"];
+
+  std::set<int> forbidden_classes;
+  std::string forbidden_classes_s = conf.gS( "analysis", "forbidden_classesTrain", "" );
+  classNames.getSelection( forbidden_classes_s, forbidden_classes );
+  
+  std::set<int> forbidden_classesForActiveLearning;
+  std::string forbidden_classesForActiveLearning_s = conf.gS( "analysis", "forbidden_classesForActiveLearning", "" );
+  classNames.getSelection( forbidden_classesForActiveLearning_s, forbidden_classesForActiveLearning );
+  
+  
+  int positiveClass;
+  
+  //check whether we have a single positive class
+  std::string positiveClass_s = conf.gS ( "SemSegNoveltyBinary", "positiveClass", "" );
+  std::set<int> positiveClassNumberTmp;
+  classNames.getSelection ( positiveClass_s, positiveClassNumberTmp );  
+
+  switch ( positiveClassNumberTmp.size() )
+  {
+    case 0:
+    {
+      positiveClass = 0;
+//       std::cerr << "no positive class given, assume 0 as positive class" << std::endl;
+      break;
+    }
+    case 1:
+    {
+      positiveClass = *(positiveClassNumberTmp.begin());
+//       std::cerr << "positive class will be number" << positiveClass << " with the name: " << positiveClass_s << std::endl;
+      break;
+    }
+    default:
+    {
+      //we specified more than a single positive class. right now, this is not what we are interested in, but 
+      //in theory we could also accept this and convert positiveClass into a set of ints of possible positive classes
+      positiveClass = 0;
+//       std::cerr << "no positive class given, assume 0 as positive class" << std::endl;
+      break;
+    }
+  }  
+ 
+  
+  std::cerr << "number of AL iterations: " << activeIterations << std::endl;
+  for (int iterationCount = 0; iterationCount < activeIterations; iterationCount++)
+  {    
+    std::cerr << "SemSeg AL Iteration: " << iterationCount << std::endl;
+    semseg->setIterationCountSuffix(iterationCount);
+
+    int fileno = 0;
+
+    std::cerr << "start looping over all files" << std::endl;
+    LOOP_ALL_S( *testFiles )
+    {
+      EACH_INFO( classno, info );
+      std::string file = info.img();
+
+      NICE::Image lm;
+      NICE::MultiChannelImageT<double> probabilities;
+
+      if ( info.hasLocalizationInfo() )
+      {
+        const LocalizationResult *l_gt = info.localization();
+
+        lm.resize( l_gt->xsize, l_gt->ysize );
+        //lm.set( 0 );
+        l_gt->calcLabeledImage( lm, classNames.getBackgroundClass() );
+      }
+
+      ((SemanticSegmentation*)semseg)->semanticseg( file, lm, probabilities );
+
+      fprintf( stderr, "testSemanticSegmentation: Segmentation finished !\n" );
+
+      //ground truth image, needed for updating the confusion matrix
+      //TODO check whether this is really needed, since we computed such a label image already within SemSegNovelty
+      NICE::Image lm_gt;
+
+      if ( info.hasLocalizationInfo() )
+      {
+        const LocalizationResult *l_gt = info.localization();
+
+        lm_gt.resize( l_gt->xsize, l_gt->ysize );
+        lm_gt.set( 0 );
+
+        fprintf( stderr, "testSemanticSegmentation: Generating Labeled NICE::Image (Ground-Truth)\n" );
+        l_gt->calcLabeledImage( lm_gt, classNames.getBackgroundClass() );
+      }
+
+      if ( show_result || write_results )
+      {
+        NICE::ColorImage orig( file );
+        NICE::ColorImage rgb;
+        NICE::ColorImage rgb_gt;
+
+        classNames.labelToRGB( lm, rgb );
+
+        classNames.labelToRGB( lm_gt, rgb_gt );
+
+        if ( write_results )
+        {
+          
+          std::stringstream out;       
+          std::vector< std::string > myList;
+          StringTools::split ( Globals::getCurrentImgFN (), '/', myList );
+          out << resultdir << "/" << myList.back();
+          cerr << "Writing to file " << resultdir << "/"<< myList.back() << endl;
+          
+          std::string noveltyMethodString = conf.gS( "SemSegNoveltyBinary",  "noveltyMethod", "gp-variance");
+          orig.write ( out.str() + "_orig.ppm" );
+          rgb.write ( out.str() + "_" + noveltyMethodString + "_result_run_" + NICE::intToString(iterationCount) + ".ppm" );
+          rgb_gt.write ( out.str() + "_groundtruth.ppm" );
+        }
+
+        if ( show_result )
+        {
+  #ifndef NOVISUAL
+          showImage( rgb, "Result" );
+          showImage( rgb_gt, "Groundtruth" );
+          showImage( orig, "Input" );
+  #endif
+        }
+      }
+
+
+      fileno++;
+
+    } //Loop over all test images
+
+
+    //**********************************************
+    //                  EVALUATION 
+    //   COMPUTE CONFUSION MAT AND FINAL SCORES
+    //**********************************************
+    timer.start();
+    
+    double score = semseg->getAUCPerformance();
+    std::cerr << "auc scores of run : " << iterationCount << " : " << score << std::endl;
+    
+    long maxMemory;
+    rs.getMaximumMemory(maxMemory);
+    cerr << "Maximum memory used: " << maxMemory << " KB" << endl;
+
+    
+    timer.stop();
+    std::cout << "AL time for evaluation: " << timer.getLastAbsolute() << std::endl;
+    
+
+    
+    //**********************************************
+    //          INCLUDE THE NEW INFORMATION
+    //           AND UPDATE THE CLASSIFIER
+    //**********************************************    
+      
+     timer.start();
+     semseg->addNovelExamples(); 
+     
+     timer.stop();
+     std::cout << "AL time for incremental update: " << timer.getLastAbsolute() << std::endl;
+     //alternatively, we could call the destructor of semseg, and create it again, which does the same thing 
+     // (add new features, save the classifier, re-read it after initialization)
+     //BUT this would not setup the forbidden and known classes properly!!! We should fix that!
+     
+     const Examples * novelExamples = semseg->getNovelExamples(); 
+//      std::cerr << " ==================================== " << std::endl;
+//      std::cerr << "new examples to be added: " << std::endl;
+//      for ( uint i = 0 ; i < novelExamples->size() ; i++ )
+//      {
+//         std::cerr << (*novelExamples)[i].first << " "; (*novelExamples)[i].second.store(std::cerr);
+//      }
+//      std::cerr << " ==================================== " << std::endl;
+     
+    //check which classes will be added using the features from the novel region
+    std::set<int> newClassNumbers;
+    newClassNumbers.clear(); //just to be sure  
+    for ( uint i = 0 ; i < novelExamples->size() ; i++ )
+    {
+      if (newClassNumbers.find( (*novelExamples)[i].first /* classNumber*/) == newClassNumbers.end() )
+      {
+        newClassNumbers.insert( (*novelExamples)[i].first );
+      }
+    }
+
+    //accept the new classes as valid information
+    for (std::set<int>::const_iterator clNoIt = newClassNumbers.begin(); clNoIt != newClassNumbers.end(); clNoIt++)
+    {
+      if ( forbidden_classes.find ( *clNoIt ) != forbidden_classes.end() )
+      {
+        forbidden_classes.erase(*clNoIt);
+      }
+    }       
+
+    std::cerr << "iteration finished - start the next round" << std::endl;
+    
+  } //iterationCount
+
+  delete semseg;
+
+  return 0;
+}

semseg/SemSegNovelty.cpp

@@ -3,13 +3,16 @@
 
 #include "SemSegNovelty.h"
 
-#include "core/image/FilterT.h"
-#include "gp-hik-exp/GPHIKClassifierNICE.h"
-#include "vislearning/baselib/ICETools.h"
-#include "vislearning/baselib/Globals.h"
-#include "vislearning/features/fpfeatures/SparseVectorFeature.h"
-#include "core/basics/StringTools.h"
-#include "core/basics/Timer.h"
+#include <core/image/FilterT.h>
+#include <core/basics/numerictools.h>
+#include <core/basics/StringTools.h>
+#include <core/basics/Timer.h>
+
+#include <gp-hik-exp/GPHIKClassifierNICE.h>
+#include <vislearning/baselib/ICETools.h>
+#include <vislearning/baselib/Globals.h>
+#include <vislearning/features/fpfeatures/SparseVectorFeature.h>
+
 #include "segmentation/GenericRegionSegmentationMethodSelection.h"
 
 using namespace std;
@@ -28,14 +31,31 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
 
   featExtract = new LFColorWeijer ( conf );
 
-  save_cache = conf->gB ( "FPCPixel", "save_cache", true );
-  read_cache = conf->gB ( "FPCPixel", "read_cache", false );
-//   uncertdir = conf->gS("debug", "uncertainty", "uncertainty");
+  this->reuseSegmentation = conf->gB ( "FPCPixel", "reuseSegmentation", true ); //save and read segmentation results from files
+  this->save_classifier = conf->gB ( "FPCPixel", "save_classifier", true ); //save the classifier to a file
+  this->read_classifier = conf->gB ( "FPCPixel", "read_classifier", false ); //read the classifier from a file
+
   //write uncertainty results in the same folder as done for the segmentation results
-  uncertdir = conf->gS("debug", "resultdir", "result");
+  resultdir = conf->gS("debug", "resultdir", "result");
   cache = conf->gS ( "cache", "root", "" );
   
-  classifier = new GPHIKClassifierNICE ( conf, "ClassiferGPHIK" );;
+  
+  //stupid work around of the const attribute
+  Config confCopy = *conf;
+  
+  //just to make sure, that we do NOT perform an optimization after every iteration step
+  //this would just take a lot of time, which is not desired so far
+  confCopy.sB("ClassifierGPHIK","performOptimizationAfterIncrement",false);
+  
+  classifierString = conf->gS ( section, "classifier", "ClassifierGPHIK" );  
+  classifier = NULL;
+  vclassifier = NULL;
+  if ( classifierString.compare("ClassifierGPHIK") == 0)
+    classifier = new GPHIKClassifierNICE ( &confCopy, "ClassifierGPHIK" );
+  else
+    vclassifier = GenericClassifierSelection::selectVecClassifier ( conf, classifierString );
+  
+
 
   findMaximumUncert = conf->gB(section, "findMaximumUncert", true);
   whs = conf->gI ( section, "window_size", 10 );
@@ -53,7 +73,7 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   else
   {
     RegionSegmentationMethod *tmpRegionSeg = GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(conf, rsMethode);    
-    if ( save_cache )
+    if ( reuseSegmentation )
       regionSeg = new RSCache ( conf, tmpRegionSeg );
     else
       regionSeg = tmpRegionSeg;
@@ -61,17 +81,23 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   
   cn = md->getClassNames ( "train" );
 
-  if ( read_cache )
+  if ( read_classifier )
   {
-    string classifierdst = "/classifier.data";
-    fprintf ( stderr, "SemSegNovelty:: Reading classifier data from %s\n", ( cache + classifierdst ).c_str() );
-
     try
     {
       if ( classifier != NULL )
       {
+        string classifierdst = "/classifier.data";        
+        fprintf ( stderr, "SemSegNovelty:: Reading classifier data from %s\n", ( cache + classifierdst ).c_str() );        
         classifier->read ( cache + classifierdst );
       }
+      else
+      {
+        string classifierdst = "/veccl.data";        
+        fprintf ( stderr, "SemSegNovelty:: Reading classifier data from %s\n", ( cache + classifierdst ).c_str() );          
+        vclassifier->read ( cache + classifierdst );      
+      }
+      
 
       fprintf ( stderr, "SemSegNovelty:: successfully read\n" );
     }
@@ -90,56 +116,84 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   if (noveltyMethodString.compare("gp-variance") == 0)  // novel = large variance
   {
     this->noveltyMethod = GPVARIANCE;
+    this->mostNoveltyWithMaxScores = true;
   }
   else if (noveltyMethodString.compare("gp-uncertainty") == 0) //novel = large uncertainty (mean / var)
   {
     this->noveltyMethod = GPUNCERTAINTY;
+    this->mostNoveltyWithMaxScores = false;
+    globalMaxUncert = numeric_limits<double>::max();
   } 
   else if (noveltyMethodString.compare("gp-mean") == 0) //novel = small mean
   {
     this->noveltyMethod = GPMINMEAN;
+    this->mostNoveltyWithMaxScores = false;
+    globalMaxUncert = numeric_limits<double>::max();
   }
   else if (noveltyMethodString.compare("gp-meanRatio") == 0)  //novel = small difference between mean of most plausible class and mean of snd
                                                               //        most plausible class (not useful in binary settings)
   {
     this->noveltyMethod = GPMEANRATIO;
+    this->mostNoveltyWithMaxScores = false;
+    globalMaxUncert = numeric_limits<double>::max();
   }
   else if (noveltyMethodString.compare("gp-weightAll") == 0) // novel = large weight in alpha vector after updating the model (can be predicted exactly)
   {
     this->noveltyMethod = GPWEIGHTALL;
+    this->mostNoveltyWithMaxScores = true;
   }
   else if (noveltyMethodString.compare("gp-weightRatio") == 0) // novel = small difference between weights for alpha vectors 
                                                                //     with assumptions of GT label to be the most 
                                                                //     plausible against the second most plausible class   
   {
     this->noveltyMethod = GPWEIGHTRATIO;
-  } 
+    this->mostNoveltyWithMaxScores = false;
+    globalMaxUncert = numeric_limits<double>::max();
+  }
+  else if (noveltyMethodString.compare("random") == 0) 
+  {
+     initRand(); 
+     this->noveltyMethod = RANDOM;
+  }
   else
   {
     this->noveltyMethod = GPVARIANCE;
+    this->mostNoveltyWithMaxScores = true;
   }
+  
+  //we don't have queried any region so far
+  queriedRegions.clear();
+  visualizeALimages = conf->gB(section, "visualizeALimages", false);
 }
 
 SemSegNovelty::~SemSegNovelty()
 {
   if(newTrainExamples.size() > 0)
   {
-    // most uncertain region
-    showImage(maskedImg);
-    //classifier->add(newTrainExamples)
+    // show most uncertain region
+    if (visualizeALimages)
+      showImage(maskedImg);
+    
+    //incorporate new information into the classifier
+    if (classifier != NULL)
+      classifier->addMultipleExamples(newTrainExamples);
+    
+    //store the classifier, such that we can read it again in the next round (if we like that)
     classifier->save ( cache + "/classifier.data" );
   }
   
   // clean-up
   if ( classifier != NULL )
     delete classifier;
+  if ( vclassifier != NULL )
+    delete vclassifier;
   if ( featExtract != NULL )
     delete featExtract;
 }
 
 void SemSegNovelty::visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage)
 {
-  vector<uchar> color;
+  std::vector<uchar> color;
   color.push_back(255);
   color.push_back(0);
   color.push_back(0);
@@ -179,14 +233,7 @@ void SemSegNovelty::train ( const MultiDataset *md )
   ////////////////////////
   // feature extraction //
   ////////////////////////
-
-  std::string forbidden_classes_s = conf->gS ( "analysis", "donttrain", "" );
-  if ( forbidden_classes_s == "" )
-  {
-    forbidden_classes_s = conf->gS ( "analysis", "forbidden_classes", "" );
-  }
-  cn.getSelection ( forbidden_classes_s, forbidden_classes );
-  
+ 
   //check the same thing for the training classes - this is very specific to our setup 
   std::string forbidden_classesTrain_s = conf->gS ( "analysis", "donttrainTrain", "" );
   if ( forbidden_classesTrain_s == "" )
@@ -194,7 +241,7 @@ void SemSegNovelty::train ( const MultiDataset *md )
     forbidden_classesTrain_s = conf->gS ( "analysis", "forbidden_classesTrain", "" );
   }
   cn.getSelection ( forbidden_classesTrain_s, forbidden_classesTrain );
-
+  
 
   ProgressBar pb ( "Local Feature Extraction" );
   pb.show();
@@ -264,7 +311,7 @@ void SemSegNovelty::train ( const MultiDataset *md )
       feats.calcIntegral ( c );
     }
 
-    for ( int y = 0; y < ysize; y += trainWsize )
+    for ( int y = 0; y < ysize; y += trainWsize)
     {
       for ( int x = 0; x < xsize; x += trainWsize )
       {
@@ -295,8 +342,12 @@ void SemSegNovelty::train ( const MultiDataset *md )
 
         example.position = imgnb;
         examples.push_back ( pair<int, Example> ( classnoTmp, example ) );
+
       }
     }
+ 
+    
+    
 
     delete ce;
     imgnb++;
@@ -327,19 +378,39 @@ void SemSegNovelty::train ( const MultiDataset *md )
   delete f;
 
   if ( classifier != NULL )
+  {
+    std::cerr << "train FP-classifier with " << examples.size() << " examples" << std::endl;
     classifier->train ( fp, examples );
+    std::cerr << "training finished" << std::endl;
+  }
   else
   {
-    cerr << "no classifier selected?!" << endl;
-    exit ( -1 );
-  }
+    LabeledSetVector lvec;
+    convertExamplesToLSet ( examples, lvec );
+    vclassifier->teach ( lvec );
+//     if ( usegmm )
+//       convertLSetToSparseExamples ( examples, lvec );
+//     else
+    std::cerr << "classifierString: " << classifierString << std::endl;
+    if (this->classifierString.compare("nn") == 0)
+    {
+      convertLSetToExamples ( examples, lvec, true /* only remove pointers to the data in the LSet-struct*/);
+    }
+    else
+    {
+      convertLSetToExamples ( examples, lvec, false /* remove all training examples of the LSet-struct */);
+    }
+    vclassifier->finishTeaching();
+  }  
 
   fp.destroy();
 
-  if ( save_cache )
+  if ( save_classifier )
   {
     if ( classifier != NULL )
       classifier->save ( cache + "/classifier.data" );
+    else
+      vclassifier->save ( cache + "/veccl.data" );    
   }
 
   ////////////
@@ -354,12 +425,16 @@ void SemSegNovelty::train ( const MultiDataset *md )
   cerr << "SemSeg training finished" << endl;
 }
 
+
 void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NICE::MultiChannelImageT<double> & probabilities )
-{
+{  
   Timer timer;
   timer.start();
   
+  //segResult contains the GT labels when this method is called
+  // we simply store them in labels, to have an easy access to the GT information lateron
   Image labels = segresult;
+  //just to be sure that we do not have a GT-biased result :)
   segresult.set(0);
 
   int featdim = -1;
@@ -402,16 +477,24 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   {
     feats.calcIntegral ( c );
   }
-
-  FloatImage noveltyImage ( xsize, ysize );
-  noveltyImage.set ( 0.0 );
   
   timer.stop();
-  cout << "first: " << timer.getLastAbsolute() << endl;
+  std::cout << "AL time for preparation: " << timer.getLastAbsolute() << std::endl;
     
   timer.start();
-  this->computeClassificationResults( feats, segresult, probabilities, xsize, ysize, featdim);
-  timer.stop();
+  //classification results currently only needed to be computed separately if we use the vclassifier, i.e., the nearest neighbor used 
+  // for the "novel feature learning" approach
+  //in all other settings, such as active sem seg in general, we do this within the novelty-computation-methods
+  if ( classifier == NULL )
+  {
+    this->computeClassificationResults( feats, segresult, probabilities, xsize, ysize, featdim);
+  }
+//   timer.stop();
+//   
+//   std::cerr << "classification results computed" << std::endl;
+  
+  FloatImage noveltyImage ( xsize, ysize );
+  noveltyImage.set ( 0.0 );  
   
   switch (noveltyMethod)
   {
@@ -427,6 +510,7 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
     }
     case GPMINMEAN:
     {
+         std::cerr << "compute novelty using the minimum mean" << std::endl;
          this->computeNoveltyByGPMean( noveltyImage, feats, segresult, probabilities, xsize, ysize,  featdim );
          break;         
     }
@@ -445,6 +529,11 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
          this->computeNoveltyByGPWeightRatio( noveltyImage, feats, segresult, probabilities, xsize, ysize,  featdim );
          break;         
     }    
+    case RANDOM:
+    {
+         this->computeNoveltyByRandom( noveltyImage, feats, segresult, probabilities, xsize, ysize,  featdim );
+         break;               
+    }
     default:
     {
          //do nothing, keep the image constant to 0.0
@@ -453,10 +542,18 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
          
   }
   
-
-
-  //       std::cerr << "uncertainty: " << gpUncertaintyVal << " minMean: " << gpMeanVal << " gpMeanRatio: " << gpMeanRatioVal << " weightAll: " << gpWeightAllVal << " weightRatio: "<< gpWeightRatioVal << std::endl;
+  timer.stop();
+  std::cout << "AL time for novelty score computation: " << timer.getLastAbsolute() << std::endl;
   
+  if (visualizeALimages)
+  {
+      ColorImage imgrgbTmp (xsize, ysize);
+      ICETools::convertToRGB ( noveltyImage, imgrgbTmp );
+      showImage(imgrgbTmp, "Novelty Image without Region Segmentation");  
+  }
+  
+    
+  timer.start();
   
   //Regionen ermitteln
   if(regionSeg != NULL)
@@ -465,13 +562,14 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
     int amountRegions = regionSeg->segRegions ( img, mask );
     
     //compute probs per region
-    vector<vector<double> > regionProb(amountRegions,vector<double>(probabilities.channels(),0.0));
-    vector<double> regionNoveltyMeasure (amountRegions, 0.0);
+    std::vector<std::vector<double> > regionProb(amountRegions, std::vector<double>(probabilities.channels(),0.0));
+    std::vector<double> regionNoveltyMeasure (amountRegions, 0.0);
 
-    vector<int> regionCounter(amountRegions, 0);
-    for ( int y = 0; y < ysize; y++)
+    std::vector<int> regionCounter(amountRegions, 0);
+    std::vector<int> regionCounterNovelty(amountRegions, 0);
+    for ( int y = 0; y < ysize; y += trainWsize) //y++)
     {
-      for (int x = 0; x < xsize; x++)
+      for (int x = 0; x < xsize; x += trainWsize) //x++)
       {
         int r = mask(x,y);
         regionCounter[r]++;
@@ -480,56 +578,98 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
           regionProb[r][j] += probabilities ( x, y, j );
         }
         
-        //count the amount of "novelty" for the corresponding region
-        regionNoveltyMeasure[r] += noveltyImage(x,y);       
+        if ( forbidden_classesActiveLearning.find( labels(x,y) ) == forbidden_classesActiveLearning.end() )
+        {
+          //count the amount of "novelty" for the corresponding region
+          regionNoveltyMeasure[r] += noveltyImage(x,y);
+          regionCounterNovelty[r]++;
+        }
       }
     }
        
     //find best class per region
-    vector<int> bestClassPerRegion(amountRegions,0);
+    std::vector<int> bestClassPerRegion(amountRegions,0);
     
     double maxNoveltyScore = -numeric_limits<double>::max();
+    if (!mostNoveltyWithMaxScores)
+    {
+      maxNoveltyScore = numeric_limits<double>::max();
+    }   
+    
     int maxUncertRegion = -1;
     
+    //loop over all regions and compute averaged novelty scores
     for(int r = 0; r < amountRegions; r++)
     {
+      
+      //check for the most plausible class per region
       double maxval = -numeric_limits<double>::max();
+      
+      //loop over all classes
       for(int c = 0; c < probabilities.channels(); c++)
       {
         regionProb[r][c] /= regionCounter[r];
-        if(maxval < regionProb[r][c] && regionProb[r][c] != 0.0)
-        {
-          maxval = regionProb[r][c];
-          bestClassPerRegion[r] = c;
+        
+        if(  (maxval < regionProb[r][c]) ) //&& (regionProb[r][c] != 0.0) ) 
+        {        
+              maxval = regionProb[r][c];
+              bestClassPerRegion[r] = c;
         }
       }
-      //normalize summed novelty scores to region size
-      regionNoveltyMeasure[r] /= regionCounter[r];
-      //    
+       
+      //if the region only contains unvalid information (e.g., background) skip it
+      if (regionCounterNovelty[r] == 0)
+      {
+        continue;
+      }
       
-      if(maxNoveltyScore < regionNoveltyMeasure[r])
+      //normalize summed novelty scores to region size
+      regionNoveltyMeasure[r] /= regionCounterNovelty[r];
+    
+      //did we find a region that has a higher score as the most novel region known so far within this image?
+      if(   (  mostNoveltyWithMaxScores && (maxNoveltyScore < regionNoveltyMeasure[r]) )    // if we look for large novelty scores, e.g., variance
+        || ( !mostNoveltyWithMaxScores && (maxNoveltyScore > regionNoveltyMeasure[r]) ) )  // if we look for small novelty scores, e.g., min mean
       {
-        maxNoveltyScore = regionNoveltyMeasure[r];
-        maxUncertRegion = r;
+                   //did we already query a region of this image?                --   and it was this specific region
+        if ( (queriedRegions.find( currentFile ) != queriedRegions.end() ) && ( queriedRegions[currentFile].find(r) != queriedRegions[currentFile].end() ) )
+        {
+          continue;
+        }
+        else //only accept the region as novel if we never queried it before
+        {
+          maxNoveltyScore = regionNoveltyMeasure[r];
+          maxUncertRegion = r;        
+        }
+
       }
+
     }
     
+    // after finding the most novel region for the current image, check whether this region is also the most novel with respect
+    // to all previously seen test images
+    // if so, store the corresponding features, since we want to "actively" query them to incorporate useful information
     if(findMaximumUncert)
     {
-      if(maxNoveltyScore > globalMaxUncert)
+      if(    (   mostNoveltyWithMaxScores && (maxNoveltyScore > globalMaxUncert) )
+          || (  !mostNoveltyWithMaxScores && (maxNoveltyScore < globalMaxUncert) ) )
       {
-        //save new important features
+        //current most novel region of the image has "higher" novelty score then previous most novel region of all test images worked on so far
+        // -> save new important features of this region
         Examples examples;
-        for ( int y = 0; y < ysize; y += testWSize )
+        for ( int y = 0; y < ysize; y += trainWsize )
         {
-          for ( int x = 0; x < xsize; x += testWSize)
+          for ( int x = 0; x < xsize; x += trainWsize)
           {
             if(mask(x,y) == maxUncertRegion)
             {
+              int classnoTmp = labels(x,y);
+              if ( forbidden_classesActiveLearning.find(classnoTmp) != forbidden_classesActiveLearning.end() )
+                continue;
+              
               Example example;
               example.vec = NULL;
               example.svec = new SparseVector ( featdim );
-              int classnoTmp = labels(x,y);
+              
               for ( int f = 0; f < featdim; f++ )
               {
                 double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
@@ -544,15 +684,27 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
         
         if(examples.size() > 0)
         {
+          std::cerr << "found " << examples.size() << " new examples in the queried region" << std::endl << std::endl;
           newTrainExamples.clear();
           newTrainExamples = examples;
           globalMaxUncert = maxNoveltyScore;
-          visualizeRegion(img,mask,maxUncertRegion,maskedImg);
+          //prepare for later visualization
+//           if (visualizeALimages)
+            visualizeRegion(img,mask,maxUncertRegion,maskedImg);
         }
+        else
+        {
+          std::cerr << "the queried region has no valid information" << std::endl << std::endl;
+        }
+        
+        //save filename and region index
+        currentRegionToQuery.first = currentFile;
+        currentRegionToQuery.second = maxUncertRegion;
       }
     }
 
     //write back best results per region
+    //i.e., write normalized novelty scores for every region into the novelty image
     for ( int y = 0; y < ysize; y++)
     {
       for (int x = 0; x < xsize; x++)
@@ -567,10 +719,13 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
         noveltyImage(x,y) = regionNoveltyMeasure[r];
       }
     }
-  }
-
+  } // if regionSeg != null
+  
   timer.stop();
-  cout << "second: " << timer.getLastAbsolute() << endl;
+  std::cout << "AL time for determination of novel regions: " << timer.getLastAbsolute() << std::endl;
+
+//   timer.stop();
+//   cout << "second: " << timer.getLastAbsolute() << endl;
   timer.start();
 
   ColorImage imgrgb ( xsize, ysize );
@@ -578,13 +733,18 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   std::stringstream out;
   std::vector< std::string > list2;
   StringTools::split ( Globals::getCurrentImgFN (), '/', list2 );
-  out << uncertdir << "/" << list2.back();
+  out << resultdir << "/" << list2.back();
   
-  //TODO append a suffix according to the novelty strategie chosen
-  noveltyImage.writeRaw(out.str() + "_" + noveltyMethodString+".rawfloat");
+  noveltyImage.writeRaw(out.str() + "_run_" +  NICE::intToString(this->iterationCountSuffix) + "_" + noveltyMethodString+".rawfloat");
+  
+  if (visualizeALimages)
+  {
+    ICETools::convertToRGB ( noveltyImage, imgrgb );
+    showImage(imgrgb, "Novelty Image");
+  }
 
   timer.stop();
-  cout << "last: " << timer.getLastAbsolute() << endl;
+  cout << "AL time for writing the raw novelty image: " << timer.getLastAbsolute() << endl;
 }
 
 inline void SemSegNovelty::computeClassificationResults( const NICE::MultiChannelImageT<double> & feats, 
@@ -595,7 +755,99 @@ inline void SemSegNovelty::computeClassificationResults( const NICE::MultiChanne
                                                    const int & featdim
                                                        )
 {
-  #pragma omp parallel for
+  std::cerr << "featdim: " << featdim << std::endl;
+  
+  if ( classifier != NULL )
+  {  
+  
+    #pragma omp parallel for
+    for ( int y = 0; y < ysize; y += testWSize )
+    {
+      Example example;
+      example.vec = NULL;
+      example.svec = new SparseVector ( featdim );
+      for ( int x = 0; x < xsize; x += testWSize)
+      {
+        for ( int f = 0; f < featdim; f++ )
+        {
+          double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
+          if ( val > 1e-10 )
+            ( *example.svec ) [f] = val;
+        }
+        example.svec->normalize();
+
+        ClassificationResult cr = classifier->classify ( example );
+
+        int xs = std::max(0, x - testWSize/2);
+        int xe = std::min(xsize - 1, x + testWSize/2);
+        int ys = std::max(0, y - testWSize/2);
+        int ye = std::min(ysize - 1, y + testWSize/2);
+        for (int yl = ys; yl <= ye; yl++)
+        {
+          for (int xl = xs; xl <= xe; xl++)
+          {
+            for ( int j = 0 ; j < cr.scores.size(); j++ )
+            {
+              probabilities ( xl, yl, j ) = cr.scores[j];
+            }
+            segresult ( xl, yl ) = cr.classno;
+          }
+        }
+        
+        example.svec->clear();
+      }
+      delete example.svec;
+      example.svec = NULL;
+    }
+  }
+  else //vclassifier
+  {
+    std::cerr << "compute classification results with vclassifier" << std::endl;
+    #pragma omp parallel for
+    for ( int y = 0; y < ysize; y += testWSize )
+    {
+      for ( int x = 0; x < xsize; x += testWSize)
+      {
+        NICE::Vector v(featdim);
+        for ( int f = 0; f < featdim; f++ )
+        {
+          double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
+          v[f] = val;
+        }
+        v.normalizeL1();
+
+        ClassificationResult cr = vclassifier->classify ( v );
+
+        int xs = std::max(0, x - testWSize/2);
+        int xe = std::min(xsize - 1, x + testWSize/2);
+        int ys = std::max(0, y - testWSize/2);
+        int ye = std::min(ysize - 1, y + testWSize/2);
+        for (int yl = ys; yl <= ye; yl++)
+        {
+          for (int xl = xs; xl <= xe; xl++)
+          {
+            for ( int j = 0 ; j < cr.scores.size(); j++ )
+            {
+              probabilities ( xl, yl, j ) = cr.scores[j];
+            }
+            segresult ( xl, yl ) = cr.classno;
+          }
+        }
+      }
+    }    
+
+  }
+}
+
+// compute novelty images depending on the strategy chosen
+
+void SemSegNovelty::computeNoveltyByRandom(         NICE::FloatImage & noveltyImage, 
+                                              const NICE::MultiChannelImageT<double> & feats,  
+                                                    NICE::Image & segresult,
+                                                    NICE::MultiChannelImageT<double> & probabilities,
+                                             const int & xsize, const int & ysize, const int & featdim  )
+{
+#pragma omp parallel for
   for ( int y = 0; y < ysize; y += testWSize )
   {
     Example example;
@@ -612,11 +864,14 @@ inline void SemSegNovelty::computeClassificationResults( const NICE::MultiChanne
       example.svec->normalize();
 
       ClassificationResult cr = classifier->classify ( example );
-
+  
       int xs = std::max(0, x - testWSize/2);
       int xe = std::min(xsize - 1, x + testWSize/2);
       int ys = std::max(0, y - testWSize/2);
       int ye = std::min(ysize - 1, y + testWSize/2);
+      
+      double randVal = randDouble();
+
       for (int yl = ys; yl <= ye; yl++)
       {
         for (int xl = xs; xl <= xe; xl++)
@@ -625,18 +880,14 @@ inline void SemSegNovelty::computeClassificationResults( const NICE::MultiChanne
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
+          segresult ( xl, yl ) = cr.classno;          
+          noveltyImage ( xl, yl ) = randVal; 
         }
-      }
-      
-      example.svec->clear();
+      }     
     }
-    delete example.svec;
-    example.svec = NULL;
-  }
+  }  
 }
 
-// compute novelty images depending on the strategy chosen
 
 void SemSegNovelty::computeNoveltyByVariance(       NICE::FloatImage & noveltyImage, 
                                               const NICE::MultiChannelImageT<double> & feats,  
@@ -674,8 +925,8 @@ void SemSegNovelty::computeNoveltyByVariance(       NICE::FloatImage & noveltyIm
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
-          noveltyImage ( xl, yl ) = cr.uncertainty;  
+          segresult ( xl, yl ) = cr.classno;          
+          noveltyImage ( xl, yl ) = cr.uncertainty; 
         }
       }          
       
@@ -742,13 +993,12 @@ void SemSegNovelty::computeNoveltyByGPUncertainty(  NICE::FloatImage & noveltyIm
       for (int yl = ys; yl <= ye; yl++)
       {
         for (int xl = xs; xl <= xe; xl++)
-        {
+        {         
           for ( int j = 0 ; j < cr.scores.size(); j++ )
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
-          
+          segresult ( xl, yl ) = cr.classno;          
           noveltyImage ( xl, yl ) = gpUncertaintyVal;  
         }
       }   
@@ -767,7 +1017,7 @@ void SemSegNovelty::computeNoveltyByGPMean(  NICE::FloatImage & noveltyImage,
                                              const int & xsize, const int & ysize, const int & featdim )
 {
   double gpNoise =  conf->gD("GPHIK", "noise", 0.01);  
-  
+    
 #pragma omp parallel for
   for ( int y = 0; y < ysize; y += testWSize )
   {
@@ -788,16 +1038,18 @@ void SemSegNovelty::computeNoveltyByGPMean(  NICE::FloatImage & noveltyImage,
 
       double minMeanAbs ( numeric_limits<double>::max() );
       
-      for ( int j = 0 ; j < cr.scores.size(); j++ )
+      for ( int j = 0 ; j < probabilities.channels(); j++ )
       {
         if ( forbidden_classesTrain.find ( j ) != forbidden_classesTrain.end() )
         {
           continue;
         }
-        
+
         //check whether we found a class with higher smaller abs mean than the current minimum
-        if (abs(cr.scores[j]) < minMeanAbs)  
-          minMeanAbs = abs(cr.scores[j]);     
+        if (abs(probabilities(x,y,j)) < minMeanAbs)
+        {
+          minMeanAbs = abs(cr.scores[j]); 
+        }
       }
 
       // compute results when we take the lowest mean value of all classes
@@ -815,15 +1067,11 @@ void SemSegNovelty::computeNoveltyByGPMean(  NICE::FloatImage & noveltyImage,
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
+          segresult ( xl, yl ) = cr.classno;          
           noveltyImage ( xl, yl ) = gpMeanVal; 
         }
-      }    
-      
-      example.svec->clear();
+      }     
     }
-    delete example.svec;
-    example.svec = NULL;
   }  
 }
 
@@ -893,7 +1141,7 @@ void SemSegNovelty::computeNoveltyByGPMeanRatio(  NICE::FloatImage & noveltyImag
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
+          segresult ( xl, yl ) = cr.classno;          
           noveltyImage ( xl, yl ) = gpMeanRatioVal;
         }
       }    
@@ -1001,7 +1249,7 @@ void SemSegNovelty::computeNoveltyByGPWeightAll(  NICE::FloatImage & noveltyImag
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
+          segresult ( xl, yl ) = cr.classno;          
           noveltyImage ( xl, yl ) = gpWeightAllVal;
         }
       }
@@ -1124,7 +1372,7 @@ void SemSegNovelty::computeNoveltyByGPWeightRatio(  NICE::FloatImage & noveltyIm
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
           }
-          segresult ( xl, yl ) = cr.classno;
+          segresult ( xl, yl ) = cr.classno;          
           noveltyImage ( xl, yl ) = gpWeightRatioVal;  
         }
       }
@@ -1134,4 +1382,138 @@ void SemSegNovelty::computeNoveltyByGPWeightRatio(  NICE::FloatImage & noveltyIm
     delete example.svec;
     example.svec = NULL;
   }  
-}
+}
+
+
+void SemSegNovelty::addNewExample(const NICE::Vector& newExample, const int & newClassNo)
+{
+  //accept the new class as valid information
+  if ( forbidden_classesTrain.find ( newClassNo ) != forbidden_classesTrain.end() )
+  {
+    forbidden_classesTrain.erase(newClassNo);
+    numberOfClasses++;
+  }
+  if ( classesInUse.find ( newClassNo ) == classesInUse.end() )
+  {
+    classesInUse.insert( newClassNo );
+  }    
+  
+  
+  //then add it to the classifier used
+  if ( classifier != NULL )
+  { 
+    //TODO    
+  }
+  else //vclassifier
+  {
+    if (this->classifierString.compare("nn") == 0)    
+    {
+      vclassifier->teach ( newClassNo, newExample );
+    }
+  }
+}
+
+void SemSegNovelty::addNovelExamples()
+{
+
+  Timer timer;
+  
+  //show the image that contains the most novel region
+  if (visualizeALimages)
+    showImage(maskedImg, "Most novel region");  
+  
+  timer.start();
+  
+
+    std::stringstream out;
+    std::vector< std::string > list2;
+    StringTools::split ( Globals::getCurrentImgFN (), '/', list2 );
+    out << resultdir << "/" << list2.back();     
+    
+    maskedImg.writePPM ( out.str() + "_run_" +  NICE::intToString(this->iterationCountSuffix) + "_" + noveltyMethodString+ "_query.ppm" );
+
+  
+  timer.stop();
+  std::cerr << "AL time for writing queried image: " << timer.getLast() << std::endl;
+
+  timer.start();
+  
+  //check which classes will be added using the features from the novel region
+  std::set<int> newClassNumbers;
+  newClassNumbers.clear(); //just to be sure  
+  for ( uint i = 0 ; i < newTrainExamples.size() ; i++ )
+  {
+    if (newClassNumbers.find(newTrainExamples[i].first /* classNumber*/) == newClassNumbers.end() )
+    {
+      newClassNumbers.insert(newTrainExamples[i].first );
+    }
+  }
+
+  //accept the new classes as valid information
+  for (std::set<int>::const_iterator clNoIt = newClassNumbers.begin(); clNoIt != newClassNumbers.end(); clNoIt++)
+  {
+    if ( forbidden_classesTrain.find ( *clNoIt ) != forbidden_classesTrain.end() )
+    {
+      forbidden_classesTrain.erase(*clNoIt);
+      numberOfClasses++;
+    }
+    if ( classesInUse.find ( *clNoIt ) == classesInUse.end() )
+    {
+      classesInUse.insert( *clNoIt );
+    }
+  }
+  
+  timer.stop();
+  std::cerr << "AL time for accepting possible new classes: " << timer.getLast() << std::endl;
+  
+  timer.start();
+  //then add the new features to the classifier used
+  if ( classifier != NULL )
+  { 
+    if (this->classifierString.compare("ClassifierGPHIK") == 0)    
+    {
+      classifier->addMultipleExamples ( this->newTrainExamples );
+    }    
+  }
+  else //vclassifier
+  {
+    //TODO
+  }
+  
+  timer.stop();
+  std::cerr << "AL time for actually updating the classifier: " << timer.getLast() << std::endl;
+  
+  std::cerr << "the current region to query is: " << currentRegionToQuery.first << " -- " << currentRegionToQuery.second << std::endl;
+  
+  //did we already query a region of this image?
+  if ( queriedRegions.find( currentRegionToQuery.first ) != queriedRegions.end() )
+  {
+    queriedRegions[ currentRegionToQuery.first ].insert(currentRegionToQuery.second);
+  }
+  else
+  {
+    std::set<int> tmpSet; tmpSet.insert(currentRegionToQuery.second);
+    queriedRegions.insert(std::pair<std::string,std::set<int> > (currentRegionToQuery.first, tmpSet ) );
+  }  
+  
+  std::cerr << "Write already queried regions: " << std::endl;
+  for (std::map<std::string,std::set<int> >::const_iterator it = queriedRegions.begin(); it != queriedRegions.end(); it++)
+  {
+    std::cerr << "image: " << it->first << " --   ";
+    for (std::set<int>::const_iterator itReg = it->second.begin(); itReg != it->second.end(); itReg++)
+    {
+      std::cerr << *itReg << " ";
+    } 
+    std::cerr << std::endl;
+  }
+  
+  //clear the latest results, since one iteration is over
+  globalMaxUncert = -numeric_limits<double>::max();
+  if (!mostNoveltyWithMaxScores)
+    globalMaxUncert = numeric_limits<double>::max();
+}
+
+const Examples * SemSegNovelty::getNovelExamples() const
+{
+  return &(this->newTrainExamples);
+}

semseg/SemSegNovelty.h

@@ -1,7 +1,7 @@
 /**
  * @file SemSegNovelty.h
  * @brief semantic segmentation using the method from Csurka08
- * @author Björn Fröhlich
+ * @author Björn Fröhlich, Alexander Freytag
  * @date 04/24/2009
  */
 #ifndef SemSegNoveltyINCLUDE
@@ -11,6 +11,7 @@
 
 #include "SemSegTools.h"
 #include "vislearning/classifier/classifierbase/FeaturePoolClassifier.h"
+#include "vislearning/classifier/genericClassifierSelection.h"
 #include "vislearning/features/localfeatures/LFColorWeijer.h"
 
 #include "segmentation/RegionSegmentationMethod.h"
@@ -24,17 +25,21 @@ class SemSegNovelty : public SemanticSegmentation
 {
 
   protected:
-    //! boolean whether to save the cache or not
-    bool save_cache;
+    //! boolean whether to reuse segmentation results for single images in different runs
+    bool reuseSegmentation;
 
-    //! boolean whether to read the cache or not, if read_cache is false, everything will be trained
-    bool read_cache;
+    //! boolean whether to read the initial classifier from a file. If not, training will be performed
+    bool read_classifier;
+    
+    //! boolean whether to save the final classifier or not
+    bool save_classifier;
 
     //! The cached Data
     std::string cache;
     
     //! Classifier
     FeaturePoolClassifier *classifier;
+    VecClassifier *vclassifier;
     
     //! feature extraction
     LFColorWeijer *featExtract;
@@ -57,29 +62,42 @@ class SemSegNovelty : public SemanticSegmentation
     //! low level Segmentation method
     RegionSegmentationMethod *regionSeg;
     
-    //! set of forbidden/background classes
-    std::set<int> forbidden_classes;
+    //! set of forbidden/background classes for the initial training
     std::set<int> forbidden_classesTrain;
+    //! set of forbidden/background classes for the whole process of learning over time
+    std::set<int> forbidden_classesActiveLearning;
+    //! store the class numbers currently used
     std::set<int> classesInUse;
+        
+    //! obviously, the number of classes used for training (i.e., classesInUse.size() )
+    int numberOfClasses; 
     
-    //! obviously, the number of classes used for training
-    int numberOfClasses;
-    
-    //! where to save the uncertainty
-    std::string uncertdir;
+    //! where to save the resulting images (uncertainty and classification results)
+    std::string resultdir;
     
-    //! find the maximum uncertainty or not
+    //! find the maximum uncertainty or not within the whole test set
     bool findMaximumUncert;
     
     //! image with most uncertain region
     NICE::ColorImage maskedImg;
     
-    //! maximum uncertainty over all images
+    //! for debugging and visualization: show novelty images with and without region segmentation and the most novel region
+    bool visualizeALimages;
+    
+    //! maximum uncertainty over all images, i.e., the novelty score of the most "novel" region of all test images
     double globalMaxUncert;
     
+    //! determine whether a "novelty" method computes large scores for novel objects (e.g., variance), or small scores (e.g., min abs mean)
+    bool mostNoveltyWithMaxScores;
+    
     //! current examples for most uncertain region
     Examples newTrainExamples;
     
+    //! contains filenames of images and indices of contained regions, that where already queried, to prevent them from being queried again
+    std::map<std::string,std::set<int> > queriedRegions;
+    
+    std::pair<std::string, int> currentRegionToQuery;
+    
     enum NoveltyMethod{
       GPVARIANCE, // novel = large variance
       GPUNCERTAINTY, //novel = small uncertainty (mean / var)
@@ -87,22 +105,32 @@ class SemSegNovelty : public SemanticSegmentation
       GPMEANRATIO,  //novel = small difference between mean of most plausible class and mean of snd
                    //        most plausible class (not useful in binary settings)
       GPWEIGHTALL, // novel = large weight in alpha vector after updating the model (can be predicted exactly)
-      GPWEIGHTRATIO // novel = small difference between weights for alpha vectors with assumptions of GT label to be the most 
+      GPWEIGHTRATIO, // novel = small difference between weights for alpha vectors with assumptions of GT label to be the most 
                     //         plausible against the second most plausible class
+      RANDOM        // query regions randomly
     }; 
     
     //! specify how "novelty" shall be computed, e.g., using GP-variance, GP-uncertainty, or predicted weight entries
     NoveltyMethod noveltyMethod;
     std::string noveltyMethodString;
     
+    //! just store the name of our classifier
+    std::string classifierString;
+    
     inline void computeClassificationResults( const NICE::MultiChannelImageT<double> & feats, 
                                                     NICE::Image & segresult,
                                                     NICE::MultiChannelImageT<double> & probabilities,
                                                     const int & xsize,
                                                     const int & ysize,
                                                     const int & featdim );
-   
-   void computeNoveltyByVariance( NICE::FloatImage & noveltyImage, 
+
+   void computeNoveltyByRandom(         NICE::FloatImage & noveltyImage, 
+                                  const NICE::MultiChannelImageT<double> & feats,  
+                                        NICE::Image & segresult,
+                                        NICE::MultiChannelImageT<double> & probabilities,
+                                  const int & xsize, const int & ysize, const int & featdim );    
+    
+   void computeNoveltyByVariance(       NICE::FloatImage & noveltyImage, 
                                   const NICE::MultiChannelImageT<double> & feats,  
                                         NICE::Image & segresult,
                                         NICE::MultiChannelImageT<double> & probabilities,
@@ -170,6 +198,29 @@ class SemSegNovelty : public SemanticSegmentation
      * @return void
      **/
     void visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage);
+
+    /**
+     * @brief Add a new example to the known training data
+     *
+     * @param newExample (NICE::Vector) the feature vector of the new examples
+     * @param newClassNo (int) the corresponding GT class number
+     * @return void
+     **/    
+    void addNewExample(const NICE::Vector & newExample, const int & newClassNo);
+    
+    /**
+     * @brief Add those examples, which belong to the most novel region seen so far
+     *
+     * @return void
+     **/    
+    virtual void addNovelExamples();    
+
+    /**
+     * @brief Get a pointer to the examples extracted from the most novel region seen so far
+     *
+     * @return Examples *
+     **/        
+    virtual const Examples * getNovelExamples() const; 
 };
 
 } //namespace

semseg/SemanticSegmentation.cpp

@@ -45,7 +45,7 @@ void SemanticSegmentation::convertLSetToSparseExamples ( Examples &examples, Lab
 #endif
 }
 
-void SemanticSegmentation::convertLSetToExamples ( Examples &examples, LabeledSetVector &lvec )
+void SemanticSegmentation::convertLSetToExamples ( Examples &examples, LabeledSetVector &lvec, const bool & removeOldDataPointer )
 {
 #ifdef DEBUG_PRINTS
   cout << "SemSegRegionBased::convertLSetToExamples starts" << endl;
@@ -59,7 +59,20 @@ void SemanticSegmentation::convertLSetToExamples ( Examples &examples, LabeledSe
       examples.push_back ( pair<int, Example> ( iter->first, ex ) );
     }
   }
-  lvec.clear();
+  
+  if (!removeOldDataPointer)
+  {
+    //NOTE this is only useful, if our classifier does NOT need the data explicitely
+    lvec.clear();
+  }
+  else
+  {
+    lvec.removePointersToDataWithoutDeletion();
+    //after setting all the pointers to NULL, we can savely clear the LSet without deleting the previously
+    //stored features, which might be needed somewhere else, e.g., in the VCNearestNeighbour
+    lvec.clear();
+  }
+
 #ifdef DEBUG_PRINTS
   cout << "SemSegRegionBased::convertLSetToExamples finished" << endl;
 #endif
@@ -83,7 +96,11 @@ void SemanticSegmentation::convertExamplesToLSet ( Examples &examples, LabeledSe
     {
       if ( examples[i].second.svec != NULL )
       {
-        throw ( "Transform SVEC to VEC not yet implemented" );
+        NICE::Vector v;
+        examples[i].second.svec->convertToVectorT(v);
+        lvec.add ( examples[i].first, v );
+        delete examples[i].second.svec;
+        examples[i].second.svec = NULL;        
       }
       else
       {
@@ -92,7 +109,7 @@ void SemanticSegmentation::convertExamplesToLSet ( Examples &examples, LabeledSe
     }
 
   }
-  examples.clear();
+  examples.clear();  
 #ifdef DEBUG_PRINTS
   cout << "SemSegRegionBased::convertExamplesToLSet finished" << endl;
 #endif

semseg/SemanticSegmentation.h

@@ -14,6 +14,11 @@
 #include "vislearning/cbaselib/Example.h"
 
 
+#define ROADWORKSADD fthrow(NICE::Exception, "addNewExample(const NICE::Vector & newExample, const int & newClassNo): not yet implemented!");
+#define ROADWORKSADDNOVEL fthrow(NICE::Exception, "addNovelExamples(): not yet implemented!");
+#define ROADWORKSGETNOVEL fthrow(NICE::Exception, "getNovelExamples(): not yet implemented!");
+
+
 namespace OBJREC
 {
 
@@ -35,6 +40,8 @@ class SemanticSegmentation
 
     /** whether to load images with color information */
     int imagetype;
+    
+    int iterationCountSuffix;
 
   public:
 
@@ -65,7 +72,7 @@ class SemanticSegmentation
     void convertVVectorToExamples ( NICE::VVector &feats,OBJREC::Examples &examples, std::vector<int> &label );
     void convertExamplesToVVector ( NICE::VVector &feats,OBJREC::Examples &examples, std::vector<int> &label );
     void convertExamplesToLSet ( OBJREC::Examples &examples, OBJREC::LabeledSetVector &lvec );
-    void convertLSetToExamples ( OBJREC::Examples &examples, OBJREC::LabeledSetVector &lvec );
+    void convertLSetToExamples ( OBJREC::Examples &examples, OBJREC::LabeledSetVector &lvec, const bool & removeOldDataPointer=false );
     void convertLSetToSparseExamples ( OBJREC::Examples &examples, OBJREC::LabeledSetVector &lvec );
 
 
@@ -73,6 +80,34 @@ class SemanticSegmentation
     void semanticseg ( const std::string & filename,
                        NICE::Image & segresult,
                        NICE::MultiChannelImageT<double> & probabilities );
+    
+    virtual void addNewExample(const NICE::Vector & newExample, const int & newClassNo)
+    {
+      ROADWORKSADD;      
+    };
+    /**
+     * @brief Add those examples, which belong to the most novel region seen so far
+     *
+     * @return void
+     **/    
+    virtual void addNovelExamples()
+    {
+      ROADWORKSADDNOVEL;      
+    };    
+    
+    /**
+     * @brief Get a pointer to the examples extracted from the most novel region seen so far
+     *
+     * @return Examples *
+     **/        
+    virtual const Examples * getNovelExamples() const
+    {
+      ROADWORKSGETNOVEL;      
+    };        
+    
+    void setIterationCountSuffix( const int & _iterationCountSuffix) { iterationCountSuffix = _iterationCountSuffix; };
+    
+    
 
 };