added segmentation to semsegnovelty, find most uncertain region over all images, visualize it, and save features for this region

progs/testSemanticSegmentation.cpp

@@ -142,7 +142,7 @@ int main( int argc, char **argv )
       const LocalizationResult *l_gt = info.localization();
 
       lm.resize( l_gt->xsize, l_gt->ysize );
-      lm.set( 0 );
+      //lm.set( 0 );
       l_gt->calcLabeledImage( lm, classNames.getBackgroundClass() );
     }
 

semseg/SemSegNovelty.cpp

@@ -22,6 +22,8 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
 {
   this->conf = conf;
 
+  globalMaxUncert = -numeric_limits<double>::max();
+  
   string section = "SemSegNovelty";
 
   featExtract = new LFColorWeijer ( conf );
@@ -33,6 +35,7 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   
   classifier = new GPHIKClassifierNICE ( conf, "ClassiferGPHIK" );;
 
+  findMaximumUncert = conf->gB(section, "findMaximumUncert", true);
   whs = conf->gI ( section, "window_size", 10 );
   featdist = conf->gI ( section, "grid", 10 );
   testWSize = conf->gI (section, "test_window_size", 10);
@@ -44,11 +47,11 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   }
   else
   {
-    RegionSegmentationMethod * tmpRegionSeg = GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(&conf, rsMethode);    
+    RegionSegmentationMethod *tmpRegionSeg = GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(conf, rsMethode);    
     if ( save_cache )
       regionSeg = new RSCache ( conf, tmpRegionSeg );
     else
-      regionSeg = tmpseg;
+      regionSeg = tmpRegionSeg;
   }
   
   cn = md->getClassNames ( "train" );
@@ -80,6 +83,14 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
 
 SemSegNovelty::~SemSegNovelty()
 {
+  if(newTrainExamples.size() > 0)
+  {
+    // most uncertain region
+    showImage(maskedImg);
+    //classifier->add(newTrainExamples)
+    classifier->save ( cache + "/classifier.data" );
+  }
+  
   // clean-up
   if ( classifier != NULL )
     delete classifier;
@@ -87,6 +98,40 @@ SemSegNovelty::~SemSegNovelty()
     delete featExtract;
 }
 
+void SemSegNovelty::visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage)
+{
+  vector<uchar> color;
+  color.push_back(255);
+  color.push_back(0);
+  color.push_back(0);
+    
+  int width = img.width();
+  int height = img.height();
+  
+  outimage.resize(width,height);
+  
+  for(int y = 0; y < height; y++)
+  {
+    for(int x = 0; x < width; x++)
+    {
+      if(regions(x,y) == region)
+      {
+        for(int c = 0; c < 3; c++)
+        {
+          outimage(x,y,c) = color[c];
+        }
+      }
+      else
+      {
+        for(int c = 0; c < 3; c++)
+        {
+          outimage(x,y,c) = img(x,y,c);
+        }
+      } 
+    }
+  }
+}
+
 void SemSegNovelty::train ( const MultiDataset *md )
 {
   const LabeledSet train = * ( *md ) ["train"];
@@ -274,11 +319,9 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
 {
   Timer timer;
   timer.start();
-
-  Examples examples;
-  examples.filename = "testing";
-
-  segresult.set ( 0 );
+  
+  Image labels = segresult;
+  segresult.set(0);
 
   int featdim = -1;
 
@@ -289,8 +332,8 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   ce->getImageSize ( xsize, ysize );
 
   probabilities.reInit( xsize, ysize, cn.getMaxClassno() + 1);
-  probabilities.set ( 0.0 );
-
+  probabilities.setAll ( 0.0 );
+   
   NICE::ColorImage img;
   try {
     img = ColorImage ( currentFile );
@@ -559,7 +602,8 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
     int amountRegions = regionSeg->segRegions ( img, mask );
     
     //compute probs per region
-    vector<vector<double> > regionProb(amountRegions,vector<double>(probabilities.channels(),0.0);
+    vector<vector<double> > regionProb(amountRegions,vector<double>(probabilities.channels(),0.0));
+    vector<double> regionNoveltyMeasure (amountRegions, 0.0);
     vector<int> regionCounter(amountRegions, 0);
     for ( int y = 0; y < ysize; y++)
     {
@@ -569,28 +613,76 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
         regionCounter[r]++;
         for(int j = 0; j < probabilities.channels(); j++)
         {
-          regionProb[r][j] = probabilities ( x, y, j );
+          regionProb[r][j] += probabilities ( x, y, j );
         }
+        regionNoveltyMeasure[r] += uncert(x,y);
       }
     }
-    
+       
     //find best class per region
     vector<int> bestClassPerRegion(amountRegions,0);
     
+    double maxuncert = -numeric_limits<double>::max();
+    int maxUncertRegion = -1;
+    
     for(int r = 0; r < amountRegions; r++)
     {
-      double maxval = -numeric_limit<double>::max();
+      double maxval = -numeric_limits<double>::max();
       for(int c = 0; c < probabilities.channels(); c++)
       {
         regionProb[r][c] /= regionCounter[r];
-        if(maxval < regionProb[r][c])
+        if(maxval < regionProb[r][c] && regionProb[r][c] != 0.0)
         {
           maxval = regionProb[r][c];
           bestClassPerRegion[r] = c;
         }
       }
+      regionNoveltyMeasure[r] /= regionCounter[r];
+      if(maxuncert < regionNoveltyMeasure[r])
+      {
+        maxuncert = regionNoveltyMeasure[r];
+        maxUncertRegion = r;
+      }
     }
     
+    if(findMaximumUncert)
+    {
+      if(maxuncert > globalMaxUncert)
+      {
+        //save new important features
+        Examples examples;
+        for ( int y = 0; y < ysize; y += testWSize )
+        {
+          for ( int x = 0; x < xsize; x += testWSize)
+          {
+            if(mask(x,y) == maxUncertRegion)
+            {
+              Example example;
+              example.vec = NULL;
+              example.svec = new SparseVector ( featdim );
+              int classnoTmp = labels(x,y);
+              for ( int f = 0; f < featdim; f++ )
+              {
+                double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
+                if ( val > 1e-10 )
+                  ( *example.svec ) [f] = val;
+              }
+              example.svec->normalize();
+              examples.push_back ( pair<int, Example> ( classnoTmp, example ) );
+            }
+          }
+        }
+        
+        if(examples.size() > 0)
+        {
+          newTrainExamples.clear();
+          newTrainExamples = examples;
+          globalMaxUncert = maxuncert;
+          visualizeRegion(img,mask,maxUncertRegion,maskedImg);
+        }
+      }
+    }
+
     //write back best results per region
     for ( int y = 0; y < ysize; y++)
     {

semseg/SemSegNovelty.h

@@ -13,6 +13,8 @@
 #include "vislearning/classifier/classifierbase/FeaturePoolClassifier.h"
 #include "vislearning/features/localfeatures/LFColorWeijer.h"
 
+#include "segmentation/RegionSegmentationMethod.h"
+
 
 /** @brief pixelwise labeling systems */
 
@@ -65,6 +67,18 @@ class SemSegNovelty : public SemanticSegmentation
     
     //! where to save the uncertainty
     std::string uncertdir;
+    
+    //! find the maximum uncertainty or not
+    bool findMaximumUncert;
+    
+    //! image with most uncertain region
+    NICE::ColorImage maskedImg;
+    
+    //! maximum uncertainty over all images
+    double globalMaxUncert;
+    
+    //! current examples for maximum uncertain region
+    Examples newTrainExamples;
    
   public:
 
@@ -90,6 +104,17 @@ class SemSegNovelty : public SemanticSegmentation
     void semanticseg ( CachedExample *ce,
                        NICE::Image & segresult,
                        NICE::MultiChannelImageT<double> & probabilities );
+    
+    /**
+     * @brief visualize a specific region in the original image
+     *
+     * @param img input image
+     * @param regions map of the regions
+     * @param region visualize this region
+     * @param outimage result
+     * @return void
+     **/
+    void visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage);
 };
 
 } //namespace