Merge branch 'master', remote branch 'origin/master'

progs/testSemanticSegmentation.cpp

@@ -142,7 +142,7 @@ int main( int argc, char **argv )
       const LocalizationResult *l_gt = info.localization();
 
       lm.resize( l_gt->xsize, l_gt->ysize );
-      lm.set( 0 );
+      //lm.set( 0 );
       l_gt->calcLabeledImage( lm, classNames.getBackgroundClass() );
     }
 

semseg/SemSegCsurka.cpp

@@ -11,6 +11,7 @@ using namespace NICE;
 using namespace OBJREC;
 
 #undef DEBUG_CSURK
+
 #undef UNCERTAINTY
 // #define UNCERTAINTY
 

semseg/SemSegNovelty.cpp

@@ -10,6 +10,7 @@
 #include "vislearning/features/fpfeatures/SparseVectorFeature.h"
 #include "core/basics/StringTools.h"
 #include "core/basics/Timer.h"
+#include "segmentation/GenericRegionSegmentationMethodSelection.h"
 
 using namespace std;
 using namespace NICE;
@@ -21,6 +22,8 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
 {
   this->conf = conf;
 
+  globalMaxUncert = -numeric_limits<double>::max();
+  
   string section = "SemSegNovelty";
 
   featExtract = new LFColorWeijer ( conf );
@@ -29,14 +32,28 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   read_cache = conf->gB ( "FPCPixel", "read_cache", false );
   uncertdir = conf->gS("debug", "uncertainty", "uncertainty");
   cache = conf->gS ( "cache", "root", "" );
-
-
+  
   classifier = new GPHIKClassifierNICE ( conf, "ClassiferGPHIK" );;
 
+  findMaximumUncert = conf->gB(section, "findMaximumUncert", true);
   whs = conf->gI ( section, "window_size", 10 );
   featdist = conf->gI ( section, "grid", 10 );
   testWSize = conf->gI (section, "test_window_size", 10);
-
+  string rsMethode = conf->gS ( section, "segmentation", "none" );
+ 
+  if(rsMethode == "none")
+  {
+    regionSeg = NULL;
+  }
+  else
+  {
+    RegionSegmentationMethod *tmpRegionSeg = GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(conf, rsMethode);    
+    if ( save_cache )
+      regionSeg = new RSCache ( conf, tmpRegionSeg );
+    else
+      regionSeg = tmpRegionSeg;
+  }
+  
   cn = md->getClassNames ( "train" );
 
   if ( read_cache )
@@ -66,6 +83,14 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
 
 SemSegNovelty::~SemSegNovelty()
 {
+  if(newTrainExamples.size() > 0)
+  {
+    // most uncertain region
+    showImage(maskedImg);
+    //classifier->add(newTrainExamples)
+    classifier->save ( cache + "/classifier.data" );
+  }
+  
   // clean-up
   if ( classifier != NULL )
     delete classifier;
@@ -73,6 +98,40 @@ SemSegNovelty::~SemSegNovelty()
     delete featExtract;
 }
 
+void SemSegNovelty::visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage)
+{
+  vector<uchar> color;
+  color.push_back(255);
+  color.push_back(0);
+  color.push_back(0);
+    
+  int width = img.width();
+  int height = img.height();
+  
+  outimage.resize(width,height);
+  
+  for(int y = 0; y < height; y++)
+  {
+    for(int x = 0; x < width; x++)
+    {
+      if(regions(x,y) == region)
+      {
+        for(int c = 0; c < 3; c++)
+        {
+          outimage(x,y,c) = color[c];
+        }
+      }
+      else
+      {
+        for(int c = 0; c < 3; c++)
+        {
+          outimage(x,y,c) = img(x,y,c);
+        }
+      } 
+    }
+  }
+}
+
 void SemSegNovelty::train ( const MultiDataset *md )
 {
   const LabeledSet train = * ( *md ) ["train"];
@@ -260,11 +319,9 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
 {
   Timer timer;
   timer.start();
-
-  Examples examples;
-  examples.filename = "testing";
-
-  segresult.set ( 0 );
+  
+  Image labels = segresult;
+  segresult.set(0);
 
   int featdim = -1;
 
@@ -275,8 +332,8 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   ce->getImageSize ( xsize, ysize );
 
   probabilities.reInit( xsize, ysize, cn.getMaxClassno() + 1);
-  probabilities.set ( 0.0 );
-
+  probabilities.setAll ( 0.0 );
+   
   NICE::ColorImage img;
   try {
     img = ColorImage ( currentFile );
@@ -330,7 +387,6 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   double maxGPWeightAll = -numeric_limits<double>::max();  
   double maxGPWeightRatio = -numeric_limits<double>::max();  
 
-  
   timer.stop();
   cout << "first: " << timer.getLastAbsolute() << endl;
   
@@ -503,7 +559,7 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
         {
           for ( int j = 0 ; j < cr.scores.size(); j++ )
           {
-            probabilities ( xl, yl, j ) = cr.scores[j];           
+            probabilities ( xl, yl, j ) = cr.scores[j];
           }
           segresult ( xl, yl ) = cr.classno;
           uncert ( xl, yl ) = cr.uncertainty;
@@ -530,15 +586,117 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
       if (maxGPWeightRatio < gpWeightRatioVal)
         maxGPWeightRatio = gpWeightRatioVal;      
       
-//       std::cerr << "uncertainty: " << gpUncertaintyVal << " minMean: " << gpMeanVal << " gpMeanRatio: " << gpMeanRatioVal << " weightAll: " << gpWeightAllVal << " weightRatio: "<< gpWeightRatioVal << std::endl;
-      
       example.svec->clear();
     }
     delete example.svec;
     example.svec = NULL;
   }
 
-  cout << "maxunertainty: " << maxunc << endl;
+  //       std::cerr << "uncertainty: " << gpUncertaintyVal << " minMean: " << gpMeanVal << " gpMeanRatio: " << gpMeanRatioVal << " weightAll: " << gpWeightAllVal << " weightRatio: "<< gpWeightRatioVal << std::endl;
+  
+  
+  //Regionen ermitteln
+  if(regionSeg != NULL)
+  {
+    NICE::Matrix mask;
+    int amountRegions = regionSeg->segRegions ( img, mask );
+    
+    //compute probs per region
+    vector<vector<double> > regionProb(amountRegions,vector<double>(probabilities.channels(),0.0));
+    vector<double> regionNoveltyMeasure (amountRegions, 0.0);
+    vector<int> regionCounter(amountRegions, 0);
+    for ( int y = 0; y < ysize; y++)
+    {
+      for (int x = 0; x < xsize; x++)
+      {
+        int r = mask(x,y);
+        regionCounter[r]++;
+        for(int j = 0; j < probabilities.channels(); j++)
+        {
+          regionProb[r][j] += probabilities ( x, y, j );
+        }
+        regionNoveltyMeasure[r] += uncert(x,y);
+      }
+    }
+       
+    //find best class per region
+    vector<int> bestClassPerRegion(amountRegions,0);
+    
+    double maxuncert = -numeric_limits<double>::max();
+    int maxUncertRegion = -1;
+    
+    for(int r = 0; r < amountRegions; r++)
+    {
+      double maxval = -numeric_limits<double>::max();
+      for(int c = 0; c < probabilities.channels(); c++)
+      {
+        regionProb[r][c] /= regionCounter[r];
+        if(maxval < regionProb[r][c] && regionProb[r][c] != 0.0)
+        {
+          maxval = regionProb[r][c];
+          bestClassPerRegion[r] = c;
+        }
+      }
+      regionNoveltyMeasure[r] /= regionCounter[r];
+      if(maxuncert < regionNoveltyMeasure[r])
+      {
+        maxuncert = regionNoveltyMeasure[r];
+        maxUncertRegion = r;
+      }
+    }
+    
+    if(findMaximumUncert)
+    {
+      if(maxuncert > globalMaxUncert)
+      {
+        //save new important features
+        Examples examples;
+        for ( int y = 0; y < ysize; y += testWSize )
+        {
+          for ( int x = 0; x < xsize; x += testWSize)
+          {
+            if(mask(x,y) == maxUncertRegion)
+            {
+              Example example;
+              example.vec = NULL;
+              example.svec = new SparseVector ( featdim );
+              int classnoTmp = labels(x,y);
+              for ( int f = 0; f < featdim; f++ )
+              {
+                double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
+                if ( val > 1e-10 )
+                  ( *example.svec ) [f] = val;
+              }
+              example.svec->normalize();
+              examples.push_back ( pair<int, Example> ( classnoTmp, example ) );
+            }
+          }
+        }
+        
+        if(examples.size() > 0)
+        {
+          newTrainExamples.clear();
+          newTrainExamples = examples;
+          globalMaxUncert = maxuncert;
+          visualizeRegion(img,mask,maxUncertRegion,maskedImg);
+        }
+      }
+    }
+
+    //write back best results per region
+    for ( int y = 0; y < ysize; y++)
+    {
+      for (int x = 0; x < xsize; x++)
+      {
+        int r = mask(x,y);
+        for(int j = 0; j < probabilities.channels(); j++)
+        {
+          probabilities ( x, y, j ) = regionProb[r][j];
+        }
+        segresult(x,y) = bestClassPerRegion[r];
+      }
+    }
+  }
 
   timer.stop();
   cout << "second: " << timer.getLastAbsolute() << endl;

semseg/SemSegNovelty.h

@@ -13,6 +13,8 @@
 #include "vislearning/classifier/classifierbase/FeaturePoolClassifier.h"
 #include "vislearning/features/localfeatures/LFColorWeijer.h"
 
+#include "segmentation/RegionSegmentationMethod.h"
+
 
 /** @brief pixelwise labeling systems */
 
@@ -52,6 +54,9 @@ class SemSegNovelty : public SemanticSegmentation
     //! name of all classes
     ClassNames cn;
     
+    //! low level Segmentation method
+    RegionSegmentationMethod *regionSeg;
+    
     //! set of forbidden/background classes
     std::set<int> forbidden_classes;
     std::set<int> forbidden_classesTrain;
@@ -62,6 +67,18 @@ class SemSegNovelty : public SemanticSegmentation
     
     //! where to save the uncertainty
     std::string uncertdir;
+    
+    //! find the maximum uncertainty or not
+    bool findMaximumUncert;
+    
+    //! image with most uncertain region
+    NICE::ColorImage maskedImg;
+    
+    //! maximum uncertainty over all images
+    double globalMaxUncert;
+    
+    //! current examples for maximum uncertain region
+    Examples newTrainExamples;
    
   public:
 
@@ -87,6 +104,17 @@ class SemSegNovelty : public SemanticSegmentation
     void semanticseg ( CachedExample *ce,
                        NICE::Image & segresult,
                        NICE::MultiChannelImageT<double> & probabilities );
+    
+    /**
+     * @brief visualize a specific region in the original image
+     *
+     * @param img input image
+     * @param regions map of the regions
+     * @param region visualize this region
+     * @param outimage result
+     * @return void
+     **/
+    void visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage);
 };
 
 } //namespace