current

semseg/SemSegContextTree.cpp

@@ -62,7 +62,7 @@ SemSegContextTree::SemSegContextTree (const Config *conf, const MultiDataset *md
 
   pixelWiseLabeling = false;
 
-  useRegionFeature = true;
+  useRegionFeature = conf->gB (section, "use_reagion_feat", true);
   if (segmentationtype == "meanshift")
     segmentation = new RSMeanShift (conf);
   else if (segmentationtype == "none")
@@ -578,6 +578,9 @@ void SemSegContextTree::train (const MultiDataset *md)
 #endif
 
   std::string forbidden_classes_s = conf->gS ("analysis", "donttrain", "");
+  
+  vector<vector<vector<double> > > regionProbs;
+  vector<int> amountRegionpI;
 
   if (forbidden_classes_s == "")
   {
@@ -589,6 +592,7 @@ void SemSegContextTree::train (const MultiDataset *md)
   int imgcounter = 0;
 
   int amountPixels = 0;
+  
 
   LOOP_ALL_S (*trainp)
   {
@@ -639,8 +643,14 @@ void SemSegContextTree::train (const MultiDataset *md)
     MultiChannelImageT<double> feats;
     allfeats.push_back (feats);
 
+    int amountRegions;
     // read image and do some simple transformations
-    extractBasicFeatures (allfeats[imgcounter], img, currentFile);
+    extractBasicFeatures (allfeats[imgcounter], img, currentFile, amountRegions);
+    
+    if(useRegionFeature)
+    {
+      amountRegionpI.push_back(amountRegions);
+    }
 
     // getting groundtruth
     NICE::Image pixelLabels (xsize, ysize);
@@ -682,6 +692,14 @@ void SemSegContextTree::train (const MultiDataset *md)
     classes++;
   }
 
+  if(useRegionFeature)
+  {
+    for(int a = 0; a < (int)amountRegionpI.size(); a++)
+    {
+      regionProbs.push_back(vector<vector<double> > (amountRegionpI[a], vector<double> (classes, 0.0)));
+    }
+  }
+
 ////////////////////////////////////////////////////
   //define which featurextraction methods should be used for each channel
   rawChannels = 3;
@@ -1143,13 +1161,12 @@ void SemSegContextTree::train (const MultiDataset *md)
   timer.start();
 }
 
-void SemSegContextTree::extractBasicFeatures (NICE::MultiChannelImageT<double> &feats, const ColorImage &img, const string &currentFile)
+void SemSegContextTree::extractBasicFeatures (NICE::MultiChannelImageT<double> &feats, const ColorImage &img, const string &currentFile, int &amountRegions)
 {
   int xsize = img.width();
   int ysize = img.height();
   //TODO: resize image?!
 
-
   feats.reInit (xsize, ysize, 3);
 
   for (int x = 0; x < xsize; x++)
@@ -1233,6 +1250,29 @@ void SemSegContextTree::extractBasicFeatures (NICE::MultiChannelImageT<double> &
       }
     }
   }
+  
+  if(useRegionFeature)
+  {
+    //using segmentation
+    Matrix regions;
+    amountRegions = segmentation->segRegions (img, regions);
+    
+    int cchannel = feats.channels();
+    feats.addChannel(1);
+    
+    assert(feats.width() == regions.cols());
+    for(int y = 0; y < regions.rows(); y++)
+    {
+      for(int x = 0; x < regions.cols(); x++)
+      {
+        feats(x,y,cchannel) = regions(x,y);
+      }
+    }
+  }
+  else
+  {
+    amountRegions = -1;
+  }
 }
 
 void SemSegContextTree::semanticseg (CachedExample *ce, NICE::Image & segresult, NICE::MultiChannelImageT<double> & probabilities)
@@ -1268,7 +1308,9 @@ void SemSegContextTree::semanticseg (CachedExample *ce, NICE::Image & segresult,
     return;
   }
 
-  extractBasicFeatures (feats, img, currentFile); //read image and do some simple transformations
+  //TODO add to features!
+  int amountRegions;
+  extractBasicFeatures (feats, img, currentFile, amountRegions); //read image and do some simple transformations
 
   bool allleaf = false;
 
@@ -1452,9 +1494,8 @@ void SemSegContextTree::semanticseg (CachedExample *ce, NICE::Image & segresult,
       {
         double maxvalue = - numeric_limits<double>::max(); //TODO: das kann auch nur pro knoten gemacht werden, nicht pro pixel
         int maxindex = 0;
-        uint s = forest[0][0].dist.size();
 
-        for (uint i = 0; i < s; i++)
+        for (uint i = 0; i < classes; i++)
         {
           int currentclass = labelmapback[i];
           if (useclass[currentclass])
@@ -1515,15 +1556,42 @@ void SemSegContextTree::semanticseg (CachedExample *ce, NICE::Image & segresult,
   }
   else
   {
-    //final labeling using segmentation
+    //using segmentation
     Matrix regions;
-    //showImage(img);
-    int regionNumber = segmentation->segRegions (img, regions);
-    cout << "regions: " << regionNumber << endl;
-
-    int dSize = forest[0][0].dist.size();
-    vector<vector<double> > regionProbs (regionNumber, vector<double> (dSize, 0.0));
-    vector<int> bestlabels (regionNumber, 0);
+   
+    if(useRegionFeature)
+    {
+      int rchannel = -1;
+      for(uint i = 0; i < channelType.size(); i++) 
+      {
+        if(channelType[i] == 1)
+        {
+          rchannel = i;
+          break;
+        }
+      }
+      
+      assert(rchannel > -1);
+      
+      int xsize = feats.width();
+      int ysize = feats.height();
+      regions.resize(xsize, ysize);
+      for(int y = 0; y < ysize; y++)
+      {
+        for(int x = 0; x < xsize; x++)
+        {
+          regions(x,y) = feats(x,y,rchannel);
+        }
+      }
+    }
+    else
+    {
+      amountRegions = segmentation->segRegions (img, regions);
+    }
+    
+    vector<vector<double> > regionProbs(amountRegions, vector<double> (classes, 0.0));
+    
+    vector<int> bestlabels (amountRegions, 0);
 
     for (int y = 0; y < img.height(); y++)
     {
@@ -1531,19 +1599,19 @@ void SemSegContextTree::semanticseg (CachedExample *ce, NICE::Image & segresult,
       {
         int cregion = regions (x, y);
 
-        for (int d = 0; d < dSize; d++)
+        for (int d = 0; d < classes; d++)
         {
           regionProbs[cregion][d] += getMeanProb (x, y, d, currentfeats);
         }
       }
     }
 
-    for (int r = 0; r < regionNumber; r++)
+    for (int r = 0; r < amountRegions; r++)
     {
       double maxval = regionProbs[r][0];
       bestlabels[r] = 0;
 
-      for (int d = 1; d < dSize; d++)
+      for (int d = 1; d < classes; d++)
       {
         if (maxval < regionProbs[r][d])
         {

semseg/SemSegContextTree.h

@@ -41,7 +41,7 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
 
     /** size for neighbourhood */
     int windowSize;
-
+    
     /** how many feats should be considered for a split */
     int featsPerSplit;
 
@@ -172,7 +172,7 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
      * @param currentFile image filename
      * @return void
      **/
-    void extractBasicFeatures ( NICE::MultiChannelImageT<double> &feats, const NICE::ColorImage &img, const std::string &currentFile);
+    void extractBasicFeatures ( NICE::MultiChannelImageT<double> &feats, const NICE::ColorImage &img, const std::string &currentFile, int &amountRegions);
     
     /**
      * @brief computes integral image for Sparse Multichannel Image

semseg/SemSegCsurka.cpp

@@ -1535,11 +1535,10 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
   }
   else
   {
-    
-#pragma omp parallel for
+//#pragma omp parallel for
     for ( int s = 0; s < scalesize; s++ )
     {
-#pragma omp parallel for
+//#pragma omp parallel for
       for ( int i = s; i < ( int ) pce.size(); i += scalesize )
       {
         ClassificationResult r = vclassifier->classify ( * ( pce[i].second.vec ) );

semseg/SemSegNovelty.cpp

@@ -3,6 +3,7 @@
 
 #include "SemSegNovelty.h"
 
+#include "core/image/FilterT.h"
 #include "fast-hik/GPHIKClassifier.h"
 #include "vislearning/baselib/ICETools.h"
 #include "vislearning/baselib/Globals.h"
@@ -20,17 +21,21 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
 {
   this->conf = conf;
 
+  string section = "SemSegNovelty";
+
   featExtract = new LFColorWeijer ( conf );
 
   save_cache = conf->gB ( "FPCPixel", "save_cache", true );
   read_cache = conf->gB ( "FPCPixel", "read_cache", false );
-  uncertdir = conf->gS("debug", "uncertainty","uncertainty");
+  uncertdir = conf->gS("debug", "uncertainty", "uncertainty");
   cache = conf->gS ( "cache", "root", "" );
 
+
   classifier = new GPHIKClassifier ( conf, "ClassiferGPHIK" );;
 
-  whs = conf->gI ( "SemSegNovelty", "window_size", 10 );
-  featdist = conf->gI ( "SemSegNovelty", "grid", 10 );
+  whs = conf->gI ( section, "window_size", 10 );
+  featdist = conf->gI ( section, "grid", 10 );
+  testWSize = conf->gI (section, "test_window_size", 10);
 
   cn = md->getClassNames ( "train" );
 
@@ -134,6 +139,16 @@ void SemSegNovelty::train ( const MultiDataset *md )
     // extract features
     featExtract->getFeats ( img, feats );
     featdim = feats.channels();
+    feats.addChannel(featdim);
+
+    for (int c = 0; c < featdim; c++)
+    {
+      ImageT<double> tmp = feats[c];
+      ImageT<double> tmp2 = feats[c+featdim];
+
+      NICE::FilterT<double, double, double>::gradientStrength (tmp, tmp2);
+    }
+    featdim += featdim;
 
     // compute integral images
     for ( int c = 0; c < featdim; c++ )
@@ -150,7 +165,7 @@ void SemSegNovelty::train ( const MultiDataset *md )
         if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
           continue;
 
-       
+
         Example example;
         example.vec = NULL;
         example.svec = new SparseVector ( featdim );
@@ -160,9 +175,9 @@ void SemSegNovelty::train ( const MultiDataset *md )
           if ( val > 1e-10 )
             ( *example.svec ) [f] = val;
         }
-        
+
         example.svec->normalize();
-        
+
         example.position = imgnb;
         examples.push_back ( pair<int, Example> ( classno, example ) );
       }
@@ -218,7 +233,7 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
 {
   Timer timer;
   timer.start();
-  
+
   Examples examples;
   examples.filename = "testing";
 
@@ -232,9 +247,9 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   int xsize, ysize;
   ce->getImageSize ( xsize, ysize );
 
-  probabilities.reInit( xsize, ysize, cn.getMaxClassno()+1);
+  probabilities.reInit( xsize, ysize, cn.getMaxClassno() + 1);
   probabilities.set ( 0.0 );
-  
+
   NICE::ColorImage img;
   try {
     img = ColorImage ( currentFile );
@@ -248,7 +263,17 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   // extract features
   featExtract->getFeats ( img, feats );
   featdim = feats.channels();
-  
+  feats.addChannel(featdim);
+
+  for (int c = 0; c < featdim; c++)
+  {
+    ImageT<double> tmp = feats[c];
+    ImageT<double> tmp2 = feats[c+featdim];
+
+    NICE::FilterT<double, double, double>::gradientStrength (tmp, tmp2);
+  }
+  featdim += featdim;
+
   // compute integral images
   for ( int c = 0; c < featdim; c++ )
   {
@@ -263,12 +288,12 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   cout << "first: " << timer.getLastAbsolute() << endl;
   timer.start();
 #pragma omp parallel for
-  for ( int y = 0; y < ysize; y++ )
+  for ( int y = 0; y < ysize; y += testWSize )
   {
     Example example;
     example.vec = NULL;
     example.svec = new SparseVector ( featdim );
-    for ( int x = 0; x < xsize; x++ )
+    for ( int x = 0; x < xsize; x += testWSize)
     {
       for ( int f = 0; f < featdim; f++ )
       {
@@ -277,29 +302,40 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
           ( *example.svec ) [f] = val;
       }
       example.svec->normalize();
-      
+
       ClassificationResult cr = classifier->classify ( example );
 
-      for ( int j = 0 ; j < cr.scores.size(); j++ )
+      int xs = std::max(0, x - testWSize/2);
+      int xe = std::min(xsize - 1, x + testWSize/2);
+      int ys = std::max(0, y - testWSize/2);
+      int ye = std::min(ysize - 1, y + testWSize/2);
+      for (int yl = ys; yl <= ye; yl++)
       {
-        probabilities ( x, y, j ) = cr.scores[j];
+        for (int xl = xs; xl <= xe; xl++)
+        {
+          for ( int j = 0 ; j < cr.scores.size(); j++ )
+          {
+            probabilities ( xl, yl, j ) = cr.scores[j];
+          }
+          segresult ( xl, yl ) = cr.classno;
+          uncert ( xl, yl ) = cr.uncertainty;
+        }
       }
-      segresult ( x, y ) = cr.classno;
-      if(maxunc < cr.uncertainty)
+
+      if (maxunc < cr.uncertainty)
         maxunc = cr.uncertainty;
-      uncert ( x, y ) = cr.uncertainty;
       example.svec->clear();
     }
     delete example.svec;
     example.svec = NULL;
   }
-  
+
   cout << "maxunertainty: " << maxunc << endl;
 
   timer.stop();
   cout << "second: " << timer.getLastAbsolute() << endl;
   timer.start();
-  
+
   ColorImage imgrgb ( xsize, ysize );
 
   std::stringstream out;
@@ -307,13 +343,13 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   StringTools::split ( Globals::getCurrentImgFN (), '/', list2 );
   out << uncertdir << "/" << list2.back();
 
-  uncert.writeRaw(out.str()+".rawfloat");
-  uncert(0,0) = 0.0;
-  uncert(0,1) = 1.0;
+  uncert.writeRaw(out.str() + ".rawfloat");
+  uncert(0, 0) = 0.0;
+  uncert(0, 1) = 1.0;
   ICETools::convertToRGB ( uncert, imgrgb );
   imgrgb.write ( out.str() + "rough.png" );
 
-  
+
   timer.stop();
   cout << "last: " << timer.getLastAbsolute() << endl;
 }

semseg/SemSegNovelty.h

@@ -46,6 +46,9 @@ class SemSegNovelty : public SemanticSegmentation
     //! half of the window size for local features
     int whs;
     
+    //! rectangle size for classification, 1 means pixelwise
+    int testWSize;
+    
     //! name of all classes
     ClassNames cn;