hoiem features

semseg/SemSegContextTree.cpp

@@ -9,6 +9,8 @@
 #include "objrec/segmentation/RSMeanShift.h"
 #include "objrec/segmentation/RSGraphBased.h"
 #include "core/basics/numerictools.h"
+#include "core/basics/StringTools.h"
+#include "core/basics/FileName.h"
 #include "vislearning/baselib/ICETools.h"
 
 #include "core/basics/Timer.h"
@@ -114,6 +116,7 @@ SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *m
     cops.push_back ( new GlobalFeats() );
   useGradient = conf->gB ( featsec, "use_gradient", true );
   useRegionFeature = conf->gB ( featsec, "use_region", true );
+ 
   
   
   //define which featurextraction methods should be used for each channel
@@ -126,6 +129,14 @@ SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *m
   if(useGradient)
     colorchannels *= 2;
   
+  // geometric features of hoiem
+  useHoiemFeatures = conf->gB( featsec, "use_hoiem_features", false );
+  if ( useHoiemFeatures ) {
+    hoiemDirectory = conf->gS( featsec, "hoiem_directory" );
+    // FIXME: do we have to change colorchannels
+    // or change channelType??
+  }
+  
   for(int i = 0; i < colorchannels; i++)
   {
     channelType.push_back(0);
@@ -1180,6 +1191,53 @@ void SemSegContextTree::extractBasicFeatures ( NICE::MultiChannelImageT<double>
       NICE::FilterT<double>::sobel ( tmp, tmp2 );
     }
   }
+
+  // read the geometric cues produced by Hoiem et al. 
+  if ( useHoiemFeatures )
+  {
+    // we could also give the following set as a config option
+    string hoiemClasses_s = "sky 000 090-045 090-090 090-135 090 090-por 090-sol";
+    vector<string> hoiemClasses;
+    StringTools.split ( hoiemClasses, ' ', hoiemClasses );
+
+    // Now we have to do some fancy regular expressions :)
+    // Original image filename: basel_000083.jpg
+    // hoiem result: basel_000083_c_sky.png
+
+    // Fancy class of Ferid which supports string handling especially for filenames
+    FileName fn ( currentFile );
+    fn.removeExtension();
+    FileName fnBase = fn.extractFileName();
+
+    // counter for the channel index, starts with the current size of the destination multi-channel image
+    int currentChannel = feats.size();
+
+    // add a channel for each feature in advance
+    feats.addChannel ( hoiemClasses.size() );
+
+    // loop through all geometric categories and add the images
+    for ( vector<string>::const_iterator i = hoiemClasses.begin(); i != hoiemClasses.end(); i++, currentChannel++ )
+    {
+      string hoiemClass = *i;
+      FileName fnConfidenceImage ( hoiemDirectory + fnBase() + "_c_" + hoiemClass + ".png" );
+      if ( ! fnConfidenceImage.fileExists() )
+      {
+        fthrow(Exception, "Unable to read the Hoiem geometric confidence image: " << fnConfidenceImage() << " (original image is " << currentFile << ")" );
+      } else {
+        Image confidenceImage ( fnConfidenceImage() );
+        // check whether the image size is consistent
+        if ( confidenceImage.width() != feats.width() || confidenceImage.height() != feats.height() )
+        {
+          fthrow(Exception, "The size of the geometric confidence image does not match with the original image size: " << fnConfidenceImage());
+        }
+        ImageT<double> dst = feats[currentChannel];
+        // copy standard image to double image
+        for ( uint y = 0 ; y < confidenceImage.height(); y++ )
+          for ( uint x = 0 ; x < confidenceImage.width(); x++ )
+            dst.setPixel ( x, y, (double)confidenceImage.getPixel(x,y) );
+      }
+    }
+  }
 }
 
 void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult, NICE::MultiChannelImageT<double> & probabilities )

semseg/SemSegContextTree.h

@@ -114,8 +114,14 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
     bool useRegionFeature;
 
     /** how to handle each channel */
-    vector<int> channelType;
-    
+    std::vector<int> channelType;
+
+    /** whether we should use the geometric features of Hoeim (only offline computation with MATLAB supported) */
+    bool useHoiemFeatures;
+
+    /** directory of the geometric features */
+    std::string hoiemDirectory; 
+
   public:
     /** simple constructor */
     SemSegContextTree ( const NICE::Config *conf, const MultiDataset *md );