orientation ray feature & minor changes

progs/testSemanticSegmentation.cpp

@@ -122,10 +122,10 @@ int main ( int argc, char **argv )
   ResourceStatistics rs;
 
   /*-------------I/O CONFIGURATION-------------*/
+  bool postProcessing = conf.gB( "main", "post_process", false);
+  bool run_3Dseg = conf.gB( "SSContextTree", "run_3dseg", false);
   bool show_result = conf.gB ( "debug", "show_results", false );
   bool write_results = conf.gB ( "debug", "write_results", false );
-  bool run_3dseg = conf.gB ( "debug", "run_3dseg", true );
-  bool postProcessing = conf.gB( "debug", "post_process", false);
   string output_type = conf.gS ( "debug", "output_type", "ppm" );
   string output_postfix = conf.gS ( "debug", "output_postfix", "" );
   string resultdir = conf.gS ( "debug", "resultdir", "." );
@@ -160,7 +160,7 @@ int main ( int argc, char **argv )
 //   pb.show();
 
   vector< int > zsizeVec;
-  semseg->getDepthVector ( testFiles, zsizeVec, run_3dseg );
+  semseg->getDepthVector ( testFiles, zsizeVec, run_3Dseg );
 
   int depthCount = 0, idx = 0;
   vector< string > filelist;
@@ -202,7 +202,7 @@ int main ( int argc, char **argv )
       gt.addChannel ( lm_gt );
 
       int depthBoundary = 0;
-      if ( run_3dseg )
+      if ( run_3Dseg )
       {
         depthBoundary = zsizeVec[idx];
       }
@@ -233,7 +233,7 @@ int main ( int argc, char **argv )
             for ( int x = 0 ; x < segresult.width(); x++ )
             {
               lm.setPixel ( x, y, segresult.get ( x, y, ( uint ) z ) );
-              if ( run_3dseg )
+              if ( run_3Dseg )
                 lm_gt.setPixel ( x, y, gt.get ( x, y, ( uint ) z ) );
             }
           }

semseg/SemSegContextTree.cpp

@@ -42,6 +42,7 @@ SemSegContextTree::SemSegContextTree () : SemanticSegmentation ()
 {
   this->lfcw                = NULL;
   this->firstiteration      = true;
+  this->run3Dseg            = false;
   this->maxSamples          = 2000;
   this->minFeats            = 50;
   this->maxDepth            = 10;
@@ -83,6 +84,7 @@ SemSegContextTree::SemSegContextTree (
 
   this->lfcw                = new LFColorWeijer ( conf );
   this->firstiteration      = true;
+  this->run3Dseg            = conf->gB ( section, "run_3dseg", false );
   this->maxSamples          = conf->gI ( section, "max_samples", 2000 );
   this->minFeats            = conf->gI ( section, "min_feats", 50 );
   this->maxDepth            = conf->gI ( section, "max_depth", 10 );
@@ -316,8 +318,8 @@ void SemSegContextTree::initOperations()
     tops5.push_back ( new RayDiff() );
   if ( conf->gB ( featsec, "ray_dist", false ) )
     tops5.push_back ( new RayDist() );
-  if ( conf->gB ( featsec, "ray_theta", false ) )
-    tops5.push_back ( new RayTheta() );
+  if ( conf->gB ( featsec, "ray_orient", false ) )
+    tops5.push_back ( new RayOrient() );
   if ( conf->gB ( featsec, "ray_norm", false ) )
     tops5.push_back ( new RayNorm() );
 
@@ -433,9 +435,6 @@ double SemSegContextTree::getBestSplit (
     }
     while ( channelsPerType[ft].size() == 0 );
 
-    if ( firstiteration )
-      ft = 0;
-
     int tmpws = windowSize;
 
     if ( ft == 2 || ft == 4 )
@@ -644,41 +643,6 @@ inline double SemSegContextTree::getMeanProb (
   return val / ( double ) nbTrees;
 }
 
-void SemSegContextTree::closestContourPoint( const NICE::Image &B,
-                                             const NICE::Image &G,
-                                             NICE::Matrix &D,
-                                             NICE::Matrix &N,
-                                             int x,
-                                             int y,
-                                             const int &dir)
-{
-  int xo = 0, yo = 0; // offsets
-
-  switch (dir)
-  {
-    case 0: yo = -1; break;
-    case 1: xo = 1; yo = -1; break;
-    case 2: xo = 1; x = (B.width()-1)-x; break;
-    case 3: xo = 1; yo = -1; break;
-    case 4: yo = 1; y = (B.height()-1)-y; break;
-    case 5: xo = -1 ; yo = 1; y = (B.height()-1)-y; break;
-    case 6: xo = -1; break;
-    case 7: xo = -1; yo = -1; break;
-    default: return;
-  }
-
-  if (B.getPixelQuick(x,y) != 0 || x+xo < 0 || x+xo >= B.width() || y+yo < 0 || y+yo >= B.height() )
-  {
-    D(x, y) = 0;
-    N(x, y) = G.getPixelQuick(x,y);
-  }
-  else
-  {
-    D(x, y) = D(x+xo,y+yo) + 1;
-    N(x, y) = N(x+xo,y+yo);
-  }
-}
-
 void SemSegContextTree::computeRayFeatImage (
     NICE::MultiChannelImage3DT<double> &feats,
     int firstChannel )
@@ -693,7 +657,6 @@ void SemSegContextTree::computeRayFeatImage (
   for ( int z = 0; z < zsize; z++)
   {
     // canny image from raw channel
-    NICE::Image gra = feats.getChannel( z, 1 );
     NICE::Image med (xsize,ysize);
     NICE::median ( feats.getChannel( z, 0 ), &med, 2);
     NICE::Image* can = NICE::canny( med, 5, 25);
@@ -702,10 +665,49 @@ void SemSegContextTree::computeRayFeatImage (
     {
       NICE::Matrix dist(xsize,ysize,0);
       NICE::Matrix norm(xsize,ysize,0);
+      NICE::Matrix orient(xsize,ysize,0);
 
       for (int y = 0; y < ysize; y++)
         for ( int x = 0; x < xsize; x++)
-          this->closestContourPoint(*(can), gra, dist, norm, x, y, dir);
+        {
+          int xo = 0, yo = 0; // offsets
+          int theta = 0;
+
+          switch (dir)
+          {
+            case 0: theta =   0; yo = -1; break;
+            case 1: theta =  45; xo =  1; yo = -1; break;
+            case 2: theta =  90; xo =  1; x = (xsize-1)-x; break;
+            case 3: theta = 135; xo =  1; yo = -1; break;
+            case 4: theta = 180; yo =  1; y = (ysize-1)-y; break;
+            case 5: theta = 225; xo = -1; yo = 1; y = (ysize-1)-y; break;
+            case 6: theta = 270; xo = -1; break;
+            case 7: theta = 315; xo = -1; yo = -1; break;
+            default: return;
+          }
+
+          if (can->getPixelQuick(x,y) != 0
+              || x+xo < 0
+              || x+xo >= xsize
+              || y+yo < 0
+              || y+yo >= ysize )
+          {
+            double gx = feats.get(x, y, z, 1);
+            double gy = feats.get(x, y, z, 2);
+
+            //double go = atan2 (gy, gx);
+
+            norm(x, y)   = sqrt(gx*gx+gy*gy);
+            orient(x, y) = ( gx*cos(theta)+gy*sin(theta) ) / norm(x,y);
+            dist(x, y)   = 0;
+          }
+          else
+          {
+            orient(x, y) = orient(x+xo,y+yo);
+            norm(x, y)   = norm(x+xo,y+yo);
+            dist(x, y)   = dist(x+xo,y+yo) + 1;
+          }
+        }
 
       for (int y = 0; y < ysize; y++)
         for (int x = 0; x < xsize; x++)
@@ -714,6 +716,8 @@ void SemSegContextTree::computeRayFeatImage (
           feats.set( x, y, z, dist(x,y), firstChannel + dir );
           // norm feature maps
           feats.set( x, y, z, norm(x,y), firstChannel + amountDirs + dir );
+          // orientation feature maps
+          feats.set( x, y, z, norm(x,y), firstChannel + (amountDirs*2) + dir );
         }
     }
 
@@ -828,9 +832,8 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
   Timer timer;
   timer.start();
   
-  bool run_3dseg = conf->gB ( "debug", "run_3dseg", true );
   vector<int> zsizeVec;
-  getDepthVector ( trainp, zsizeVec, run_3dseg );
+  getDepthVector ( trainp, zsizeVec, run3Dseg );
 
   //FIXME: memory usage
   vector<MultiChannelImage3DT<double> > allfeats;
@@ -857,7 +860,7 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
     rawChannels = 1;
 
   if ( useGradient )
-    rawChannels *= 2;
+    rawChannels *= 3;
 
   if ( useWeijer )
     rawChannels += 11;
@@ -905,7 +908,7 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
       fprintf ( stderr, "SSContext: Collecting pixel examples from localization info: %s\n", file.c_str() );
 
       int depthBoundary = 0;
-      if ( run_3dseg )
+      if ( run3Dseg )
       {
         depthBoundary = zsizeVec[imgCounter];
       }
@@ -956,7 +959,7 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
             if ( useFeat1 )
               rSize[imgCounter][allfeats[imgCounter] ( x, y, z, rawChannels ) ]++;
 
-            if ( run_3dseg )
+            if ( run3Dseg )
               classno = pixelLabels ( x, y, ( uint ) z );
             else
               classno = pL.getPixelQuick ( x,y );
@@ -994,7 +997,6 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
 ///////////////////////////////////////////////////////////////////////////////
 
   // Type 0: single pixel & pixel-comparison features on gray value channels
-  if ( useFeat0 )
   for ( int i = 0; i < rawChannels; i++ )
     channelType.push_back ( 0 );
 
@@ -1029,7 +1031,7 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
 
   // Type 5: ray features for shape modeling on canny-map
   if ( useFeat5 )
-    for ( int i = 0; i < 16; i++ )
+    for ( int i = 0; i < 24; i++ )
       channelType.push_back ( 5 );
 
   // 'amountTypes' sets upper bound for usable feature types
@@ -1140,6 +1142,29 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
       }
     }
 
+    //compute integral & ray images
+    int multi = 0, multi2 = 0;
+    if (useFeat3) multi=1;
+    if (useFeat4) multi=2;
+    if (useFeat5) multi2 = 1;
+
+    if ( firstiteration && ( useFeat2 || useFeat3 || useFeat4) )
+      for ( int i = 0; i < imgCounter; i++ )
+        allfeats[i].addChannel ( (multi*classes) + rawChannels );
+
+    if ( firstiteration && useFeat5 )
+      for ( int i = 0; i < imgCounter; i++ )
+        allfeats[i].addChannel ( 24 );
+
+    int firstChannel = channelType.size() - (multi*classes) - (multi2*24);
+    if ( useFeat2 || useFeat3 || useFeat4 )
+      for ( int i = 0; i < imgCounter; i++ )
+        computeIntegralImage ( nodeIndices[i], allfeats[i], firstChannel );
+
+    if ( firstiteration && useFeat5 )
+      for ( int i = 0; i < imgCounter; i++ )
+        computeRayFeatImage( allfeats[i], channelType.size()-24);
+
 #ifdef VERBOSE
     Timer timerDepth;
     timerDepth.start();
@@ -1367,29 +1392,6 @@ void SemSegContextTree::train ( const LabeledSet * trainp )
       }
     }
 
-    //compute integral & ray images
-    int multi = 0, multi2 = 0;
-    if (useFeat3) multi=1;
-    if (useFeat4) multi=2;
-    if (useFeat5) multi2 = 1;
-
-    if ( firstiteration && (useFeat3 || useFeat4) )
-      for ( int i = 0; i < imgCounter; i++ )
-        allfeats[i].addChannel ( (multi*classes) + rawChannels );
-
-    if ( firstiteration && useFeat5 )
-      for ( int i = 0; i < imgCounter; i++ )
-        allfeats[i].addChannel ( 16 );
-
-    int firstChannel = channelType.size() - (multi*classes) - (multi2*16);
-    if ( useFeat3 || useFeat4 )
-      for ( int i = 0; i < imgCounter; i++ )
-        computeIntegralImage ( nodeIndices[i], allfeats[i], firstChannel );
-
-    if ( firstiteration && useFeat5 )
-      for ( int i = 0; i < imgCounter; i++ )
-        computeRayFeatImage( allfeats[i], channelType.size()-16);
-
     if ( firstiteration ) firstiteration = false;
 
 #ifdef VERBOSE
@@ -1555,17 +1557,22 @@ void SemSegContextTree::addFeatureMaps (
   if ( useGradient )
   {
     int currentsize = imgData.channels();
-    imgData.addChannel ( currentsize );
+    imgData.addChannel ( 2*currentsize );
 
     for ( int z = 0; z < zsize; z++ )
       for ( int c = 0; c < currentsize; c++ )
       {
         ImageT<double> tmp = imgData.getChannelT(z, c);
-        ImageT<double> tmp2( xsize, ysize );
-        NICE::FilterT<double, double, double>::gradientStrength ( tmp, tmp2 );
+        ImageT<double> sobX( xsize, ysize );
+        ImageT<double> sobY( xsize, ysize );
+        NICE::FilterT<double, double, double>::sobelX ( tmp, sobX );
+        NICE::FilterT<double, double, double>::sobelY ( tmp, sobY );
         for ( int y = 0; y < ysize; y++ )
           for ( int x = 0; x < xsize; x++ )
-            imgData.set(x, y, z, tmp2.getPixelQuick(x,y), c+currentsize);
+          {
+            imgData.set( x, y, z, sobX.getPixelQuick(x,y), c+currentsize );
+            imgData.set( x, y, z, sobY.getPixelQuick(x,y), c+(currentsize*2) );
+          }
       }
   }
 
@@ -1745,6 +1752,29 @@ void SemSegContextTree::classify (
     depth++;
     vector<vector<double> > lastRegionProbs = regionProbs;
 
+    int multi = 0, multi2 = 0;
+    if (useFeat3) multi=1;
+    if (useFeat4) multi=2;
+    if (useFeat5) multi2=1;
+
+    if ( depth < maxDepth )
+    {
+      //compute integral images
+      if ( firstiteration && ( useFeat2 || useFeat3 || useFeat4 ) )
+        imgData.addChannel ( (multi*classes) + rawChannels );
+
+      //compute canny image
+      if ( firstiteration && useFeat5 )
+        imgData.addChannel ( 24 );
+
+      int firstChannel = channelType.size() - (multi*classes) - (multi2*24);
+      if ( useFeat2 || useFeat3 || useFeat4 )
+        computeIntegralImage ( nodeIndices, imgData, firstChannel );
+
+      if ( firstiteration && useFeat5 )
+        computeRayFeatImage( imgData, channelType.size()-24 );
+    }
+
     if ( useFeat1 )
     {
       int numRegions = ( int ) regionProbs.size();
@@ -1854,30 +1884,7 @@ void SemSegContextTree::classify (
       }
     }
 
-    int multi = 0, multi2 = 0;
-    if (useFeat3) multi=1;
-    if (useFeat4) multi=2;
-    if (useFeat5) multi2=1;
-
-    if ( depth < maxDepth )
-    {
-      //compute integral images
-      if ( firstiteration && ( useFeat3 || useFeat4 ) )
-        imgData.addChannel ( (multi*classes) + rawChannels );
-
-      //compute canny image
-      if ( firstiteration && useFeat5 )
-        imgData.addChannel ( 16 );
-
-      int firstChannel = channelType.size() - (multi*classes) - (multi2*16);
-      if ( useFeat3 || useFeat4 )
-        computeIntegralImage ( nodeIndices, imgData, firstChannel );
-
-      if ( firstiteration && useFeat5 )
-        computeRayFeatImage( imgData, channelType.size()-16 );
-
-      if ( firstiteration ) firstiteration = false;
-    }
+    if ( (depth < maxDepth) && firstiteration ) firstiteration = false;
   }
 
   vector<int> classesInImg;

semseg/SemSegContextTree.h

@@ -35,6 +35,9 @@ private:
   /** local features */
   LFColorWeijer *lfcw;
 
+  /** whether to run in 3D mode or not */
+  bool run3Dseg;
+
   /** whether to use a particular feature type or not */
   bool useFeat0, useFeat1, useFeat2, useFeat3, useFeat4, useFeat5;
 
@@ -160,24 +163,6 @@ private:
    */
   void train ( const LabeledSet * trainp );
 
-  /**
-   * @brief compute closest contour point in given direction
-   * @param B     contour image (canny)
-   * @param G     gradient image
-   * @param D     distances
-   * @param N     norms
-   * @param x     current x coordinate
-   * @param y     current y coordinate
-   * @param dir   direction
-   */
-  void closestContourPoint ( const NICE::Image &B,
-                             const NICE::Image &G,
-                             NICE::Matrix &D,
-                             NICE::Matrix &N,
-                             int x,
-                             int y,
-                             const int &dir);
-
 public:
   /** simple constructor */
   SemSegContextTree ();
@@ -205,6 +190,11 @@ public:
    */
   void train ( const MultiDataset *md );
 
+  bool active3DMode ()
+  {
+    return run3Dseg;
+  }
+
   /**
    * @brief computes integral image of given feats
    *

semseg/operations/Operations.cpp

@@ -398,8 +398,8 @@ double Haar3Stack::getVal ( const Features &feats, const int &x, const int &y, c
 double RayDiff::getVal( const Features &feats, const int &x, const int &y, const int &z )
 {
 
-  double dist1 = feats.feats->get( x, y, z, feats.feats->channels()-16+z1 );
-  double dist2 = feats.feats->get( x, y, z, feats.feats->channels()-16+z2 );
+  double dist1 = feats.feats->get( x, y, z, feats.feats->channels()-24+z1 );
+  double dist2 = feats.feats->get( x, y, z, feats.feats->channels()-24+z2 );
 
   if (dist1 != 0)
     return (dist1-dist2)/dist1;
@@ -409,15 +409,15 @@ double RayDiff::getVal( const Features &feats, const int &x, const int &y, const
 
 double RayDist::getVal( const Features &feats, const int &x, const int &y, const int &z )
 {
-  return feats.feats->get( x, y, z, feats.feats->channels()-16+z1 );
+  return feats.feats->get( x, y, z, feats.feats->channels()-24+z1 );
 }
 
-double RayTheta::getVal( const Features &feats, const int &x, const int &y, const int &z )
+double RayNorm::getVal( const Features &feats, const int &x, const int &y, const int &z )
 {
-  return 0.0;
+  return feats.feats->get( x, y, z, feats.feats->channels()-16+z1 );
 }
 
-double RayNorm::getVal( const Features &feats, const int &x, const int &y, const int &z )
+double RayOrient::getVal( const Features &feats, const int &x, const int &y, const int &z )
 {
   return feats.feats->get( x, y, z, feats.feats->channels()-8+z1 );
 }

semseg/operations/Operations.h

@@ -51,7 +51,7 @@ enum OperationTypes {
   EQUALITY,
   RAYDIFF,
   RAYDIST,
-  RAYTHETA,
+  RAYORIENT,
   RAYNORM,
   MINUS_C,
   MINUSABS_C,
@@ -1525,7 +1525,7 @@ class RayDist: public Operation
 /**
  * @brief Ray Orientation features
  */
-class RayTheta: public Operation
+class RayOrient: public Operation
 {
   public:
     /**
@@ -1543,7 +1543,7 @@ class RayTheta: public Operation
      **/
     virtual Operation* clone()
     {
-      return new RayTheta();
+      return new RayOrient();
     }
 
     /**
@@ -1552,7 +1552,7 @@ class RayTheta: public Operation
      **/
     virtual std::string writeInfos()
     {
-      return "(-)RayTheta             " + Operation::writeInfos();
+      return "(-)RayOrient            " + Operation::writeInfos();
     }
 
     /**
@@ -1561,7 +1561,7 @@ class RayTheta: public Operation
      **/
     virtual OperationTypes getOps()
     {
-      return RAYTHETA;
+      return RAYORIENT;
     }
 };