current

semseg/SemSegContextTree.cpp

@@ -36,6 +36,7 @@ SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *m
   this->conf = conf;
   string section = "SSContextTree";
   lfcw = new LFColorWeijer ( conf );
+  firstiteration = true;
 
   grid = conf->gI ( section, "grid", 10 );
 
@@ -62,12 +63,14 @@ SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *m
 
   pixelWiseLabeling = false;
 
+  useRegionFeature = true;
   if ( segmentationtype == "meanshift" )
     segmentation = new RSMeanShift ( conf );
   else if ( segmentationtype == "none" )
   {
     segmentation = NULL;
     pixelWiseLabeling = true;
+    useRegionFeature = false;
   }
   else if ( segmentationtype == "felzenszwalb" )
     segmentation = new RSGraphBased ( conf );
@@ -78,53 +81,85 @@ SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *m
 
   string featsec = "Features";
 
+  vector<Operation*> tops;
+
   if ( conf->gB ( featsec, "minus", true ) )
-    ops.push_back ( new Minus() );
+    tops.push_back ( new Minus() );
   if ( conf->gB ( featsec, "minus_abs", true ) )
-    ops.push_back ( new MinusAbs() );
+    tops.push_back ( new MinusAbs() );
   if ( conf->gB ( featsec, "addition", true ) )
-    ops.push_back ( new Addition() );
+    tops.push_back ( new Addition() );
   if ( conf->gB ( featsec, "only1", true ) )
-    ops.push_back ( new Only1() );
+    tops.push_back ( new Only1() );
   if ( conf->gB ( featsec, "rel_x", true ) )
-    ops.push_back ( new RelativeXPosition() );
+    tops.push_back ( new RelativeXPosition() );
   if ( conf->gB ( featsec, "rel_y", true ) )
-    ops.push_back ( new RelativeYPosition() );
+    tops.push_back ( new RelativeYPosition() );
+
+  ops.push_back ( tops );
+
+  tops.clear();
+  tops.push_back ( new Equality() );
+  ops.push_back ( tops );
 
+  tops.clear();
   if ( conf->gB ( featsec, "bi_int_cent", true ) )
-    cops.push_back ( new BiIntegralCenteredOps() );
+    tops.push_back ( new BiIntegralCenteredOps() );
   if ( conf->gB ( featsec, "int_cent", true ) )
-    cops.push_back ( new IntegralCenteredOps() );
+    tops.push_back ( new IntegralCenteredOps() );
   if ( conf->gB ( featsec, "int", true ) )
-    cops.push_back ( new IntegralOps() );
+    tops.push_back ( new IntegralOps() );
   if ( conf->gB ( featsec, "haar_horz", true ) )
-    cops.push_back ( new HaarHorizontal() );
+    tops.push_back ( new HaarHorizontal() );
   if ( conf->gB ( featsec, "haar_vert", true ) )
-    cops.push_back ( new HaarVertical() );
+    tops.push_back ( new HaarVertical() );
   if ( conf->gB ( featsec, "haar_diag", true ) )
-    cops.push_back ( new HaarDiag() );
+    tops.push_back ( new HaarDiag() );
   if ( conf->gB ( featsec, "haar3_horz", true ) )
-    cops.push_back ( new Haar3Horiz() );
+    tops.push_back ( new Haar3Horiz() );
   if ( conf->gB ( featsec, "haar3_vert", true ) )
-    cops.push_back ( new Haar3Vert() );
+    tops.push_back ( new Haar3Vert() );
   if ( conf->gB ( featsec, "glob", true ) )
-    cops.push_back ( new GlobalFeats() );
+    tops.push_back ( new GlobalFeats() );
+
+  ops.push_back ( tops );
+  ops.push_back ( tops );
+
+  tops.clear();
+  if ( conf->gB ( featsec, "minus", true ) )
+    tops.push_back ( new Minus() );
+  if ( conf->gB ( featsec, "minus_abs", true ) )
+    tops.push_back ( new MinusAbs() );
+  if ( conf->gB ( featsec, "addition", true ) )
+    tops.push_back ( new Addition() );
+  if ( conf->gB ( featsec, "only1", true ) )
+    tops.push_back ( new Only1() );
+  if ( conf->gB ( featsec, "rel_x", true ) )
+    tops.push_back ( new RelativeXPosition() );
+  if ( conf->gB ( featsec, "rel_y", true ) )
+    tops.push_back ( new RelativeYPosition() );
+
+  ops.push_back ( tops );
+
   useGradient = conf->gB ( featsec, "use_gradient", true );
-  useRegionFeature = conf->gB ( featsec, "use_region", true );
- 
+
   // geometric features of hoiem
-  useHoiemFeatures = conf->gB( featsec, "use_hoiem_features", false );
+  useHoiemFeatures = conf->gB ( featsec, "use_hoiem_features", false );
   if ( useHoiemFeatures )
   {
-    hoiemDirectory = conf->gS( featsec, "hoiem_directory" );
+    hoiemDirectory = conf->gS ( featsec, "hoiem_directory" );
   }
 
   opOverview = vector<int> ( NBOPERATIONS, 0 );
   contextOverview = vector<vector<double> > ( maxDepth, vector<double> ( 2, 0.0 ) );
 
+  calcVal.push_back ( new MCImageAccess() );
+  calcVal.push_back ( new MCImageAccess() );
+  calcVal.push_back ( new MCImageAccess() );
   calcVal.push_back ( new MCImageAccess() );
   calcVal.push_back ( new ClassificationResultAccess() );
 
+
   classnames = md->getClassNames ( "train" );
 
   ///////////////////////////////////
@@ -151,7 +186,7 @@ SemSegContextTree::~SemSegContextTree()
 {
 }
 
-double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<NICE::MultiChannelImageT<unsigned short int> > &currentfeats, std::vector<NICE::MultiChannelImageT<double> > &integralImgs, const std::vector<NICE::MatrixT<int> > &labels, int node, Operation *&splitop, double &splitval, const int &tree )
+double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<NICE::MultiChannelImageT<unsigned short int> > &currentfeats, const std::vector<NICE::MatrixT<int> > &labels, int node, Operation *&splitop, double &splitval, const int &tree )
 {
   Timer t;
   t.start();
@@ -252,11 +287,17 @@ double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<do
 
     int tmpws = windowSize;
 
-    if ( integralImgs[0].width() == 0 )
+    if ( ft > 1 && firstiteration )
       ft = 0;
 
-    if ( ft > 0 )
+    if ( channelsPerType[ft].size() == 0 )
     {
+      ft = 0;
+    }
+
+    if ( ft > 1 )
+    {
+      //use larger window size for context features
       tmpws *= 4;
     }
 
@@ -265,50 +306,26 @@ double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<do
     y1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
     y2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
 
-    if ( ft == 0 )
-    {
-      int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) featdim );
-      int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) featdim );
-      int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ops.size() );
-      Operation *op = ops[o]->clone();
-      op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
-      op->setContext ( false );
-      featsel.push_back ( op );
-    }
-    else if ( ft == 1 )
-    {
-      int opssize = ( int ) ops.size();
-      //opssize = 0;
-      int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( ( ( double ) cops.size() ) + ( double ) opssize ) );
+    int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) channelsPerType[ft].size() );
+    int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) channelsPerType[ft].size() );
+    int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ops[ft].size() );
 
-      Operation *op;
+    Operation *op = ops[ft][o]->clone();
 
-      if ( o < opssize )
-      {
-        int chans = ( int ) forest[0][0].dist.size();
-        int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
-        int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
-        op = ops[o]->clone();
-        op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
-        op->setContext ( true );
-      }
-      else
-      {
-        int chans = integralImgs[0].channels();
-        int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
-        int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
-
-        o -= opssize;
-        op = cops[o]->clone();
-        op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
-        if ( f1 < (int)forest[0][0].dist.size() )
-          op->setContext ( true );
-        else
-          op->setContext ( false );
-      }
+    op->set ( x1, y1, x2, y2, channelsPerType[ft][f1], channelsPerType[ft][f2], calcVal[ft] );
 
-      featsel.push_back ( op );
-    }
+    //if( i < 10)
+    //cerr << "type: " << ft << " features: " << op->writeInfos() << endl;
+
+    if ( ft == 3 || ft == 4 )
+      op->setContext ( true );
+    else
+      op->setContext ( false );
+
+    //if(ft == 4)
+    //cerr << "type: " << ft << " features: " << op->writeInfos() << endl;
+
+    featsel.push_back ( op );
   }
 
 #pragma omp parallel for private(mapit)
@@ -328,7 +345,6 @@ double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<do
       feat.cfeats = &currentfeats[ ( *it ) [0]];
       feat.cTree = tree;
       feat.tree = &forest[tree];
-      feat.integralImg = &integralImgs[ ( *it ) [0]];
       double val = featsel[f]->getVal ( feat, ( *it ) [1], ( *it ) [2] );
       vals.push_back ( val );
       maxval = std::max ( val, maxval );
@@ -489,65 +505,38 @@ void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<Sp
   }
 }
 
-void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<unsigned short int> &currentfeats, const NICE::MultiChannelImageT<double> &lfeats, NICE::MultiChannelImageT<double> &integralImage )
+void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<unsigned short int> &currentfeats, NICE::MultiChannelImageT<double> &feats, int firstChannel )
 {
+
   int xsize = currentfeats.width();
   int ysize = currentfeats.height();
 
-  int channels = ( int ) forest[0][0].dist.size();
-#pragma omp parallel for
-  for ( int c = 0; c < channels; c++ )
-  {
-    integralImage.set ( 0, 0, getMeanProb ( 0, 0, c, currentfeats ), c );
-
-    //first column
+  xsize = feats.width();
+  ysize = feats.height();
 
-    for ( int y = 1; y < ysize; y++ )
-    {
-      integralImage.set ( 0, y, getMeanProb ( 0, y, c, currentfeats ) + integralImage.get ( 0, y - 1, c ), c );
-    }
-
-    //first row
-    for ( int x = 1; x < xsize; x++ )
-    {
-      integralImage.set ( x, 0, getMeanProb ( x, 0, c, currentfeats ) + integralImage.get ( x - 1, 0, c ), c );
-    }
-
-    //rest
-    for ( int y = 1; y < ysize; y++ )
-    {
-      for ( int x = 1; x < xsize; x++ )
-      {
-        double val = getMeanProb ( x, y, c, currentfeats ) + integralImage.get ( x, y - 1, c ) + integralImage.get ( x - 1, y, c ) - integralImage.get ( x - 1, y - 1, c );
-        integralImage.set ( x, y, val, c );
-      }
-    }
-  }
-
-  int channels2 = ( int ) lfeats.channels();
-
-  xsize = lfeats.width();
-  ysize = lfeats.height();
-
-  if ( integralImage.get ( xsize - 1, ysize - 1, channels ) == 0.0 )
+  if ( firstiteration )
   {
+    firstiteration = false;
 #pragma omp parallel for
-    for ( int c1 = 0; c1 < channels2; c1++ )
+    for ( int it = 0; it < integralMap.size(); it++ )
     {
-      int c = channels + c1;
-      integralImage.set ( 0, 0, lfeats.get ( 0, 0, c1 ), c );
+      int c1 = integralMap[it].first;
+      int c = integralMap[it].second;
 
-      //first column
+      feats( 0, 0, c ) = feats( 0, 0, c1 );
 
+      //first column
       for ( int y = 1; y < ysize; y++ )
       {
-        integralImage.set ( 0, y, lfeats.get ( 0, y, c1 ) + integralImage.get ( 0, y, c ), c );
+        feats( 0, y, c ) = feats.get ( 0, y, c1 )
+                         + feats.get ( 0, y-1, c );
       }
 
       //first row
       for ( int x = 1; x < xsize; x++ )
       {
-        integralImage.set ( x, 0, lfeats.get ( x, 0, c1 ) + integralImage.get ( x, 0, c ), c );
+        feats( x, 0, c ) = feats.get ( x, 0, c1 ) 
+                         + feats.get ( x-1, 0, c );
       }
 
       //rest
@@ -555,12 +544,48 @@ void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<un
       {
         for ( int x = 1; x < xsize; x++ )
         {
-          double val = lfeats.get ( x, y, c1 ) + integralImage.get ( x, y - 1, c ) + integralImage.get ( x - 1, y, c ) - integralImage.get ( x - 1, y - 1, c );
-          integralImage.set ( x, y, val, c );
+          feats ( x, y, c ) = feats ( x, y, c1 ) 
+                            + feats ( x, y - 1, c ) 
+                            + feats ( x - 1, y, c ) 
+                            - feats ( x - 1, y - 1, c );
         }
       }
     }
   }
+
+  int channels = ( int ) forest[0][0].dist.size();
+  
+#pragma omp parallel for
+  for ( int c = 0; c < channels; c++ )
+  {
+    feats ( 0, 0, firstChannel + c ) = getMeanProb ( 0, 0, c, currentfeats );
+
+    //first column
+    for ( int y = 1; y < ysize; y++ )
+    {
+      feats ( 0, y, firstChannel + c ) = getMeanProb ( 0, y, c, currentfeats )
+                                         + feats ( 0, y - 1, firstChannel + c );
+    }
+
+    //first row
+    for ( int x = 1; x < xsize; x++ )
+    {
+      feats ( x, 0, firstChannel + c ) = getMeanProb ( x, 0, c, currentfeats )
+                                         + feats ( x - 1, 0, firstChannel + c );
+    }
+
+    //rest
+    for ( int y = 1; y < ysize; y++ )
+    {
+      for ( int x = 1; x < xsize; x++ )
+      {
+        feats ( x, y, firstChannel + c ) = getMeanProb ( x, y, c, currentfeats )
+                                           + feats ( x, y - 1, firstChannel + c )
+                                           + feats ( x - 1, y, firstChannel + c )
+                                           - feats ( x - 1, y - 1, firstChannel + c );
+      }
+    }
+  }
 }
 
 inline double computeWeight ( const double &d, const double &dim )
@@ -642,13 +667,13 @@ void SemSegContextTree::train ( const MultiDataset *md )
     }
 
     Globals::setCurrentImgFN ( currentFile );
-    
+
     //TODO: resize image?!
     MultiChannelImageT<double> feats;
     allfeats.push_back ( feats );
-    
+
     // read image and do some simple transformations
-    extractBasicFeatures (allfeats[imgcounter], img, currentFile);
+    extractBasicFeatures ( allfeats[imgcounter], img, currentFile );
 
     // getting groundtruth
     NICE::Image pixelLabels ( xsize, ysize );
@@ -693,41 +718,67 @@ void SemSegContextTree::train ( const MultiDataset *md )
 ////////////////////////////////////////////////////
   //define which featurextraction methods should be used for each channel
 #ifdef LOCALFEATS
-  int colorchannels = 9;
+  rawChannels = 9;
 #else
-  int colorchannels = 3;
+  rawChannels = 3;
 #endif
-  
-  if(useGradient)
-    colorchannels *= 2;
-  
-  // gray value images 
-  for(int i = 0; i < colorchannels; i++)
+
+  // how many channels without integral image
+  int shift = 0;
+
+  if ( useGradient )
+    rawChannels *= 2;
+
+  if ( useHoiemFeatures )
+    rawChannels += 8;
+
+  // gray value images
+  for ( int i = 0; i < rawChannels; i++ )
   {
-    channelType.push_back(0);
+    channelType.push_back ( 0 );
   }
-  
+
   // regions
-  if(useRegionFeature)
-    channelType.push_back(2);
-  
+  if ( useRegionFeature )
+  {
+    channelType.push_back ( 1 );
+    shift++;
+  }
+
+  for ( int i = 0; i < rawChannels; i++ )
+  {
+    channelType.push_back ( 2 );
+  }
+
   // integral images
-  for(int i = 0; i < colorchannels+classes; i++)
+  for ( int i = 0; i < classes; i++ )
   {
-    channelType.push_back(1);
+    channelType.push_back ( 3 );
   }
-  
-  int amountTypes = 3;
-  
-  channelsPerType = vector<vector<int> >(amountTypes, vector<int>());
-  
-  for(int i = 0; i < channelType.size(); i++)
+
+  integralMap.clear();
+  int integralImageAmount = rawChannels;
+  for ( int ii = 0; ii < integralImageAmount; ii++ )
   {
-    channelsPerType[channelType[i]].push_back(i);
+    integralMap.push_back ( pair<int, int> ( ii, ii + integralImageAmount + shift ) );
   }
-  
-  ftypes = std::min(amountTypes,ftypes);
-  
+
+  int amountTypes = 5;
+
+  channelsPerType = vector<vector<int> > ( amountTypes, vector<int>() );
+
+  for ( int i = 0; i < channelType.size(); i++ )
+  {
+    channelsPerType[channelType[i]].push_back ( i );
+  }
+
+  for ( int i = 0; i < classes; i++ )
+  {
+    channelsPerType[channelsPerType.size()-1].push_back ( i );
+  }
+
+  ftypes = std::min ( amountTypes, ftypes );
+
 ////////////////////////////////////////////////////
 
   //balancing
@@ -789,8 +840,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
   bool allleaf = false;
   //int baseFeatSize = allfeats[0].size();
 
-  vector<MultiChannelImageT<double> > integralImgs ( imgcounter, MultiChannelImageT<double>() );
-
   while ( !allleaf && depth < maxDepth )
   {
     depth++;
@@ -823,14 +872,9 @@ void SemSegContextTree::train ( const MultiDataset *md )
       {
         if ( !forest[tree][i].isleaf && forest[tree][i].left < 0 )
         {
-#if 0
-          timer.stop();
-          cout << "time 1: " << timer.getLast() << endl;
-          timer.start();
-#endif
           Operation *splitfeat = NULL;
           double splitval;
-          double bestig = getBestSplit ( allfeats, lastfeats, integralImgs, labels, i, splitfeat, splitval, tree );
+          double bestig = getBestSplit ( allfeats, lastfeats, labels, i, splitfeat, splitval, tree );
 
           for ( int ii = 0; ii < lastfeats.size(); ii++ )
           {
@@ -838,22 +882,9 @@ void SemSegContextTree::train ( const MultiDataset *md )
             {
               short unsigned int minv, maxv;
               lastfeats[ii].statistics ( minv, maxv, c );
-              //cout << "min: " << minv << " max: " << maxv << endl;
             }
           }
 
-#if 0
-          timer.stop();
-          double tl = timer.getLast();
-
-          if ( tl > 10.0 )
-          {
-            cout << "time 2: " << tl << endl;
-            cout << "slow split: " << splitfeat->writeInfos() << endl;
-            getchar();
-          }
-          timer.start();
-#endif
           forest[tree][i].feat = splitfeat;
           forest[tree][i].decision = splitval;
 
@@ -880,12 +911,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
             uniquenumber++;
             forest[tree][right].featcounter = 0;
 
-#if 0
-            timer.stop();
-            cout << "time 3: " << timer.getLast() << endl;
-            timer.start();
-#endif
-
 #pragma omp parallel for
             for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
             {
@@ -903,7 +928,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
                     feat.cfeats = &lastfeats[iCounter];
                     feat.cTree = tree;
                     feat.tree = &forest[tree];
-                    feat.integralImg = &integralImgs[iCounter];
                     double val = splitfeat->getVal ( feat, x, y );
 
                     int subx = x / grid;
@@ -939,12 +963,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
                 }
               }
             }
-#if 0
-            timer.stop();
-            cout << "time 4: " << timer.getLast() << endl;
-            timer.start();
-#endif
-//            forest[tree][right].featcounter = forest[tree][i].featcounter - forest[tree][left].featcounter;
 
             double lcounter = 0.0, rcounter = 0.0;
 
@@ -963,11 +981,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
                 rcounter += forest[tree][right].dist[d];
               }
             }
-#if 0
-            timer.stop();
-            cout << "time 5: " << timer.getLast() << endl;
-            timer.start();
-#endif
+
             if ( lcounter <= 0 || rcounter <= 0 )
             {
               cout << "lcounter : " << lcounter << " rcounter: " << rcounter << endl;
@@ -995,7 +1009,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
                       feat.cfeats = &lastfeats[iCounter];
                       feat.cTree = tree;
                       feat.tree = &forest[tree];
-                      feat.integralImg = &integralImgs[iCounter];
 
                       double val = splitfeat->getVal ( feat, x, y );
 
@@ -1020,36 +1033,22 @@ void SemSegContextTree::train ( const MultiDataset *md )
           }
         }
       }
-#if 0
-      timer.stop();
-      cout << "time after tree: " << timer.getLast() << endl;
-      timer.start();
-#endif
     }
 
     //compute integral images
-    int channels = classes + allfeats[0].channels();
-
-    if ( integralImgs[0].width() == 0 )
+    if ( firstiteration )
     {
       for ( int i = 0; i < imgcounter; i++ )
       {
-        int xsize = allfeats[i].width();
-        int ysize = allfeats[i].height();
-        integralImgs[i].reInit ( xsize, ysize, channels );
-        integralImgs[i].setAll ( 0.0 );
+        int pos = allfeats.size();
+        allfeats[i].addChannel ( ( int ) ( classes + rawChannels ) );
       }
     }
-#if 0
-    timer.stop();
-    cout << "time for part1: " << timer.getLast() << endl;
-    timer.start();
-#endif
 
 #pragma omp parallel for
     for ( int i = 0; i < imgcounter; i++ )
     {
-      computeIntegralImage ( currentfeats[i], allfeats[i], integralImgs[i] );
+      computeIntegralImage ( currentfeats[i], allfeats[i], channelType.size() - classes );
 #ifdef TEXTONMAP
       computeIntegralImage ( textonMap[i], integralTexton[i] );
 #endif
@@ -1122,12 +1121,10 @@ void SemSegContextTree::train ( const MultiDataset *md )
 
   for ( uint c = 0; c < ops.size(); c++ )
   {
-    cout << ops[c]->writeInfos() << ": " << opOverview[ops[c]->getOps() ] << endl;
-  }
-
-  for ( uint c = 0; c < cops.size(); c++ )
-  {
-    cout << cops[c]->writeInfos() << ": " << opOverview[cops[c]->getOps() ] << endl;
+    for ( int t = 0; t < ops[c].size(); t++ )
+    {
+      cout << ops[c][t]->writeInfos() << ": " << opOverview[ops[c][t]->getOps() ] << endl;
+    }
   }
 
   for ( int d = 0; d < maxDepth; d++ )
@@ -1143,7 +1140,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
 #endif
 }
 
-void SemSegContextTree::extractBasicFeatures ( NICE::MultiChannelImageT<double> &feats, const ColorImage &img, const string &currentFile)
+void SemSegContextTree::extractBasicFeatures ( NICE::MultiChannelImageT<double> &feats, const ColorImage &img, const string &currentFile )
 {
   int xsize = img.width();
   int ysize = img.height();
@@ -1182,7 +1179,7 @@ void SemSegContextTree::extractBasicFeatures ( NICE::MultiChannelImageT<double>
     }
   }
 
-  // read the geometric cues produced by Hoiem et al. 
+  // read the geometric cues produced by Hoiem et al.
   if ( useHoiemFeatures )
   {
     // we could also give the following set as a config option
@@ -1212,20 +1209,20 @@ void SemSegContextTree::extractBasicFeatures ( NICE::MultiChannelImageT<double>
       FileName fnConfidenceImage ( hoiemDirectory + fnBase.str() + "_c_" + hoiemClass + ".png" );
       if ( ! fnConfidenceImage.fileExists() )
       {
-        fthrow(Exception, "Unable to read the Hoiem geometric confidence image: " << fnConfidenceImage.str() << " (original image is " << currentFile << ")" );
+        fthrow ( Exception, "Unable to read the Hoiem geometric confidence image: " << fnConfidenceImage.str() << " (original image is " << currentFile << ")" );
       } else {
         Image confidenceImage ( fnConfidenceImage.str() );
         // check whether the image size is consistent
         if ( confidenceImage.width() != feats.width() || confidenceImage.height() != feats.height() )
         {
-          fthrow(Exception, "The size of the geometric confidence image does not match with the original image size: " << fnConfidenceImage.str());
+          fthrow ( Exception, "The size of the geometric confidence image does not match with the original image size: " << fnConfidenceImage.str() );
         }
         ImageT<double> dst = feats[currentChannel];
-        
+
         // copy standard image to double image
         for ( uint y = 0 ; y < confidenceImage.height(); y++ )
           for ( uint x = 0 ; x < confidenceImage.width(); x++ )
-            feats(x, y, currentChannel) = (double)confidenceImage(x,y);
+            feats ( x, y, currentChannel ) = ( double ) confidenceImage ( x, y );
       }
     }
   }
@@ -1236,6 +1233,9 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
   int xsize;
   int ysize;
   ce->getImageSize ( xsize, ysize );
+  firstiteration = true;
+
+  int classes = labelmapback.size();
 
   int numClasses = classNames->numClasses();
 
@@ -1251,7 +1251,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 
   std::string currentFile = Globals::getCurrentImgFN();
   MultiChannelImageT<double> feats;
-  
+
   NICE::ColorImage img;
 
   try {
@@ -1260,12 +1260,10 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
     cerr << "SemSeg: error opening image file <" << currentFile << ">" << endl;
     return;
   }
-  
-  extractBasicFeatures(feats, img, currentFile); //read image and do some simple transformations
 
-  bool allleaf = false;
+  extractBasicFeatures ( feats, img, currentFile ); //read image and do some simple transformations
 
-  MultiChannelImageT<double> integralImg;
+  bool allleaf = false;
 
   MultiChannelImageT<unsigned short int> currentfeats ( xsize, ysize, nbTrees );
 
@@ -1308,7 +1306,6 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
             feat.cfeats = &lastfeats;
             feat.cTree = tree;
             feat.tree = &forest[tree];
-            feat.integralImg = &integralImg;
 
             double val = forest[tree][t].feat->getVal ( feat, x, y );
 
@@ -1340,43 +1337,29 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
               }
 #endif
             }
-
-            /*if ( x == xpos && y == ypos )
-            {
-              cout << "val: " << val << " decision: " << forest[tree][t].decision << " details: " << forest[tree][t].feat->writeInfos() << endl;
-
-            }*/
           }
         }
       }
 
       if ( depth < maxDepth )
       {
-        //compute integral image
-        int channels = ( int ) labelmap.size() + feats.channels();
-
-        if ( integralImg.width() == 0 )
+        //compute integral images
+        if ( firstiteration )
         {
-          int xsize = feats.width();
-          int ysize = feats.height();
-
-          integralImg.reInit ( xsize, ysize, channels );
-          integralImg.setAll ( 0.0 );
+          feats.addChannel ( classes + rawChannels );
         }
       }
     }
 
     if ( depth < maxDepth )
     {
-      computeIntegralImage ( currentfeats, feats, integralImg );
+      computeIntegralImage ( currentfeats, feats, channelType.size() - classes );
 #ifdef TEXTONMAP
       computeIntegralImage ( textonMap, integralTexton );
 #endif
     }
   }
 
-//  cout << forest[0][currentfeats.get ( xpos, ypos, 0 ) ].dist << endl;
-
 #ifdef WRITEGLOB
   ofstream outstream ( "globtest.feat", ofstream::app );
   outstream << 0 << endl;
@@ -1385,8 +1368,8 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 #endif
 
   string cndir = conf->gS ( "SSContextTree", "cndir", "" );
-  int classes = ( int ) probabilities.channels();
-  vector<int> useclass ( classes, 1 );
+  int allClasses = ( int ) probabilities.channels();
+  vector<int> useclass ( allClasses, 1 );
 #ifdef WRITEGLOB
 
 
@@ -1401,7 +1384,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 
   if ( cndir != "" )
   {
-    useclass = vector<int> ( classes, 0 );
+    useclass = vector<int> ( allClasses, 0 );
     ifstream infile ( ( cndir + "/" + orgname + ".dat" ).c_str() );
 
 #undef OLD
@@ -1410,12 +1393,12 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
     {
       int tmp;
       infile >> tmp;
-      assert ( tmp >= 0 && tmp < classes );
+      assert ( tmp >= 0 && tmp < allClasses );
       useclass[tmp] = 1;
     }
 #else
     int c = 0;
-    vector<double> probs ( classes, 0.0 );
+    vector<double> probs ( allClasses, 0.0 );
 
     while ( !infile.eof() && infile.good() )
     {
@@ -1431,7 +1414,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
     if ( thr < 0.0 )
       thr = 0.0;
 
-    for ( int c = 0; c < classes; c++ )
+    for ( int c = 0; c < allClasses; c++ )
     {
       if ( probs[c] < thr )
       {
@@ -1441,7 +1424,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 
 #endif
 
-    for ( int c = 0; c < classes; c++ )
+    for ( int c = 0; c < allClasses; c++ )
     {
       if ( useclass[c] == 0 )
       {
@@ -1633,13 +1616,21 @@ void SemSegContextTree::store ( std::ostream & os, int format ) const
       }
     }
   }
-  
+
   os << channelType.size() << endl;
-  for(int i = 0; i < channelType.size(); i++)
+  for ( int i = 0; i < channelType.size(); i++ )
   {
     os << channelType[i] << " ";
   }
   os << endl;
+
+  os << integralMap.size() << endl;
+  for ( int i = 0; i < integralMap.size(); i++ )
+  {
+    os << integralMap[i].first << " " << integralMap[i].second << endl;
+  }
+
+  os << rawChannels << endl;
 }
 
 void SemSegContextTree::restore ( std::istream & is, int format )
@@ -1703,39 +1694,51 @@ void SemSegContextTree::restore ( std::istream & is, int format )
       {
         for ( uint o = 0; o < ops.size(); o++ )
         {
-          if ( ops[o]->getOps() == feattype )
-          {
-            forest[t][n].feat = ops[o]->clone();
-            break;
-          }
-        }
-
-        if ( forest[t][n].feat == NULL )
-        {
-          for ( uint o = 0; o < cops.size(); o++ )
+          for ( uint o2 = 0; o2 < ops[o].size(); o2++ )
           {
-            if ( cops[o]->getOps() == feattype )
+            if ( forest[t][n].feat == NULL )
             {
-              forest[t][n].feat = cops[o]->clone();
-              break;
+              for ( uint c = 0; c < ops[o].size(); c++ )
+              {
+                if ( ops[o][o2]->getOps() == feattype )
+                {
+                  forest[t][n].feat = ops[o][o2]->clone();
+                  break;
+                }
+              }
             }
           }
         }
+
         assert ( forest[t][n].feat != NULL );
         forest[t][n].feat->restore ( is );
       }
     }
   }
-  
+
   channelType.clear();
   int ctsize;
   is >> ctsize;
-  for(int i = 0; i < ctsize; i++)
+  for ( int i = 0; i < ctsize; i++ )
   {
     int tmp;
     is >> tmp;
-    channelType.push_back(tmp);
+    channelType.push_back ( tmp );
   }
+
+  integralMap.clear();
+  int iMapSize;
+  is >> iMapSize;
+  for ( int i = 0; i < iMapSize; i++ )
+  {
+    int first;
+    int second;
+    is >> first;
+    is >> second;
+    integralMap.push_back ( pair<int, int> ( first, second ) );
+  }
+
+  is >> rawChannels;
 }
 
 

semseg/SemSegContextTree.h

@@ -76,10 +76,7 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
     int randomTests;
 
     /** operations for pairwise features */
-    std::vector<Operation*> ops;
-
-    /** operations for pairwise context features */
-    std::vector<Operation*> cops;
+    std::vector<std::vector<Operation*> > ops;
 
     std::vector<ValueAccess*> calcVal;
 
@@ -110,7 +107,13 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
     /** use Regions as extra feature channel or not */
     bool useRegionFeature;
 
-    /** how to handle each channel */
+    /** how to handle each channel
+     * 0: simple grayvalue features
+     * 1: which pixel belongs to which region
+     * 2: graycolor integral images
+     * 3: context integral images
+     * 4: context features (not in MultiChannelImageT encoded)
+     */
     std::vector<int> channelType;
 
     /** list of channels per feature type */
@@ -120,7 +123,16 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
     bool useHoiemFeatures;
 
     /** directory of the geometric features */
-    std::string hoiemDirectory; 
+    std::string hoiemDirectory;
+    
+    /** first iteration or not */
+    bool firstiteration;
+    
+    /** which IntegralImage channel belongs to which raw value channel */
+    std::vector<std::pair<int, int> > integralMap;
+    
+    /** amount of grayvalue Channels */
+    int rawChannels;
 
   public:
     /** simple constructor */
@@ -151,7 +163,7 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
      * @param integralImage output image (must be initilized)
      * @return void
      **/
-    void computeIntegralImage ( const NICE::MultiChannelImageT<unsigned short int> &currentfeats, const NICE::MultiChannelImageT<double> &lfeats, NICE::MultiChannelImageT<double> &integralImage );
+    void computeIntegralImage ( const NICE::MultiChannelImageT<unsigned short int> &currentfeats, NICE::MultiChannelImageT<double> &lfeats,int firstChannel );
 
     /**
      * @brief reads image and does some simple convertions
@@ -181,7 +193,7 @@ class SemSegContextTree : public SemanticSegmentation, public NICE::Persistent
      * @param splitval
      * @return best information gain
      */
-    double getBestSplit ( std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<NICE::MultiChannelImageT<unsigned short int> > &currentfeats, std::vector<NICE::MultiChannelImageT<double> > &integralImgs, const std::vector<NICE::MatrixT<int> > &labels, int node, Operation *&splitop, double &splitval, const int &tree );
+    double getBestSplit ( std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<NICE::MultiChannelImageT<unsigned short int> > &currentfeats, const std::vector<NICE::MatrixT<int> > &labels, int node, Operation *&splitop, double &splitval, const int &tree );
 
     /**
      * @brief computes the mean probability for a given class over all trees

semseg/operations/Operations.cpp

@@ -87,6 +87,15 @@ std::string Operation::writeInfos()
   return ss.str();
 }
 
+double Equality::getVal ( const Features &feats, const int &x, const int &y )
+{
+  int xsize, ysize;
+  getXY ( feats, xsize, ysize );
+  double v1 = values->getVal ( feats, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
+  double v2 = values->getVal ( feats, BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel2 );
+  return (double)(v1 == v2);
+}
+
 double Minus::getVal ( const Features &feats, const int &x, const int &y )
 {
   int xsize, ysize;
@@ -141,28 +150,28 @@ double IntegralOps::getVal ( const Features &feats, const int &x, const int &y )
 {
   int xsize, ysize;
   getXY ( feats, xsize, ysize );
-  return computeMean ( *feats.integralImg, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel1 );
+  return computeMean ( *feats.feats, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel1 );
 }
 
 double GlobalFeats::getVal ( const Features &feats, const int &x, const int &y )
 {
   int xsize, ysize;
   getXY ( feats, xsize, ysize );
-  return computeMean ( *feats.integralImg, 0, 0, xsize - 1, ysize - 1, channel1 );
+  return computeMean ( *feats.feats, 0, 0, xsize - 1, ysize - 1, channel1 );
 }
 
 double IntegralCenteredOps::getVal ( const Features &feats, const int &x, const int &y )
 {
   int xsize, ysize;
   getXY ( feats, xsize, ysize );
-  return computeMean ( *feats.integralImg, BOUND ( x - x1, 0, xsize - 1 ), BOUND ( y - y1, 0, ysize - 1 ), BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
+  return computeMean ( *feats.feats, BOUND ( x - x1, 0, xsize - 1 ), BOUND ( y - y1, 0, ysize - 1 ), BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
 }
 
 double BiIntegralCenteredOps::getVal ( const Features &feats, const int &x, const int &y )
 {
   int xsize, ysize;
   getXY ( feats, xsize, ysize );
-  return computeMean ( *feats.integralImg, BOUND ( x - x1, 0, xsize - 1 ), BOUND ( y - y1, 0, ysize - 1 ), BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 ) - computeMean ( *feats.integralImg, BOUND ( x - x2, 0, xsize - 1 ), BOUND ( y - y2, 0, ysize - 1 ), BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel1 );
+  return computeMean ( *feats.feats, BOUND ( x - x1, 0, xsize - 1 ), BOUND ( y - y1, 0, ysize - 1 ), BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 ) - computeMean ( *feats.feats, BOUND ( x - x2, 0, xsize - 1 ), BOUND ( y - y2, 0, ysize - 1 ), BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel1 );
 }
 
 double HaarHorizontal::getVal ( const Features &feats, const int &x, const int &y )
@@ -175,7 +184,7 @@ double HaarHorizontal::getVal ( const Features &feats, const int &x, const int &
   int lrx = BOUND ( x + x1, 0, xsize - 1 );
   int lry = BOUND ( y + y1, 0, ysize - 1 );
 
-  return computeMean ( *feats.integralImg, tlx, tly, lrx, y, channel1 ) - computeMean ( *feats.integralImg, tlx, y, lrx, lry, channel1 );
+  return computeMean ( *feats.feats, tlx, tly, lrx, y, channel1 ) - computeMean ( *feats.feats, tlx, y, lrx, lry, channel1 );
 }
 
 double HaarVertical::getVal ( const Features &feats, const int &x, const int &y )
@@ -188,7 +197,7 @@ double HaarVertical::getVal ( const Features &feats, const int &x, const int &y
   int lrx = BOUND ( x + x1, 0, xsize - 1 );
   int lry = BOUND ( y + y1, 0, ysize - 1 );
 
-  return computeMean ( *feats.integralImg, tlx, tly, x, lry, channel1 ) - computeMean ( *feats.integralImg, x, tly, lrx, lry, channel1 );
+  return computeMean ( *feats.feats, tlx, tly, x, lry, channel1 ) - computeMean ( *feats.feats, x, tly, lrx, lry, channel1 );
 }
 
 double HaarDiag::getVal ( const Features &feats, const int &x, const int &y )
@@ -201,7 +210,7 @@ double HaarDiag::getVal ( const Features &feats, const int &x, const int &y )
   int lrx = BOUND ( x + x1, 0, xsize - 1 );
   int lry = BOUND ( y + y1, 0, ysize - 1 );
 
-  return computeMean ( *feats.integralImg, tlx, tly, x, y, channel1 ) + computeMean ( *feats.integralImg, x, y, lrx, lry, channel1 ) - computeMean ( *feats.integralImg, tlx, y, x, lry, channel1 ) - computeMean ( *feats.integralImg, x, tly, lrx, y, channel1 );
+  return computeMean ( *feats.feats, tlx, tly, x, y, channel1 ) + computeMean ( *feats.feats, x, y, lrx, lry, channel1 ) - computeMean ( *feats.feats, tlx, y, x, lry, channel1 ) - computeMean ( *feats.feats, x, tly, lrx, y, channel1 );
 }
 
 double Haar3Horiz::getVal ( const Features &feats, const int &x, const int &y )
@@ -216,7 +225,7 @@ double Haar3Horiz::getVal ( const Features &feats, const int &x, const int &y )
   int lrx = BOUND ( x + x2, 0, xsize - 1 );
   int lry = BOUND ( y + y2, 0, ysize - 1 );
 
-  return computeMean ( *feats.integralImg, tlx, tly, lrx, mtly, channel1 ) - computeMean ( *feats.integralImg, tlx, mtly, lrx, mlry, channel1 ) + computeMean ( *feats.integralImg, tlx, mlry, lrx, lry, channel1 );
+  return computeMean ( *feats.feats, tlx, tly, lrx, mtly, channel1 ) - computeMean ( *feats.feats, tlx, mtly, lrx, mlry, channel1 ) + computeMean ( *feats.feats, tlx, mlry, lrx, lry, channel1 );
 }
 
 double Haar3Vert::getVal ( const Features &feats, const int &x, const int &y )
@@ -231,7 +240,7 @@ double Haar3Vert::getVal ( const Features &feats, const int &x, const int &y )
   int lrx = BOUND ( x + x2, 0, xsize - 1 );
   int lry = BOUND ( y + y2, 0, ysize - 1 );
 
-  return computeMean ( *feats.integralImg, tlx, tly, mtlx, lry, channel1 ) - computeMean ( *feats.integralImg, mtlx, tly, mlrx, lry, channel1 ) + computeMean ( *feats.integralImg, mlrx, tly, lrx, lry, channel1 );
+  return computeMean ( *feats.feats, tlx, tly, mtlx, lry, channel1 ) - computeMean ( *feats.feats, mtlx, tly, mlrx, lry, channel1 ) + computeMean ( *feats.feats, mlrx, tly, lrx, lry, channel1 );
 }
 
 void IntegralOps::set ( int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values )

semseg/operations/Operations.h

@@ -44,6 +44,7 @@ enum OperationTypes {
   RELATIVEXPOSITION,
   RELATIVEYPOSITION,
   GLOBALFEATS,
+  EQUALITY,
   NBOPERATIONS
 };
 
@@ -103,9 +104,6 @@ struct Features {
 
   /** tree nodes */
   std::vector<TreeNode> *tree;
-
-  /** integral images for faster feature computation */
-  NICE::MultiChannelImageT<double> *integralImg;
 };
 
 /**
@@ -220,6 +218,7 @@ class ClassificationResultAccess: public ValueAccess
     }
 };
 
+#if 0
 /**
  * @brief not finished yet, do we really need sparse feature representation or ClassificationResultAccess sufficient
  **/
@@ -265,6 +264,7 @@ class SparseImageAccess: public ValueAccess
       return CONTEXT;
     }
 };
+#endif
 
 /**
  * @brief abstract operation class
@@ -362,6 +362,55 @@ class Operation
     virtual void restore ( std::istream & is );
 };
 
+/**
+ * @brief simple equality check ?(A==B)
+ **/
+class Equality: public Operation
+{
+  public:
+    /**
+     * @brief interface for feature computation
+     * @param feats features
+     * @param cfeats number of tree node for each pixel
+     * @param tree current tree
+     * @param x current x position
+     * @param y current y position
+     * @return double distance
+     **/
+    virtual double getVal ( const Features &feats, const int &x, const int &y );
+
+    /**
+     * @brief clone operation instead of copy constructor (copy constructor does not work)
+     **/
+    virtual Operation* clone()
+    {
+      return new Equality();
+    }
+
+    /**
+     * @brief print some infos about operation extraction type
+     * @return string feature type
+     **/
+    virtual std::string writeInfos()
+    {
+      std::string out = "Equality";
+
+      if ( values != NULL )
+        out += values->writeInfos();
+
+      return out + Operation::writeInfos();
+    }
+
+    /**
+     * @brief return operation type (for store and restor)
+     * @return OperationTypes
+     **/
+    virtual OperationTypes getOps()
+    {
+      return EQUALITY;
+    }
+};
+
 /**
  * @brief simple difference operation A-B
  **/