gen aenderungen

libdepend.inc

@@ -1 +1 @@
-$(call PKG_DEPEND_INT,core)
+$(call PKG_DEPEND_INT,core)

semseg/SemSegCsurka.cpp

@@ -11,7 +11,9 @@ using namespace NICE;
 using namespace OBJREC;
 
 #undef DEBUG_CSURK
-#define UNCERTAINTY
+
+#undef UNCERTAINTY
+// #define UNCERTAINTY
 
 SemSegCsurka::SemSegCsurka ( const Config *conf,
                              const MultiDataset *md )
@@ -1480,12 +1482,38 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
   }
 
 #ifdef UNCERTAINTY
-  vector<FloatImage> uncert;
+  std::vector<FloatImage> uncert;
+  std::vector<FloatImage> gpUncertainty;
+  std::vector<FloatImage> gpMean;    
+  std::vector<FloatImage> gpMeanRatio;  
+  std::vector<FloatImage> gpWeightAll;
+  std::vector<FloatImage> gpWeightRatio;
+//   std::vector<FloatImage> gpImpactAll;
+//   std::vector<FloatImage> gpImpactRatio;
+  
+  //pre-allocate storage -- one image per scale and method
   for(int s = 0; s < scalesize; s++)
   {
     uncert.push_back(FloatImage(xsize, ysize));
     uncert[s].set(0.0);
+    
+    gpUncertainty.push_back(FloatImage(xsize, ysize));
+    gpMean.push_back(FloatImage(xsize, ysize));
+    gpMeanRatio.push_back(FloatImage(xsize, ysize));
+    gpWeightAll.push_back(FloatImage(xsize, ysize));
+    gpWeightRatio.push_back(FloatImage(xsize, ysize));
+/*    gpImpactAll.push_back(FloatImage(xsize, ysize));    
+    gpImpactRatio.push_back(FloatImage(xsize, ysize));   */  
+   
+    gpUncertainty[s].set(0.0);
+    gpMean[s].set(0.0);
+    gpMeanRatio[s].set(0.0);
+    gpWeightAll[s].set(0.0);
+    gpWeightRatio[s].set(0.0);
+//     gpImpactAll[s].set(0.0); 
+//     gpImpactRatio[s].set(0.0);   
   }
+  
   ColorImage imgrgb ( xsize, ysize );
   std::string s;
   std::stringstream out;
@@ -1495,8 +1523,15 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
   
   double maxu = -numeric_limits<double>::max();
   double minu = numeric_limits<double>::max();
+  
+  double gpNoise =  conf->gD("GPHIK", "noise", 0.01);
+  
 #endif
 
+  #ifdef UNCERTAINTY
+  std::cerr << "compute values for uncertainty stuff as well" << std::endl;
+  #endif
+  
   if ( classifier != NULL )
   {
     clog << "[log] SemSegCsruka::classifyregions: Wahrscheinlichkeitskarten erstellen: classifier != NULL" << endl;
@@ -1508,6 +1543,13 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
       {
         ClassificationResult r = classifier->classify ( pce[i].second );
 
+        #ifdef UNCERTAINTY
+        //we need this if we want to compute GP-AL-measure lateron
+        double minMeanAbs ( numeric_limits<double>::max() );
+        double maxMeanAbs ( 0.0 );
+        double sndMaxMeanAbs ( 0.0 );
+        #endif
+        
         for ( int j = 0 ; j < r.scores.size(); j++ )
         {
           if ( useclass[j] == 0 )
@@ -1515,6 +1557,23 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
 
           fV[j] += r.scores[j];
           preMap.set ( pce[i].second.x, pce[i].second.y, r.scores[j], j + s*klassen );
+          
+         #ifdef UNCERTAINTY 
+          //check whether we found a class with higher smaller abs mean than the current minimum
+         if (abs(r.scores[j]) < minMeanAbs)  
+           minMeanAbs = abs(r.scores[j]);
+         //check for larger abs mean as well
+         if (abs(r.scores[j]) > maxMeanAbs)
+         {
+           sndMaxMeanAbs = maxMeanAbs;
+           maxMeanAbs = abs(r.scores[j]);
+         }
+         // and also for the second highest mean of all classes
+         else if (abs(r.scores[j]) > sndMaxMeanAbs)
+         {
+           sndMaxMeanAbs = abs(r.scores[j]);
+         }
+         #endif          
         }
 
         /*if(r.uncertainty < 0.0)
@@ -1528,6 +1587,89 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
         uncert[s] ( pce[i].second.x, pce[i].second.y ) = r.uncertainty;
         maxu = std::max ( r.uncertainty, maxu );
         minu = std::min ( r.uncertainty, minu );
+        
+        
+        double firstTerm (1.0 / sqrt(r.uncertainty+gpNoise));
+        
+        //compute the heuristic GP-UNCERTAINTY, as proposed by Kapoor et al. in IJCV 2010
+        // GP-UNCERTAINTY : |mean| / sqrt(var^2 + gpnoise^2)
+        gpUncertainty[s] ( pce[i].second.x, pce[i].second.y ) = maxMeanAbs*firstTerm; //firstTerm = 1.0 / sqrt(r.uncertainty+gpNoise))
+        
+        // compute results when we take the lowest mean value of all classes
+        gpMean[s] ( pce[i].second.x, pce[i].second.y ) = minMeanAbs;
+        
+        //look at the difference in the absolut mean values for the most plausible class
+        // and the second most plausible class
+        gpMeanRatio[s] ( pce[i].second.x, pce[i].second.y ) = maxMeanAbs - sndMaxMeanAbs;
+        
+
+        //compute the weight in the alpha-vector for every sample after assuming it to be 
+        // added to the training set.
+        // Thereby, we measure its "importance" for the current model
+        // 
+        //double firstTerm is already computed
+        //
+        //the second term is only needed when computing impacts
+        //double secondTerm; //this is the nasty guy :/
+        
+        //--- compute the third term
+        // this is the difference between predicted label and GT label 
+        std::vector<double> diffToPositive; diffToPositive.clear();
+        std::vector<double> diffToNegative; diffToNegative.clear();
+        double diffToNegativeSum(0.0);
+        
+        for ( int j = 0 ; j < r.scores.size(); j++ )
+        {
+          if ( useclass[j] == 0 )
+            continue;
+          // look at the difference to plus 1          
+          diffToPositive.push_back(abs(r.scores[j] - 1));
+          // look at the difference to -1          
+          diffToNegative.push_back(abs(r.scores[j] + 1));
+          //sum up the difference to -1
+          diffToNegativeSum += abs(r.scores[j] - 1);
+        }
+
+        //let's subtract for every class its diffToNegative from the sum, add its diffToPositive,
+        //and use this as the third term for this specific class.
+        //the final value is obtained by minimizing over all classes
+        //
+        // originally, we minimize over all classes after building the final score
+        // however, the first and the second term do not depend on the choice of
+        // y*, therefore we minimize here already
+        double thirdTerm (numeric_limits<double>::max()) ;
+        for(uint tmpCnt = 0; tmpCnt < diffToPositive.size(); tmpCnt++)
+        {
+          double tmpVal ( diffToPositive[tmpCnt] + (diffToNegativeSum-diffToNegative[tmpCnt])   );
+          if (tmpVal < thirdTerm)
+            thirdTerm = tmpVal;
+        }
+        gpWeightAll[s] ( pce[i].second.x, pce[i].second.y ) = thirdTerm*firstTerm;        
+        
+        //now look on the ratio of the resulting weights for the most plausible
+        // against the second most plausible class
+        double thirdTermMostPlausible ( 0.0 ) ;
+        double thirdTermSecondMostPlausible ( 0.0 ) ;
+        for(uint tmpCnt = 0; tmpCnt < diffToPositive.size(); tmpCnt++)
+        {
+          if (diffToPositive[tmpCnt] > thirdTermMostPlausible)
+          {
+            thirdTermSecondMostPlausible = thirdTermMostPlausible;
+            thirdTermMostPlausible = diffToPositive[tmpCnt];
+          }
+          else if (diffToPositive[tmpCnt] > thirdTermSecondMostPlausible)
+          {
+            thirdTermSecondMostPlausible = diffToPositive[tmpCnt];
+          }
+        }
+        //compute the resulting score
+        gpWeightRatio[s] ( pce[i].second.x, pce[i].second.y ) = (thirdTermMostPlausible - thirdTermSecondMostPlausible)*firstTerm;      
+
+        //finally, look for this feature how it would affect to whole model (summarized by weight-vector alpha), if we would 
+        //use it as an additional training example
+        //TODO this would be REALLY computational demanding. Do we really want to do this?
+//         gpImpactAll[s] ( pce[i].second.x, pce[i].second.y ) = thirdTerm*firstTerm*secondTerm;
+//         gpImpactRatio[s] ( pce[i].second.x, pce[i].second.y ) = (thirdTermMostPlausible - thirdTermSecondMostPlausible)*firstTerm*secondTerm;      
 #endif
       }
     }
@@ -1541,27 +1683,162 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
       for ( int i = s; i < ( int ) pce.size(); i += scalesize )
       {
         ClassificationResult r = vclassifier->classify ( * ( pce[i].second.vec ) );
+        
+        #ifdef UNCERTAINTY
+        //we need this if we want to compute GP-AL-measure lateron
+        double minMeanAbs ( numeric_limits<double>::max() );
+        double maxMeanAbs ( 0.0 );
+        double sndMaxMeanAbs ( 0.0 );
+        #endif        
+        
         for ( int j = 0 ; j < ( int ) r.scores.size(); j++ )
         {
           if ( useclass[j] == 0 )
             continue;
           fV[j] += r.scores[j];
           preMap.set ( pce[i].second.x, pce[i].second.y, r.scores[j], j + s*klassen );
+          
+         #ifdef UNCERTAINTY 
+          //check whether we found a class with higher smaller abs mean than the current minimum
+         if (abs(r.scores[j]) < minMeanAbs)  
+           minMeanAbs = abs(r.scores[j]);
+         //check for larger abs mean as well
+         if (abs(r.scores[j]) > maxMeanAbs)
+         {
+           sndMaxMeanAbs = maxMeanAbs;
+           maxMeanAbs = abs(r.scores[j]);
+         }
+         // and also for the second highest mean of all classes
+         else if (abs(r.scores[j]) > sndMaxMeanAbs)
+         {
+           sndMaxMeanAbs = abs(r.scores[j]);
+         }
+         #endif            
         }
 #ifdef UNCERTAINTY
         uncert[s] ( pce[i].second.x, pce[i].second.y ) = r.uncertainty;
         maxu = std::max ( r.uncertainty, maxu );
         minu = std::min ( r.uncertainty, minu );
+        
+        
+        double firstTerm (1.0 / sqrt(r.uncertainty+gpNoise));
+        
+        //compute the heuristic GP-UNCERTAINTY, as proposed by Kapoor et al. in IJCV 2010
+        // GP-UNCERTAINTY : |mean| / sqrt(var^2 + gpnoise^2)
+        gpUncertainty[s] ( pce[i].second.x, pce[i].second.y ) = maxMeanAbs*firstTerm; //firstTerm = 1.0 / sqrt(r.uncertainty+gpNoise))
+        
+        // compute results when we take the lowest mean value of all classes
+        gpMean[s] ( pce[i].second.x, pce[i].second.y ) = minMeanAbs;
+        
+        //look at the difference in the absolut mean values for the most plausible class
+        // and the second most plausible class
+        gpMeanRatio[s] ( pce[i].second.x, pce[i].second.y ) = maxMeanAbs - sndMaxMeanAbs;
+        
+
+        //compute the weight in the alpha-vector for every sample after assuming it to be 
+        // added to the training set.
+        // Thereby, we measure its "importance" for the current model
+        // 
+        //double firstTerm is already computed
+        //
+        //the second term is only needed when computing impacts
+        //double secondTerm; //this is the nasty guy :/
+        
+        //--- compute the third term
+        // this is the difference between predicted label and GT label 
+        std::vector<double> diffToPositive; diffToPositive.clear();
+        std::vector<double> diffToNegative; diffToNegative.clear();
+        double diffToNegativeSum(0.0);
+        
+        for ( int j = 0 ; j < fV.size(); j++ )
+        {
+          if ( useclass[j] == 0 )
+            continue;
+          // look at the difference to plus 1          
+          diffToPositive.push_back(abs(r.scores[j] - 1));
+          // look at the difference to -1          
+          diffToNegative.push_back(abs(r.scores[j] + 1));
+          //sum up the difference to -1
+          diffToNegativeSum += abs(r.scores[j] - 1);
+        }
+
+        //let's subtract for every class its diffToNegative from the sum, add its diffToPositive,
+        //and use this as the third term for this specific class.
+        //the final value is obtained by minimizing over all classes
+        //
+        // originally, we minimize over all classes after building the final score
+        // however, the first and the second term do not depend on the choice of
+        // y*, therefore we minimize here already
+        double thirdTerm (numeric_limits<double>::max()) ;
+        for(uint tmpCnt = 0; tmpCnt < diffToPositive.size(); tmpCnt++)
+        {
+          double tmpVal ( diffToPositive[tmpCnt] + (diffToNegativeSum-diffToNegative[tmpCnt])   );
+          if (tmpVal < thirdTerm)
+            thirdTerm = tmpVal;
+        }
+        gpWeightAll[s] ( pce[i].second.x, pce[i].second.y ) = thirdTerm*firstTerm;        
+        
+        //now look on the ratio of the resulting weights for the most plausible
+        // against the second most plausible class
+        double thirdTermMostPlausible ( 0.0 ) ;
+        double thirdTermSecondMostPlausible ( 0.0 ) ;
+        for(uint tmpCnt = 0; tmpCnt < diffToPositive.size(); tmpCnt++)
+        {
+          if (diffToPositive[tmpCnt] > thirdTermMostPlausible)
+          {
+            thirdTermSecondMostPlausible = thirdTermMostPlausible;
+            thirdTermMostPlausible = diffToPositive[tmpCnt];
+          }
+          else if (diffToPositive[tmpCnt] > thirdTermSecondMostPlausible)
+          {
+            thirdTermSecondMostPlausible = diffToPositive[tmpCnt];
+          }
+        }
+        //compute the resulting score
+        gpWeightRatio[s] ( pce[i].second.x, pce[i].second.y ) = (thirdTermMostPlausible - thirdTermSecondMostPlausible)*firstTerm;      
+
+        //finally, look for this feature how it would affect to whole model (summarized by weight-vector alpha), if we would 
+        //use it as an additional training example
+        //TODO this would be REALLY computational demanding. Do we really want to do this?
+//         gpImpactAll[s] ( pce[i].second.x, pce[i].second.y ) = thirdTerm*firstTerm*secondTerm;
+//         gpImpactRatio[s] ( pce[i].second.x, pce[i].second.y ) = (thirdTermMostPlausible - thirdTermSecondMostPlausible)*firstTerm*secondTerm;      
 #endif
       }
     }
   }
 
+  #ifdef UNCERTAINTY
+  std::cerr << "uncertainty values and derived scores successfully computed" << std::endl;
+  #endif
+
 #ifdef UNCERTAINTY
   cout << "maxvdirect: " << maxu << " minvdirect: " << minu << endl;
+  //pre-allocate the image for filtering lateron
   FloatImage gaussUncert ( xsize, ysize );
+  
+  //just store the first scale
   ICETools::convertToRGB ( uncert[0], imgrgb );
-  imgrgb.write ( out.str() + "rough.png" );
+  imgrgb.write ( out.str() + "rough.ppm" );
+  
+  //pre-allocate memory for filtering of scales
+  FloatImage gaussGPUncertainty ( xsize, ysize );
+  FloatImage gaussGPMean ( xsize, ysize );
+  FloatImage gaussGPMeanRatio( xsize, ysize );
+  FloatImage gaussGPWeightAll ( xsize, ysize );
+  FloatImage gaussGPWeightRatio ( xsize, ysize );
+   
+  //just store the first scale for every method
+  ICETools::convertToRGB ( gpUncertainty[0], imgrgb );
+  imgrgb.write ( out.str() + "gpUncertainty.ppm" );
+  ICETools::convertToRGB ( gpMean[0], imgrgb );
+  imgrgb.write ( out.str() + "gpMean.ppm" );
+  ICETools::convertToRGB ( gpMeanRatio[0], imgrgb );
+  imgrgb.write ( out.str() + "gpMeanRatio.ppm" );
+  ICETools::convertToRGB ( gpWeightAll[0], imgrgb );
+  imgrgb.write ( out.str() + "gpWeightAll.ppm" );
+  ICETools::convertToRGB ( gpWeightRatio[0], imgrgb );
+  imgrgb.write ( out.str() + "gpWeightRatio.ppm" );  
+  
 #endif
 
   vector<double> scalesVec;
@@ -1659,6 +1936,19 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
 #ifdef UNCERTAINTY
     filterGaussSigmaApproximate<float, float, float> ( uncert[s], sigma, &gaussUncert );
     uncert[s] = gaussUncert;
+    
+    //apply the gauss-filtering to all scales of every method
+    filterGaussSigmaApproximate<float, float, float> ( gpUncertainty[s], sigma, &gaussGPUncertainty );
+    filterGaussSigmaApproximate<float, float, float> ( gpMean[s], sigma, &gaussGPMean );
+    filterGaussSigmaApproximate<float, float, float> ( gpMeanRatio[s], sigma, &gaussGPMeanRatio );
+    filterGaussSigmaApproximate<float, float, float> ( gpWeightAll[s], sigma, &gaussGPWeightAll );
+    filterGaussSigmaApproximate<float, float, float> ( gpWeightRatio[s], sigma, &gaussGPWeightRatio );
+    
+    gpUncertainty[s] = gaussGPUncertainty; 
+    gpMean[s] = gaussGPMean; 
+    gpMeanRatio[s] = gaussGPMeanRatio; 
+    gpWeightAll[s] = gaussGPWeightAll;
+    gpWeightRatio[s] = gaussGPWeightRatio;   
 #endif
   }
 
@@ -1691,8 +1981,20 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
       for ( int s = 0; s < ( int ) scalesize; s++ )
       {
         gaussUncert(x,y) += uncert[s](x,y);
+        //and for the other methods as well
+        gaussGPUncertainty(x,y) += gpUncertainty[s](x,y);
+        gaussGPMean(x,y) += gpMean[s](x,y);
+        gaussGPMeanRatio(x,y) += gpMeanRatio[s](x,y);
+        gaussGPWeightAll(x,y) += gpWeightAll[s](x,y);
+        gaussGPWeightRatio(x,y) += gpWeightRatio[s](x,y);
       }
       gaussUncert(x,y)/=scalesize;
+      //and for the other methods as well
+      gaussGPUncertainty(x,y)/=scalesize;
+      gaussGPMean(x,y)/=scalesize;
+      gaussGPMeanRatio(x,y)/=scalesize;
+      gaussGPWeightAll(x,y)/=scalesize;
+      gaussGPWeightRatio(x,y)/=scalesize;      
     }
   }
 
@@ -1726,8 +2028,19 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
   gaussUncert(0,0) = 0.0;
   gaussUncert(0,1) = 0.04;
   ICETools::convertToRGB ( gaussUncert, imgrgb );
-  imgrgb.write ( out.str() + "filtered.png" );
-
+  imgrgb.write ( out.str() + "filtered.ppm" );
+  
+  ICETools::convertToRGB ( gaussGPUncertainty, imgrgb );
+  imgrgb.write ( out.str() + "gpUncertaintyFiltered.ppm" );
+  ICETools::convertToRGB ( gaussGPMean, imgrgb );
+  imgrgb.write ( out.str() + "gpMeanFiltered.ppm" );
+  ICETools::convertToRGB ( gaussGPMeanRatio, imgrgb );
+  imgrgb.write ( out.str() + "gpMeanRatioFiltered.ppm" );
+  ICETools::convertToRGB ( gaussGPWeightAll, imgrgb );
+  imgrgb.write ( out.str() + "gpWeightAllFiltered.ppm" );
+  ICETools::convertToRGB ( gaussGPWeightRatio, imgrgb );
+  imgrgb.write ( out.str() + "gpWeightRatioFiltered.ppm" );  
+  
 #endif
 
 #undef VISSEMSEG
@@ -1793,7 +2106,13 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
     vector<vector <double> > regionprob;
 
 #ifdef UNCERTAINTY
-    vector<double> regionUncert;
+    std::vector<double> regionUncert;
+    
+    std::vector<double> regionGPUncertainty;
+    std::vector<double> regionGPMean;
+    std::vector<double> regionGPMeanRatio;
+    std::vector<double> regionGPWeightAll;
+    std::vector<double> regionGPWeightRatio;    
 #endif
 
     // Wahrscheinlichkeiten für jede Region initialisieren
@@ -1808,6 +2127,12 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
       Regionen.push_back ( pair<int, Example> ( 0, Example() ) );
 #ifdef UNCERTAINTY
       regionUncert.push_back ( 0.0 );
+      
+      regionGPUncertainty.push_back ( 0.0 );
+      regionGPMean.push_back ( 0.0 );
+      regionGPMeanRatio.push_back ( 0.0 );
+      regionGPWeightAll.push_back ( 0.0 );
+      regionGPWeightRatio.push_back ( 0.0 );
 #endif
     }
 
@@ -1827,11 +2152,16 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
         }
 #ifdef UNCERTAINTY
         regionUncert[pos] += gaussUncert ( x, y );
+        
+        regionGPUncertainty[pos] += gaussGPUncertainty ( x, y );
+        regionGPMean[pos] += gaussGPMean ( x, y );
+        regionGPMeanRatio[pos] += gaussGPMeanRatio ( x, y );
+        regionGPWeightAll[pos] += gaussGPWeightAll ( x, y );
+        regionGPWeightRatio[pos] += gaussGPWeightRatio ( x, y );
 #endif
       }
     }
 
-
     /*
     cout << "regions: " << regionsize << endl;
     cout << "outfeats: " << endl;
@@ -1875,9 +2205,14 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
       Regionen[i].first = maxpos;
 #ifdef UNCERTAINTY
       regionUncert[i] /= Regionen[i].second.weight;
+      
+      regionGPUncertainty[i] /= Regionen[i].second.weight;
+      regionGPMean[i] /= Regionen[i].second.weight;
+      regionGPMeanRatio[i] /= Regionen[i].second.weight;
+      regionGPWeightAll[i] /= Regionen[i].second.weight;
+      regionGPWeightRatio[i] /= Regionen[i].second.weight;
 #endif
     }
-
     // Pixel jeder Region labeln
     for ( int y = 0; y < ( int ) mask.cols(); y++ )
     {
@@ -1887,9 +2222,15 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
         segresult.setPixel ( x, y, Regionen[pos].first );
 #ifdef UNCERTAINTY
         gaussUncert ( x, y ) = regionUncert[pos];
+        
+        gaussGPUncertainty ( x, y ) = regionGPUncertainty[pos];
+        gaussGPMean ( x, y ) = regionGPMean[pos];
+        gaussGPMeanRatio ( x, y ) = regionGPMeanRatio[pos];
+        gaussGPWeightAll ( x, y ) = regionGPWeightAll[pos];
+        gaussGPWeightRatio ( x, y ) = regionGPWeightRatio[pos];        
 #endif
       }
-    }
+    }   
 #ifdef UNCERTAINTY
     maxu = -numeric_limits<float>::max();
     minu = numeric_limits<float>::max();
@@ -1907,7 +2248,18 @@ void SemSegCsurka::classifyregions ( CachedExample *ce, NICE::Image & segresult,
 //    uncert(0,0) = 1;
 //    uncert(0,1) = 0;
     ICETools::convertToRGB ( gaussUncert, imgrgb );
-    imgrgb.write ( out.str() + "region.png" );
+    imgrgb.write ( out.str() + "region.ppm" );
+    
+  ICETools::convertToRGB ( gaussGPUncertainty, imgrgb );
+  imgrgb.write ( out.str() + "gpUncertaintyRegion.ppm" );
+  ICETools::convertToRGB ( gaussGPMean, imgrgb );
+  imgrgb.write ( out.str() + "gpMeanRegion.ppm" );
+  ICETools::convertToRGB ( gaussGPMeanRatio, imgrgb );
+  imgrgb.write ( out.str() + "gpMeanRatioRegion.ppm" );
+  ICETools::convertToRGB ( gaussGPWeightAll, imgrgb );
+  imgrgb.write ( out.str() + "gpWeightAllRegion.ppm" );
+  ICETools::convertToRGB ( gaussGPWeightRatio, imgrgb );
+  imgrgb.write ( out.str() + "gpWeightRatioRegion.ppm" );      
 #endif
 
 #undef WRITEREGIONS

semseg/SemSegNovelty.cpp

@@ -10,6 +10,7 @@
 #include "vislearning/features/fpfeatures/SparseVectorFeature.h"
 #include "core/basics/StringTools.h"
 #include "core/basics/Timer.h"
+#include "segmentation/GenericRegionSegmentationMethodSelection.h"
 
 using namespace std;
 using namespace NICE;
@@ -29,14 +30,27 @@ SemSegNovelty::SemSegNovelty ( const Config *conf,
   read_cache = conf->gB ( "FPCPixel", "read_cache", false );
   uncertdir = conf->gS("debug", "uncertainty", "uncertainty");
   cache = conf->gS ( "cache", "root", "" );
-
-
+  
   classifier = new GPHIKClassifierNICE ( conf, "ClassiferGPHIK" );;
 
   whs = conf->gI ( section, "window_size", 10 );
   featdist = conf->gI ( section, "grid", 10 );
   testWSize = conf->gI (section, "test_window_size", 10);
-
+  string rsMethode = conf->gS ( section, "segmentation", "none" );
+ 
+  if(rsMethode == "none")
+  {
+    regionSeg = NULL;
+  }
+  else
+  {
+    RegionSegmentationMethod * tmpRegionSeg = GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(&conf, rsMethode);    
+    if ( save_cache )
+      regionSeg = new RSCache ( conf, tmpRegionSeg );
+    else
+      regionSeg = tmpseg;
+  }
+  
   cn = md->getClassNames ( "train" );
 
   if ( read_cache )
@@ -88,7 +102,15 @@ void SemSegNovelty::train ( const MultiDataset *md )
     forbidden_classes_s = conf->gS ( "analysis", "forbidden_classes", "" );
   }
   cn.getSelection ( forbidden_classes_s, forbidden_classes );
-  cerr << "forbidden: " << forbidden_classes_s << endl;
+  
+  //check the same thing for the training classes - this is very specific to our setup 
+  std::string forbidden_classesTrain_s = conf->gS ( "analysis", "donttrainTrain", "" );
+  if ( forbidden_classesTrain_s == "" )
+  {
+    forbidden_classesTrain_s = conf->gS ( "analysis", "forbidden_classesTrain", "" );
+  }
+  cn.getSelection ( forbidden_classesTrain_s, forbidden_classesTrain );
+
 
   ProgressBar pb ( "Local Feature Extraction" );
   pb.show();
@@ -100,6 +122,8 @@ void SemSegNovelty::train ( const MultiDataset *md )
 
   int featdim = -1;
 
+  classesInUse.clear();  
+  
   LOOP_ALL_S ( *trainp )
   {
     //EACH_S(classno, currentFile);
@@ -160,12 +184,19 @@ void SemSegNovelty::train ( const MultiDataset *md )
     {
       for ( int x = 0; x < xsize; x += featdist )
       {
-        int classno = labels ( x, y );
 
-        if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
+        int classnoTmp = labels.getPixel ( x, y );
+        
+        if ( forbidden_classesTrain.find ( classnoTmp ) != forbidden_classesTrain.end() )
+        {
           continue;
-
-
+        }
+        
+        if (classesInUse.find(classnoTmp) == classesInUse.end())
+        {
+          classesInUse.insert(classnoTmp);
+        }
+        
         Example example;
         example.vec = NULL;
         example.svec = new SparseVector ( featdim );
@@ -179,7 +210,7 @@ void SemSegNovelty::train ( const MultiDataset *md )
         example.svec->normalize();
 
         example.position = imgnb;
-        examples.push_back ( pair<int, Example> ( classno, example ) );
+        examples.push_back ( pair<int, Example> ( classnoTmp, example ) );
       }
     }
 
@@ -187,6 +218,16 @@ void SemSegNovelty::train ( const MultiDataset *md )
     imgnb++;
     pb.update ( trainp->count() );
   }
+  
+    
+  numberOfClasses = classesInUse.size();
+  std::cerr << "numberOfClasses: " << numberOfClasses << std::endl;  
+  std::cerr << "classes in use: " << std::endl;
+  for (std::set<int>::const_iterator it = classesInUse.begin(); it != classesInUse.end(); it++)
+  {
+    std::cerr << *it << " ";
+  }    
+  std::cerr << std::endl;
 
   pb.hide();
 
@@ -282,13 +323,33 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
 
   FloatImage uncert ( xsize, ysize );
   uncert.set ( 0.0 );
+  
+  FloatImage gpUncertainty ( xsize, ysize );
+  FloatImage gpMean ( xsize, ysize );    
+  FloatImage gpMeanRatio ( xsize, ysize );  
+  FloatImage gpWeightAll ( xsize, ysize );
+  FloatImage gpWeightRatio ( xsize, ysize );  
+  
+  gpUncertainty.set ( 0.0 );
+  gpMean.set ( 0.0 );
+  gpMeanRatio.set ( 0.0 );
+  gpWeightAll.set ( 0.0 );
+  gpWeightRatio.set ( 0.0 );
 
   double maxunc = -numeric_limits<double>::max();
-  double minunc = numeric_limits<double>::max();
-  double maxprob = -numeric_limits<double>::max();
-  double minprob = numeric_limits<double>::max();
+  
+  double maxGPUncertainty = -numeric_limits<double>::max();  
+  double maxGPMean = -numeric_limits<double>::max();  
+  double maxGPMeanRatio = -numeric_limits<double>::max();  
+  double maxGPWeightAll = -numeric_limits<double>::max();  
+  double maxGPWeightRatio = -numeric_limits<double>::max();  
+
   timer.stop();
   cout << "first: " << timer.getLastAbsolute() << endl;
+  
+  //we need this lateron for active learning stuff
+  double gpNoise =  conf->gD("GPHIK", "noise", 0.01);
+  
   timer.start();
 #pragma omp parallel for
   for ( int y = 0; y < ysize; y += testWSize )
@@ -307,6 +368,143 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
       example.svec->normalize();
 
       ClassificationResult cr = classifier->classify ( example );
+      
+      //we need this if we want to compute GP-AL-measure lateron
+      double minMeanAbs ( numeric_limits<double>::max() );
+      double maxMeanAbs ( 0.0 );
+      double sndMaxMeanAbs ( 0.0 );       
+      double maxMean ( -numeric_limits<double>::max() );
+      double sndMaxMean ( -numeric_limits<double>::max() );     
+      
+      for ( int j = 0 ; j < cr.scores.size(); j++ )
+      {   
+        if ( forbidden_classesTrain.find ( j ) != forbidden_classesTrain.end() )
+        {
+          continue;
+        }
+        
+        //check whether we found a class with higher smaller abs mean than the current minimum
+        if (abs(cr.scores[j]) < minMeanAbs)  
+          minMeanAbs = abs(cr.scores[j]);
+        //check for larger abs mean as well
+        if (abs(cr.scores[j]) > maxMeanAbs)
+        {
+          sndMaxMeanAbs = maxMeanAbs;
+          maxMeanAbs = abs(cr.scores[j]);
+        }
+        // and also for the second highest mean of all classes
+        else if (abs(cr.scores[j]) > sndMaxMeanAbs)
+        {
+          sndMaxMeanAbs = abs(cr.scores[j]);
+        }  
+        //check for larger mean without abs as well
+        if (cr.scores[j] > maxMean)
+        {
+          sndMaxMean = maxMean;
+          maxMean = cr.scores[j];
+        }
+        // and also for the second highest mean of all classes
+        else if (cr.scores[j] > sndMaxMean)
+        {
+          sndMaxMean = cr.scores[j];
+        }          
+      }
+
+      double firstTerm (1.0 / sqrt(cr.uncertainty+gpNoise));
+      
+      //compute the heuristic GP-UNCERTAINTY, as proposed by Kapoor et al. in IJCV 2010
+      // GP-UNCERTAINTY : |mean| / sqrt(var^2 + gpnoise^2)
+      double gpUncertaintyVal = maxMeanAbs*firstTerm; //firstTerm = 1.0 / sqrt(r.uncertainty+gpNoise))
+      
+      // compute results when we take the lowest mean value of all classes
+      double gpMeanVal = minMeanAbs;
+      
+      //look at the difference in the absolut mean values for the most plausible class
+      // and the second most plausible class
+      double gpMeanRatioVal= maxMean - sndMaxMean;
+      
+       double gpWeightAllVal ( 0.0 );
+       double gpWeightRatioVal ( 0.0 );
+
+       if ( numberOfClasses > 2)
+       {
+        //compute the weight in the alpha-vector for every sample after assuming it to be 
+        // added to the training set.
+        // Thereby, we measure its "importance" for the current model
+        // 
+        //double firstTerm is already computed
+        //
+        //the second term is only needed when computing impacts
+        //double secondTerm; //this is the nasty guy :/
+        
+        //--- compute the third term
+        // this is the difference between predicted label and GT label 
+        std::vector<double> diffToPositive; diffToPositive.clear();
+        std::vector<double> diffToNegative; diffToNegative.clear();
+        double diffToNegativeSum(0.0);
+        
+        for ( int j = 0 ; j < cr.scores.size(); j++ )
+        {
+          if ( forbidden_classesTrain.find ( j ) != forbidden_classesTrain.end() )
+          {
+            continue;
+          }          
+          
+          // look at the difference to plus 1          
+          diffToPositive.push_back(abs(cr.scores[j] - 1));
+          // look at the difference to -1          
+          diffToNegative.push_back(abs(cr.scores[j] + 1));
+          //sum up the difference to -1
+          diffToNegativeSum += abs(cr.scores[j] - 1);
+        }
+
+        //let's subtract for every class its diffToNegative from the sum, add its diffToPositive,
+        //and use this as the third term for this specific class.
+        //the final value is obtained by minimizing over all classes
+        //
+        // originally, we minimize over all classes after building the final score
+        // however, the first and the second term do not depend on the choice of
+        // y*, therefore we minimize here already
+        double thirdTerm (numeric_limits<double>::max()) ;
+        for(uint tmpCnt = 0; tmpCnt < diffToPositive.size(); tmpCnt++)
+        {
+          double tmpVal ( diffToPositive[tmpCnt] + (diffToNegativeSum-diffToNegative[tmpCnt])   );
+          if (tmpVal < thirdTerm)
+            thirdTerm = tmpVal;
+        }
+        gpWeightAllVal = thirdTerm*firstTerm;        
+        
+        //now look on the ratio of the resulting weights for the most plausible
+        // against the second most plausible class
+        double thirdTermMostPlausible ( 0.0 ) ;
+        double thirdTermSecondMostPlausible ( 0.0 ) ;
+        for(uint tmpCnt = 0; tmpCnt < diffToPositive.size(); tmpCnt++)
+        {
+          if (diffToPositive[tmpCnt] > thirdTermMostPlausible)
+          {
+            thirdTermSecondMostPlausible = thirdTermMostPlausible;
+            thirdTermMostPlausible = diffToPositive[tmpCnt];
+          }
+          else if (diffToPositive[tmpCnt] > thirdTermSecondMostPlausible)
+          {
+            thirdTermSecondMostPlausible = diffToPositive[tmpCnt];
+          }
+        }
+        //compute the resulting score
+        gpWeightRatioVal = (thirdTermMostPlausible - thirdTermSecondMostPlausible)*firstTerm;      
+
+        //finally, look for this feature how it would affect to whole model (summarized by weight-vector alpha), if we would 
+        //use it as an additional training example
+        //TODO this would be REALLY computational demanding. Do we really want to do this?
+  //         gpImpactAll[s] ( pce[i].second.x, pce[i].second.y ) = thirdTerm*firstTerm*secondTerm;
+  //         gpImpactRatio[s] ( pce[i].second.x, pce[i].second.y ) = (thirdTermMostPlausible - thirdTermSecondMostPlausible)*firstTerm*secondTerm;
+       }
+       else //binary scenario
+       {
+         gpWeightAllVal = std::min( abs(cr.scores[*classesInUse.begin()]+1), abs(cr.scores[*classesInUse.begin()]-1) );
+         gpWeightAllVal *= firstTerm;
+         gpWeightRatioVal = gpWeightAllVal;
+       }
 
       int xs = std::max(0, x - testWSize/2);
       int xe = std::min(xsize - 1, x + testWSize/2);
@@ -319,30 +517,48 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
           for ( int j = 0 ; j < cr.scores.size(); j++ )
           {
             probabilities ( xl, yl, j ) = cr.scores[j];
-            if (maxprob < cr.scores[j])
-              maxprob = cr.scores[j];
-            if (minprob > cr.scores[j])
-              minprob = cr.scores[j];
           }
           segresult ( xl, yl ) = cr.classno;
           uncert ( xl, yl ) = cr.uncertainty;
+          
+          gpUncertainty ( xl, yl ) = gpUncertaintyVal;
+          gpMean ( xl, yl ) = gpMeanVal;
+          gpMeanRatio ( xl, yl ) = gpMeanRatioVal;
+          gpWeightAll ( xl, yl ) = gpWeightAllVal;
+          gpWeightRatio ( xl, yl ) = gpWeightRatioVal;    
         }
       }
 
       if (maxunc < cr.uncertainty)
         maxunc = cr.uncertainty;
-      if (minunc > cr.uncertainty)
-        minunc = cr.uncertainty;
+      
+      if (maxGPUncertainty < gpUncertaintyVal)
+        maxGPUncertainty = gpUncertaintyVal;
+      if (maxGPMean < gpMeanVal)
+        maxGPMean = gpMeanVal;
+      if (maxGPMeanRatio < gpMeanRatioVal)
+        maxGPMeanRatio = gpMeanRatioVal;
+      if (maxGPWeightAll < gpMeanRatioVal)
+        maxGPWeightAll = gpWeightAllVal;
+      if (maxGPWeightRatio < gpWeightRatioVal)
+        maxGPWeightRatio = gpWeightRatioVal;      
+      
       example.svec->clear();
     }
     delete example.svec;
     example.svec = NULL;
   }
 
-  cout << "maxunertainty: " << maxunc << endl;
-  cout << "minunertainty: " << minunc << endl;
-  cout << "maxprob: " << maxprob << endl;
-  cout << "minprob: " << minprob << endl;
+  //       std::cerr << "uncertainty: " << gpUncertaintyVal << " minMean: " << gpMeanVal << " gpMeanRatio: " << gpMeanRatioVal << " weightAll: " << gpWeightAllVal << " weightRatio: "<< gpWeightRatioVal << std::endl;
+  
+  
+  //Regionen ermitteln
+  if(regionSeg != NULL)
+  {
+    NICE::Matrix mask;
+    int regionsize = regionSeg->segRegions ( img, mask );
+    probabilities ( xl, yl, j ) = cr.scores[j];
+  }
 
   timer.stop();
   cout << "second: " << timer.getLastAbsolute() << endl;
@@ -354,13 +570,52 @@ void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NI
   std::vector< std::string > list2;
   StringTools::split ( Globals::getCurrentImgFN (), '/', list2 );
   out << uncertdir << "/" << list2.back();
-
+  
   uncert.writeRaw(out.str() + ".rawfloat");
   uncert(0, 0) = 0.0;
-  uncert(0, 1) = 1.0;
+  uncert(0, 1) = 1.0+gpNoise;
   ICETools::convertToRGB ( uncert, imgrgb );
   imgrgb.write ( out.str() + "rough.png" );
 
+  //invert images such that large numbers correspond to high impact, high variance, high importance, high novelty, ...
+  for ( int y = 0; y < ysize; y++)
+  {
+    for (int x = 0; x < xsize; x++)
+    {
+      gpUncertainty(x,y) =  maxGPUncertainty - gpUncertainty(x,y);
+      gpMean(x,y) = maxGPMean - gpMean(x,y);
+      gpMeanRatio(x,y) = maxGPMeanRatio - gpMeanRatio(x,y);
+      gpWeightRatio(x,y) = maxGPWeightRatio - gpWeightRatio(x,y);
+    }
+  }
+  
+  
+  //  
+  gpUncertainty(0, 0) = 0.0;
+  gpUncertainty(0, 1) = maxGPUncertainty;
+  ICETools::convertToRGB ( gpUncertainty, imgrgb );
+  imgrgb.write ( out.str() + "gpUncertainty.png" );
+  //
+  gpMean(0, 0) = 0.0;
+  gpMean(0, 1) = maxGPMean;  
+  ICETools::convertToRGB ( gpMean, imgrgb );
+  imgrgb.write ( out.str() + "gpMean.png" );
+  //
+  gpMeanRatio(0, 0) = 0.0;
+  gpMeanRatio(0, 1) = maxGPMeanRatio;   
+  ICETools::convertToRGB ( gpMeanRatio, imgrgb );
+  imgrgb.write ( out.str() + "gpMeanRatio.png" );
+  //
+  gpWeightAll(0, 0) = 0.0;
+  gpWeightAll(0, 1) = maxGPWeightAll;     
+  ICETools::convertToRGB ( gpWeightAll, imgrgb );
+  imgrgb.write ( out.str() + "gpWeightAll.png" );
+  //
+  gpWeightRatio(0, 0) = 0.0;
+  gpWeightRatio(0, 1) = maxGPWeightRatio;     
+  ICETools::convertToRGB ( gpWeightRatio, imgrgb );
+  imgrgb.write ( out.str() + "gpWeightRatio.png" );    
+
 
   timer.stop();
   cout << "last: " << timer.getLastAbsolute() << endl;

semseg/SemSegNovelty.h

@@ -52,8 +52,16 @@ class SemSegNovelty : public SemanticSegmentation
     //! name of all classes
     ClassNames cn;
     
+    //! low level Segmentation method
+    RegionSegmentationMethod *regionSeg;
+    
     //! set of forbidden/background classes
     std::set<int> forbidden_classes;
+    std::set<int> forbidden_classesTrain;
+    std::set<int> classesInUse;
+    
+    //! obviously, the number of classes used for training
+    int numberOfClasses;
     
     //! where to save the uncertainty
     std::string uncertdir;