added option for optimized diagonal matrix into bruteforce evaluation

progs/ImagenetBinary.conf

@@ -26,8 +26,8 @@ noiseGPVarApproxFile = /home/luetz/code/nice/vislearning/progs/gpvarNoise.txt
 
 indexOfFirstClass = 0
 indexOfLastClass = 0
-runsPerClassToAverageTraining = 100
-runsPerClassToAverageTraining = 100
+runsPerClassToAverageTraining = 1
+runsPerClassToAverageTesting = 1000
 
 nrOfExamplesPerClass = 100
 
@@ -35,6 +35,12 @@ GPSRMean = true
 GPSRVar = true
 GPMean = true
 GPVar = true
+GPMeanApprox = true
+GPVarApprox = true
+GPOptMean = true
+GPOptVar = true
+Parzen = true
+SVDD = true
 
 [GPSR]
 nrOfRegressors = 50

progs/testImageNetBinaryBruteForce.cpp

@@ -11,6 +11,7 @@
 #include "core/basics/Config.h"
 #include "core/basics/Timer.h"
 #include "core/algebra/CholeskyRobust.h"
+#include "core/algebra/DiagonalMatrixApprox.h"
 #include "core/vector/Algorithms.h"
 #include "core/vector/SparseVectorT.h"
 
@@ -393,6 +394,56 @@ void inline trainGPSRVar(NICE::Matrix & choleskyMatrix, const double & noise, co
     std::cerr << "Precise time used for GPSRVar training class " << classNumber << ": " << tTrainPrecise.getLast()/(double)runsPerClassToAverageTraining << std::endl;    
 }
 
+void inline trainGPOptMean(NICE::Vector & rightPartGPOptMean, const double & noise, const NICE::Matrix & kernelMatrix, const int & nrOfExamplesPerClass, const int & classNumber, const int & runsPerClassToAverageTraining )
+{
+    DiagonalMatrixApprox diagApprox ( true /*verbose*/ );
+    
+    Timer tTrainPrecise;
+    tTrainPrecise.start();     
+    
+    for (int run = 0; run < runsPerClassToAverageTraining; run++)
+    { 
+      //compute optimal diagonal matrix
+      diagApprox.approx ( kernelMatrix, rightPartGPOptMean );
+      
+      // invert entries
+      // by theory we should multiply these entries with the labels
+      // but we are in an OCC setting with labels equal to one
+      for (int i = 0; i < nrOfExamplesPerClass; i++)
+      {
+        if (rightPartGPOptMean[i] != 0.0)
+          rightPartGPOptMean[i] = 1.0 / rightPartGPOptMean[i]; 
+      }
+    }
+ 
+    tTrainPrecise.stop(); 
+    std::cerr << "Precise time used for GPOptMean training class " << classNumber << ": " << tTrainPrecise.getLast()/(double)runsPerClassToAverageTraining << std::endl;    
+}
+
+void inline trainGPOptVar(NICE::Vector & DiagGPOptVar, const double & noise, const NICE::Matrix & kernelMatrix, const int & nrOfExamplesPerClass, const int & classNumber, const int & runsPerClassToAverageTraining )
+{
+    DiagonalMatrixApprox diagApprox ( true /*verbose*/ );
+    
+    Timer tTrainPrecise;
+    tTrainPrecise.start();     
+    
+    for (int run = 0; run < runsPerClassToAverageTraining; run++)
+    { 
+      //compute optimal diagonal matrix
+      diagApprox.approx ( kernelMatrix, DiagGPOptVar );
+      
+      //invert entries
+      for (int i = 0; i < nrOfExamplesPerClass; i++)
+      {
+        if (DiagGPOptVar[i] != 0.0)
+          DiagGPOptVar[i] = 1.0 / DiagGPOptVar[i];
+      }
+    }
+ 
+    tTrainPrecise.stop(); 
+    std::cerr << "Precise time used for GPOptVar training class " << classNumber << ": " << tTrainPrecise.getLast()/(double)runsPerClassToAverageTraining << std::endl;    
+}
+
 KCMinimumEnclosingBall *trainSVDD( const double & noise, const NICE::Matrix kernelMatrix, const int & nrOfExamplesPerClass, const int & classNumber, const int & runsPerClassToAverageTraining )
 {
  
@@ -589,6 +640,61 @@ void inline evaluateGPSRVar(const NICE::Vector & kernelVector,  const NICE::Matr
     r = ClassificationResult ( scores[1]<0.5 ? 0 : 1, scores );    
 }
 
+//this method is completely the same as evaluateGPMeanApprox, but for convenience, it is its own method
+void inline evaluateGPOptMean(const NICE::Vector & kernelVector, const NICE::Vector & rightPart, ClassificationResult & r, double & timeForSingleExamples, const int & runsPerClassToAverageTesting)
+{
+    double mean;
+  
+    Timer tTestSingle;
+    tTestSingle.start();
+      
+    for (int run = 0; run < runsPerClassToAverageTesting; run++)
+    { 
+      // \mean = \k_*^T \cdot D^{-1} \cdot y  where D is our nice approximation of K    
+      mean = kernelVector.scalarProduct ( rightPart );
+    }
+      
+    tTestSingle.stop();
+    timeForSingleExamples += tTestSingle.getLast()/(double)runsPerClassToAverageTesting;      
+    
+    FullVector scores ( 2 );
+    scores[0] = 0.0;
+    scores[1] = mean;
+
+    r = ClassificationResult ( scores[1]<0.5 ? 0 : 1, scores );    
+}
+
+//this method is completely the same as evaluateGPVarApprox, but for convenience, it is its own method
+void inline evaluateGPOptVar(const NICE::Vector & kernelVector, const double & kernelSelf, const NICE::Vector & matrixDInv, ClassificationResult & r, double & timeForSingleExamples, const int & runsPerClassToAverageTesting)
+{
+    double uncertainty;
+    
+    Timer tTestSingle;
+    tTestSingle.start();
+      
+    for (int run = 0; run < runsPerClassToAverageTesting; run++)
+    {       
+      // uncertainty = k{**} - \k_*^T \cdot D^{-1} \cdot k_*  where D is our nice approximation of K
+      
+      NICE::Vector rightPart (kernelVector.size());
+      for (int j = 0; j < kernelVector.size(); j++)
+      {
+        rightPart[j] = kernelVector[j] * matrixDInv[j];
+      }
+
+      uncertainty = kernelSelf - kernelVector.scalarProduct ( rightPart );
+    }
+      
+    tTestSingle.stop();
+    timeForSingleExamples += tTestSingle.getLast()/(double)runsPerClassToAverageTesting;      
+    
+    FullVector scores ( 2 );
+    scores[0] = 0.0;
+    scores[1] = 1.0 - uncertainty;
+
+    r = ClassificationResult ( scores[1]<0.5 ? 0 : 1, scores );    
+}
+
 void inline evaluateParzen(const NICE::Vector & kernelVector,  ClassificationResult & r, double & timeForSingleExamples, const int & runsPerClassToAverageTesting)
 {
     double score;
@@ -661,6 +767,8 @@ int main (int argc, char **argv)
   bool GPVar = conf.gB( "main", "GPVar", false);
   bool GPSRMean = conf.gB( "main", "GPSRMean", false);
   bool GPSRVar = conf.gB( "main", "GPSRVar", false);  
+  bool GPOptMean = conf.gB( "main", "GPOptMean", false);
+  bool GPOptVar = conf.gB( "main", "GPOptVar", false);    
   bool Parzen = conf.gB( "main", "Parzen", false);
   bool SVDD = conf.gB( "main", "SVDD", false);  
   
@@ -688,6 +796,14 @@ int main (int argc, char **argv)
     std::cerr << "GPSRVar used" << std::endl;
   else 
     std::cerr << "GPSRVar not used" << std::endl;
+  if (GPOptMean)
+    std::cerr << "GPOptMean used" << std::endl;
+  else 
+    std::cerr << "GPOptMean not used" << std::endl;
+  if (GPOptVar)
+    std::cerr << "GPOptVar used" << std::endl;
+  else 
+    std::cerr << "GPOptVar not used" << std::endl;  
   if (Parzen)
     std::cerr << "Parzen used" << std::endl;
   else 
@@ -716,6 +832,12 @@ int main (int argc, char **argv)
   //GP SR var
   NICE::Vector sigmaGPSRVarParas(nrOfClassesToConcidere,0.0);
   NICE::Vector noiseGPSRVarParas(nrOfClassesToConcidere,0.0);
+  //GP Opt mean  
+  NICE::Vector sigmaGPOptMeanParas(nrOfClassesToConcidere,0.0);
+  NICE::Vector noiseGPOptMeanParas(nrOfClassesToConcidere,0.0);
+  //GP Opt var
+  NICE::Vector sigmaGPOptVarParas(nrOfClassesToConcidere,0.0);
+  NICE::Vector noiseGPOptVarParas(nrOfClassesToConcidere,0.0);  
   //Parzen  
   NICE::Vector sigmaParzenParas(nrOfClassesToConcidere,0.0);
   NICE::Vector noiseParzenParas(nrOfClassesToConcidere,0.0);
@@ -763,7 +885,13 @@ int main (int argc, char **argv)
     readParameters(noiseGPVarApproxFile,nrOfClassesToConcidere, noiseGPSRMeanParas);
     //GP SR var  
     readParameters(sigmaGPVarApproxFile,nrOfClassesToConcidere, sigmaGPSRVarParas);
-    readParameters(noiseGPVarApproxFile,nrOfClassesToConcidere, noiseGPSRVarParas);    
+    readParameters(noiseGPVarApproxFile,nrOfClassesToConcidere, noiseGPSRVarParas); 
+    //GP Opt mean  
+    readParameters(sigmaGPVarApproxFile,nrOfClassesToConcidere, sigmaGPOptMeanParas);
+    readParameters(noiseGPVarApproxFile,nrOfClassesToConcidere, noiseGPOptMeanParas);
+    //GP Opt var  
+    readParameters(sigmaGPVarApproxFile,nrOfClassesToConcidere, sigmaGPOptVarParas);
+    readParameters(noiseGPVarApproxFile,nrOfClassesToConcidere, noiseGPOptVarParas);     
     //Parzen    
     readParameters(sigmaGPVarApproxFile,nrOfClassesToConcidere, sigmaParzenParas);
     readParameters(noiseGPVarApproxFile,nrOfClassesToConcidere, noiseParzenParas);  
@@ -793,7 +921,13 @@ int main (int argc, char **argv)
     noiseGPSRMeanParas.set(noise);
     //GP SR var  
     sigmaGPSRVarParas.set(sigma);
-    noiseGPSRVarParas.set(noise);    
+    noiseGPSRVarParas.set(noise); 
+    //GP Opt mean  
+    sigmaGPOptMeanParas.set(sigma);
+    noiseGPOptMeanParas.set(noise);
+    //GP Opt var  
+    sigmaGPOptVarParas.set(sigma);
+    noiseGPOptVarParas.set(noise);    
     //Parzen  
     sigmaParzenParas.set(sigma);
     noiseParzenParas.set(noise);
@@ -837,6 +971,8 @@ int main (int argc, char **argv)
   double OverallPerformanceGPMean(0.0);
   double OverallPerformanceGPSRMean(0.0);
   double OverallPerformanceGPSRVar(0.0);  
+  double OverallPerformanceGPOptMean(0.0);
+  double OverallPerformanceGPOptVar(0.0);   
   double OverallPerformanceParzen(0.0);
   double OverallPerformanceSVDD(0.0);
 
@@ -847,6 +983,8 @@ int main (int argc, char **argv)
   double kernelSigmaGPMean;
   double kernelSigmaGPSRMean;
   double kernelSigmaGPSRVar;
+  double kernelSigmaGPOptMean;
+  double kernelSigmaGPOptVar;  
   double kernelSigmaParzen;
   double kernelSigmaSVDD;
   
@@ -860,8 +998,10 @@ int main (int argc, char **argv)
     kernelSigmaGPVar = sigmaGPVarParas[cl];
     kernelSigmaGPMeanApprox = sigmaGPMeanApproxParas[cl];
     kernelSigmaGPMean = sigmaGPMeanParas[cl];
-    kernelSigmaGPMean = sigmaGPSRMeanParas[cl];
+    kernelSigmaGPSRMean = sigmaGPSRMeanParas[cl];
     kernelSigmaGPSRVar = sigmaGPSRVarParas[cl];
+    kernelSigmaGPOptMean = sigmaGPOptMeanParas[cl];
+    kernelSigmaGPOptVar = sigmaGPOptVarParas[cl];    
     kernelSigmaParzen = sigmaParzenParas[cl];
     kernelSigmaSVDD = sigmaSVDDParas[cl];
     
@@ -925,7 +1065,17 @@ int main (int argc, char **argv)
     NICE::Matrix GPSRVarCholesky;   
     std::vector<int> indicesOfChosenExamplesGPSRVar;
     if (GPSRVar)
-      trainGPSRVar(GPSRVarCholesky, noiseGPSRVarParas[cl], kernelMatrix, nrOfExamplesPerClass, cl, runsPerClassToAverageTraining, nrOfRegressors, indicesOfChosenExamplesGPSRVar );      
+      trainGPSRVar(GPSRVarCholesky, noiseGPSRVarParas[cl], kernelMatrix, nrOfExamplesPerClass, cl, runsPerClassToAverageTraining, nrOfRegressors, indicesOfChosenExamplesGPSRVar );  
+    
+    //train GP Opt Mean
+    NICE::Vector GPOptMeanRightPart;
+    if (GPOptMean)
+      trainGPMeanApprox(GPOptMeanRightPart, noiseGPOptMeanParas[cl], kernelMatrix, nrOfExamplesPerClass, cl, runsPerClassToAverageTraining );
+    
+    //train GP Opt Var
+    NICE::Vector DiagGPOptVar;
+    if (GPOptVar)
+      trainGPMean(DiagGPOptVar, noiseGPOptVarParas[cl], kernelMatrix, nrOfExamplesPerClass, cl, runsPerClassToAverageTraining );     
     
     //train Parzen 
     //nothing to do :)
@@ -951,6 +1101,8 @@ int main (int argc, char **argv)
     ClassificationResults resultsGPMean;
     ClassificationResults resultsGPSRMean;
     ClassificationResults resultsGPSRVar;    
+    ClassificationResults resultsGPOptMean;
+    ClassificationResults resultsGPOptVar;    
     ClassificationResults resultsParzen;
     ClassificationResults resultsSVDD;       
     
@@ -963,8 +1115,10 @@ int main (int argc, char **argv)
     double timeForSingleExamplesGPVar(0.0);
     double timeForSingleExamplesGPMeanApprox(0.0);    
     double timeForSingleExamplesGPMean(0.0);    
-    double timeForSingleExamplesGPSRMean(0.0);    
-    double timeForSingleExamplesGPSRVar(0.0);    
+    double timeForSingleExamplesGPSRMean(0.0);
+    double timeForSingleExamplesGPSRVar(0.0);
+    double timeForSingleExamplesGPOptMean(0.0);
+    double timeForSingleExamplesGPOptVar(0.0);    
     double timeForSingleExamplesParzen(0.0);    
     double timeForSingleExamplesSVDD(0.0);    
     
@@ -1016,7 +1170,16 @@ int main (int argc, char **argv)
       ClassificationResult rGPSRVar;
       if (GPSRVar)
         evaluateGPSRVar( kernelVector, GPSRVarCholesky, rGPSRVar, timeForSingleExamplesGPSRVar, runsPerClassToAverageTesting, nrOfRegressors, indicesOfChosenExamplesGPSRVar, noiseGPSRVarParas[cl] );       
-
+      
+      //evaluate GP Opt Mean
+      ClassificationResult rGPOptMean;
+      if (GPOptMean)
+        evaluateGPOptMean( kernelVector, GPOptMeanRightPart, rGPOptMean, timeForSingleExamplesGPOptMean, runsPerClassToAverageTesting );       
+      
+      //evaluate GP Opt Var
+      ClassificationResult rGPOptVar;
+      if (GPOptVar)
+        evaluateGPOptVar( kernelVector, kernelSelf, DiagGPOptVar, rGPOptVar, timeForSingleExamplesGPOptVar, runsPerClassToAverageTesting );   
       
       //evaluate Parzen
       ClassificationResult rParzen;
@@ -1036,6 +1199,8 @@ int main (int argc, char **argv)
       rGPMean.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;
       rGPSRMean.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;
       rGPSRVar.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;
+      rGPOptMean.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;
+      rGPOptVar.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;      
       rParzen.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;
       rSVDD.classno_groundtruth = (((int)imageNetTest.getPreloadedLabel ( i )) == positiveClass) ? 1 : 0;
 
@@ -1046,6 +1211,8 @@ int main (int argc, char **argv)
       resultsGPMean.push_back ( rGPMean );
       resultsGPSRMean.push_back ( rGPSRMean );
       resultsGPSRVar.push_back ( rGPSRVar );
+      resultsGPOptMean.push_back ( rGPOptMean );
+      resultsGPOptVar.push_back ( rGPOptVar );      
       resultsParzen.push_back ( rParzen );
       resultsSVDD.push_back ( rSVDD );      
     }
@@ -1059,6 +1226,8 @@ int main (int argc, char **argv)
     timeForSingleExamplesGPMean/= imageNetTest.getNumPreloadedExamples();
     timeForSingleExamplesGPSRMean/= imageNetTest.getNumPreloadedExamples();
     timeForSingleExamplesGPSRVar/= imageNetTest.getNumPreloadedExamples();
+    timeForSingleExamplesGPOptMean/= imageNetTest.getNumPreloadedExamples();
+    timeForSingleExamplesGPOptVar/= imageNetTest.getNumPreloadedExamples();    
     timeForSingleExamplesParzen/= imageNetTest.getNumPreloadedExamples();
     timeForSingleExamplesSVDD/= imageNetTest.getNumPreloadedExamples();
     
@@ -1068,6 +1237,8 @@ int main (int argc, char **argv)
     std::cerr << "GPMean -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesGPMean << std::endl;    
     std::cerr << "GPSRMean -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesGPSRMean << std::endl;    
     std::cerr << "GPSRVar -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesGPSRVar << std::endl;    
+    std::cerr << "GPOptMean -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesGPOptMean << std::endl;    
+    std::cerr << "GPOptVar -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesGPOptVar << std::endl;    
     std::cerr << "Parzen -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesParzen << std::endl;    
     std::cerr << "SVDD -- time used for evaluation single elements of class " << cl << " : " << timeForSingleExamplesSVDD << std::endl;    
 
@@ -1078,6 +1249,8 @@ int main (int argc, char **argv)
     double perfvalueGPMean( 0.0 );
     double perfvalueGPSRMean( 0.0 );
     double perfvalueGPSRVar( 0.0 );
+    double perfvalueGPOptMean( 0.0 );
+    double perfvalueGPOptVar( 0.0 );    
     double perfvalueParzen( 0.0 );
     double perfvalueSVDD( 0.0 );     
 
@@ -1093,6 +1266,10 @@ int main (int argc, char **argv)
       perfvalueGPSRMean = resultsGPSRMean.getBinaryClassPerformance( ClassificationResults::PERF_AUC );
     if (GPSRVar)
       perfvalueGPSRVar = resultsGPSRVar.getBinaryClassPerformance( ClassificationResults::PERF_AUC );
+    if (GPOptMean)
+      perfvalueGPOptMean = resultsGPOptMean.getBinaryClassPerformance( ClassificationResults::PERF_AUC );
+    if (GPOptVar)
+      perfvalueGPOptVar = resultsGPOptVar.getBinaryClassPerformance( ClassificationResults::PERF_AUC );    
     if (Parzen)
       perfvalueParzen = resultsParzen.getBinaryClassPerformance( ClassificationResults::PERF_AUC );
     if (SVDD)
@@ -1105,6 +1282,8 @@ int main (int argc, char **argv)
     std::cerr << "Performance GPMean: " << perfvalueGPMean << std::endl;
     std::cerr << "Performance GPSRMean: " << perfvalueGPSRMean << std::endl;
     std::cerr << "Performance GPSRVar: " << perfvalueGPSRVar << std::endl;
+    std::cerr << "Performance GPOptMean: " << perfvalueGPOptMean << std::endl;
+    std::cerr << "Performance GPOptVar: " << perfvalueGPOptVar << std::endl;    
     std::cerr << "Performance Parzen: " << perfvalueParzen << std::endl;
     std::cerr << "Performance SVDD: " << perfvalueSVDD << std::endl;    
     
@@ -1114,6 +1293,8 @@ int main (int argc, char **argv)
     OverallPerformanceGPMean += perfvalueGPMean;
     OverallPerformanceGPSRMean += perfvalueGPSRMean;
     OverallPerformanceGPSRVar += perfvalueGPSRVar;
+    OverallPerformanceGPOptMean += perfvalueGPOptMean;
+    OverallPerformanceGPOptVar += perfvalueGPOptVar;    
     OverallPerformanceParzen += perfvalueParzen;
     OverallPerformanceSVDD += perfvalueSVDD;   
 
@@ -1128,6 +1309,8 @@ int main (int argc, char **argv)
   OverallPerformanceGPMean /= nrOfClassesToConcidere;
   OverallPerformanceGPSRMean /= nrOfClassesToConcidere;
   OverallPerformanceGPSRVar /= nrOfClassesToConcidere;
+  OverallPerformanceGPOptMean /= nrOfClassesToConcidere;
+  OverallPerformanceGPOptVar /= nrOfClassesToConcidere;  
   OverallPerformanceParzen /= nrOfClassesToConcidere;
   OverallPerformanceSVDD /= nrOfClassesToConcidere;  
   
@@ -1137,6 +1320,8 @@ int main (int argc, char **argv)
   std::cerr << "overall performance GPMean: " << OverallPerformanceGPMean << std::endl;
   std::cerr << "overall performance GPSRMean: " << OverallPerformanceGPSRMean << std::endl;
   std::cerr << "overall performance GPSRVar: " << OverallPerformanceGPSRVar << std::endl;
+  std::cerr << "overall performance GPOptMean: " << OverallPerformanceGPOptMean << std::endl;
+  std::cerr << "overall performance GPOptVar: " << OverallPerformanceGPOptVar << std::endl;  
   std::cerr << "overall performance Parzen: " << OverallPerformanceParzen << std::endl;
   std::cerr << "overall performance SVDD: " << OverallPerformanceSVDD << std::endl;