GPHIK IL - more test units, running IL on empty classifier is now possible, updated addMultipleExamples

FMKGPHyperparameterOptimization.cpp

@@ -45,8 +45,7 @@ using namespace std;
 /////////////////////////////////////////////////////
 /////////////////////////////////////////////////////
 
-void FMKGPHyperparameterOptimization::updateAfterSingleIncrement ( 
-      const NICE::SparseVector & x,
+void FMKGPHyperparameterOptimization::updateAfterIncrement ( 
       const std::set < int > newClasses,
       const bool & performOptimizationAfterIncrement )
 {
@@ -122,7 +121,7 @@ void FMKGPHyperparameterOptimization::updateAfterSingleIncrement (
   if ( this->verboseTime )
     std::cerr << "Time used for setting up the alpha-objects: " << t1.getLast() << std::endl;
 
-    std::cerr << "update Eigendecomposition " << std::endl;
+  std::cerr << "update Eigendecomposition " << std::endl;
   t1.start();
   // we compute all needed eigenvectors for standard classification and variance prediction at ones.
   // nrOfEigenvaluesToConsiderForVarApprox should NOT be larger than 1 if a method different than approximate_fine is used!
@@ -131,6 +130,12 @@ void FMKGPHyperparameterOptimization::updateAfterSingleIncrement (
   if ( this->verboseTime )
     std::cerr << "Time used for setting up the eigenvectors-objects: " << t1.getLast() << std::endl;
 
+  
+  //////////////////////  //////////////////////
+  //   RE-RUN THE OPTIMIZATION, IF DESIRED    //
+  //////////////////////  //////////////////////    
+  
+  
   if ( this->verbose )
     std::cerr << "resulting eigenvalues for first class: " << eigenMax[0] << std::endl;
 
@@ -144,7 +149,7 @@ void FMKGPHyperparameterOptimization::updateAfterSingleIncrement (
   // as initialization
   // OPT_NONE
   // nothing to do, obviously
-  //NOTE we could skip this, if we do not want to change our parameters given new examples
+    
   if ( this->verbose )
     std::cerr << "perform optimization after increment " << std::endl;
   
@@ -157,245 +162,29 @@ void FMKGPHyperparameterOptimization::updateAfterSingleIncrement (
     this->optimizationMethod = OPT_NONE;
   }
     
-    t1.start();
-    //TODO add option for handing over previous solution!
-    this->performOptimization ( *gplike, parameterVectorSize);
-//         this->performOptimization ( *gplike, parameterVectorSize, false /* initialize not with default values but using the last solution */ );
-    t1.stop();
-    if ( this->verboseTime )
-      std::cerr << "Time used for performing the optimization: " << t1.getLast() << std::endl;
-
-    if ( this->verbose )
-      std::cerr << "Preparing after retraining for classification ..." << std::endl;
-
-    t1.start();
-    this->transformFeaturesWithOptimalParameters ( *gplike, parameterVectorSize );
-    t1.stop();
-    if ( this->verboseTime)
-      std::cerr << "Time used for transforming features with optimal parameters: " << t1.getLast() << std::endl;
-
-  if ( !performOptimizationAfterIncrement )
-  {
-    this->optimizationMethod = optimizationMethodTmpCopy;
-  }
-
-  
-  //NOTE unfortunately, the whole vector alpha differs, and not only its last entry.
-  // If we knew any method, which could update this efficiently, we could also compute A and B more efficiently by updating them.
-  // Since we are not aware of any such method, we have to compute them completely new
-  // :/
-  t1.start();
-  this->computeMatricesAndLUTs ( *gplike );
-  t1.stop();
-  if ( this->verboseTime )
-    std::cerr << "Time used for setting up the A'nB -objects: " << t1.getLast() << std::endl;
-
-  t.stop();  
-
-  NICE::ResourceStatistics rs;
-  std::cerr << "Time used for re-learning: " << t.getLast() << std::endl;
-  long maxMemory;
-  rs.getMaximumMemory ( maxMemory );
-  std::cerr << "Maximum memory used: " << maxMemory << " KB" << std::endl;
-
-  //don't waste memory
-  delete gplike;
-}
-
-void FMKGPHyperparameterOptimization::updateAfterMultipleIncrements ( 
-      const std::vector<const NICE::SparseVector*> & x, 
-      const std::set < int > newClasses,
-      const bool & performOptimizationAfterIncrement )
-{
-  Timer t;
-  t.start();
-  if ( fmk == NULL )
-    fthrow ( Exception, "FastMinKernel object was not initialized!" );
-
-  std::map<int, NICE::Vector> binaryLabels;
-  std::set<int> classesToUse;
-  this->prepareBinaryLabels ( binaryLabels, labels , classesToUse );
-  //actually, this is not needed, since we have the global set knownClasses
-  classesToUse.clear();
-  
-  std::map<int, NICE::Vector> newBinaryLabels;
-  if ( newClasses.size() > 0)
-  {
-    for (std::set<int>::const_iterator newClIt = newClasses.begin(); newClIt != newClasses.end(); newClIt++)
-    {
-      std::map<int, NICE::Vector>::iterator binLabelIt = binaryLabels.find(*newClIt);
-      newBinaryLabels.insert(*binLabelIt);
-    }
-  }
-  
-  if ( this->verbose )
-    std::cerr << "labels.size() after increment: " << this->labels.size() << std::endl;
- 
-  
-  // ************************************************************
-  //   include the information for classes we know so far    
-  // ************************************************************
-  if ( this->verbose)
-    std::cerr <<  "include the information for classes we know so far " << std::endl;
-  
-  NICE::Timer t1;
   t1.start();
-  //update the kernel combinations
-  //TODO verify that this is not needed anymore in any IKMLinearCombination class
-    
-  //we have to reset the fmk explicitely
-  ( ( GMHIKernel* ) this->ikmsum->getModel ( this->ikmsum->getNumberOfModels() - 1 ) )->setFastMinKernel ( this->fmk );
-
+  //TODO add option for handing over previous solution!
+  this->performOptimization ( *gplike, parameterVectorSize);
+//         this->performOptimization ( *gplike, parameterVectorSize, false /* initialize not with default values but using the last solution */ );
   t1.stop();
   if ( this->verboseTime )
-    std::cerr << "Time used for setting up the ikm-objects for known classes: " << t1.getLast() << std::endl;
-  
-  // *********************************************
-  //          work on the new classes
-  // *********************************************
-    
-  if ( this->verbose )    
-    std::cerr << "work on the new classes " << std::endl;
-  
-  double tmpNoise;  
-  (this->ikmsum->getModel( 0 ))->getFirstDiagonalElement(tmpNoise);
-    
-
-  
-  // ******************************************************************************************
-  //       now do everything which is independent of the number of new classes
-  // ******************************************************************************************  
+    std::cerr << "Time used for performing the optimization: " << t1.getLast() << std::endl;
 
   if ( this->verbose )
-    std::cerr << "now do everything which is independent of the number of new classes" << std::endl;
-
-  NICE::GPLikelihoodApprox * gplike;
-  uint parameterVectorSize;
-
-  t1.start();
-  this->setupGPLikelihoodApprox ( gplike, binaryLabels, parameterVectorSize );
-  t1.stop();
-  if ( this->verboseTime )
-    std::cerr << "Time used for setting up the gplike-objects: " << t1.getLast() << std::endl;
-
-  t1.start();
-  // we compute all needed eigenvectors for standard classification and variance prediction at ones.
-  // nrOfEigenvaluesToConsiderForVarApprox should NOT be larger than 1 if a method different than approximate_fine is used!
-  this->updateEigenDecomposition(  std::max ( this->nrOfEigenvaluesToConsider, this->nrOfEigenvaluesToConsiderForVarApprox) );
-  t1.stop();
-  if ( this->verboseTime )
-    std::cerr << "Time used for setting up the eigenvectors-objects: " << t1.getLast() << std::endl;
+    std::cerr << "Preparing after retraining for classification ..." << std::endl;
 
   t1.start();
-  if ( this->b_usePreviousAlphas )
-  {
-    double factor ( 1.0 / this->eigenMax[0] );
-	  
-    std::map<int, NICE::Vector>::const_iterator binaryLabelsIt = binaryLabels.begin();
-    
-    for ( std::map<int, NICE::Vector>::iterator lastAlphaIt = lastAlphas.begin() ;lastAlphaIt != lastAlphas.end(); lastAlphaIt++ )
-    {
- 
-      int oldSize ( lastAlphaIt->second.size() );
-      lastAlphaIt->second.resize ( oldSize + x.size() );
-
-      //We initialize it with the same values as we use in GPLikelihoodApprox in batch training
-      //default in GPLikelihoodApprox for the first time:
-      // alpha = (binaryLabels[classCnt] * (1.0 / eigenmax[0]) );
-         
-
-      for ( uint i = 0; i < x.size(); i++ )
-      {
-        if ( binaryLabelsIt->second[oldSize+i] > 0 ) //we only have +1 and -1, so this might be benefitial in terms of speed
-          lastAlphaIt->second[oldSize+i] = factor;
-        else
-          lastAlphaIt->second[oldSize+i] = -factor; //we follow the initialization as done in previous steps
-          //lastAlphaIt->second[oldSize+i] = 0.0; // following the suggestion of Yeh and Darrell
-      }
-
-      binaryLabelsIt++;
-    }
-
-    //compute unaffected alpha-vectors for the new classes
-    for (std::set<int>::const_iterator newClIt =  newClasses.begin(); newClIt != newClasses.end(); newClIt++)
-    {      
-      NICE::Vector alphaVec = (binaryLabels[*newClIt] * factor ); //see GPLikelihoodApprox for an explanation
-      lastAlphas.insert( std::pair<int, NICE::Vector>(*newClIt, alphaVec) );
-    }      
-
-    //TODO check that memory will not be erased when calling delete!
-    gplike->setInitialAlphaGuess ( &lastAlphas );
-  }
-  else
-  {
-  }
-  
-  //if we do not use previous alphas, we do not have to set up anything here  
+  this->transformFeaturesWithOptimalParameters ( *gplike, parameterVectorSize );
   t1.stop();
-  if ( this->verboseTime )
-    std::cerr << "Time used for setting up the alpha-objects: " << t1.getLast() << std::endl;  
-
-  if ( this->verbose )
-    std::cerr << "resulting eigenvalues of first class: " << eigenMax[0] << std::endl;
+  if ( this->verboseTime)
+    std::cerr << "Time used for transforming features with optimal parameters: " << t1.getLast() << std::endl;
 
-  // we can reuse the already given performOptimization-method:
-  // OPT_GREEDY
-  // for this strategy we can't reuse any of the previously computed scores
-  // so come on, let's do the whole thing again...
-  // OPT_DOWNHILLSIMPLEX
-  // Here we can benefit from previous results, when we use them as initialization for our optimizer
-  // ikmsums.begin()->second->getParameters ( currentParameters ); uses the previously computed optimal parameters
-  // as initialization
-  // OPT_NONE
-  // nothing to do, obviously
-  //NOTE we can skip this, if we do not want to change our parameters given new examples
-  if ( performOptimizationAfterIncrement )
-  {
-    t1.start();
-    //TODO add option for handing over previous solution!
-    this->performOptimization ( *gplike, parameterVectorSize);
-//         this->performOptimization ( *gplike, parameterVectorSize, false /* initialize not with default values but using the last solution */ );
-    t1.stop();
-    if ( this->verboseTime )
-      std::cerr << "Time used for performing the optimization: " << t1.getLast() << std::endl;
-    
-    t1.start();
-    this->transformFeaturesWithOptimalParameters ( *gplike, parameterVectorSize );
-    t1.stop();
-    if ( this->verboseTime)
-      std::cerr << "Time used for transforming features with optimal parameters: " << t1.getLast() << std::endl;
-  }
-  else
+  if ( !performOptimizationAfterIncrement )
   {
-    //deactivate the optimization method;
-    int originalOptimizationMethod = optimizationMethod;
-    this->optimizationMethod = OPT_NONE;
-    //and deactive the noise-optimization as well
-    if ( this->optimizeNoise )
-      this->optimizeNoise = false;
-    
-    t1.start();
-    //this is needed to compute the alpha vectors for the standard parameter settings
-    //TODO add option for handing over previous solution!
-    this->performOptimization ( *gplike, parameterVectorSize);
-//         this->performOptimization ( *gplike, parameterVectorSize, false /* initialize not with default values but using the last solution */ );
-    t1.stop();
-    std::cerr << "skip optimization after increment" << std::endl;
-    if ( this->verboseTime )
-      std::cerr << "Time used for performing the optimization: " << t1.getLast() << std::endl;
-
-    std::cerr << "skip feature transformation" << std::endl;
-    if ( this->verboseTime )
-      std::cerr << "Time used for transforming features with optimal parameters: " << t1.getLast() << std::endl;
-    
-    //re-activate the optimization method
-    this->optimizationMethod = originalOptimizationMethod;    
+    this->optimizationMethod = optimizationMethodTmpCopy;
   }
 
-  if ( this->verbose )
-    std::cerr << "Preparing after retraining for classification ..." << std::endl;
-
-
+  
   //NOTE unfortunately, the whole vector alpha differs, and not only its last entry.
   // If we knew any method, which could update this efficiently, we could also compute A and B more efficiently by updating them.
   // Since we are not aware of any such method, we have to compute them completely new
@@ -406,7 +195,7 @@ void FMKGPHyperparameterOptimization::updateAfterMultipleIncrements (
   if ( this->verboseTime )
     std::cerr << "Time used for setting up the A'nB -objects: " << t1.getLast() << std::endl;
 
-  t.stop();
+  t.stop();  
 
   NICE::ResourceStatistics rs;
   std::cerr << "Time used for re-learning: " << t.getLast() << std::endl;
@@ -418,7 +207,6 @@ void FMKGPHyperparameterOptimization::updateAfterMultipleIncrements (
   delete gplike;
 }
 
-
 /////////////////////////////////////////////////////
 /////////////////////////////////////////////////////
 //                 PUBLIC METHODS
@@ -807,8 +595,6 @@ void FMKGPHyperparameterOptimization::computeMatricesAndLUTs ( const GPLikelihoo
       
       precomputedT[ i->first ] = T;
     }
-    
-    //TODO update the variance-related matrices as well here - currently it is done before in the outer method!!!
   }
   
   if ( this->precomputedTForVarEst != NULL )
@@ -935,9 +721,10 @@ int FMKGPHyperparameterOptimization::prepareBinaryLabels ( std::map<int, NICE::V
   {
     //we set the labels to 1, independent of the previously given class number
     //however, the original class numbers are stored and returned in classification
-    Vector yNew ( y.size(), 1 );
-    myClasses.clear();
-    myClasses.insert ( 1 );
+    NICE::Vector yOne ( y.size(), 1 );
+    
+    binaryLabels[ *(myClasses.begin()) ] = yOne;
+    
     //we have to indicate, that we are in an OCC setting
     nrOfClasses--;
   }
@@ -996,7 +783,6 @@ void FMKGPHyperparameterOptimization::optimize ( std::map<int, NICE::Vector> & b
   // set pretty low built-in noise, because we explicitely add the noise with the IKMNoise
   fmk->setNoise ( 0.0 );
 
-  //NOTE The GMHIKernel is always the last model which is added (this is necessary for easy store and restore functionality)
   ikmsum->addModel ( new GMHIKernel ( fmk, pf, NULL /* no quantization */ ) );
 
   t1.stop();
@@ -1432,8 +1218,6 @@ void FMKGPHyperparameterOptimization::computePredictiveVarianceApproximateFine (
   }  
   
   // ---------------- compute the first term --------------------
-//   Timer t;
-//   t.start();
 
   double kSelf ( 0.0 );
   int dim ( 0 );
@@ -1444,27 +1228,18 @@ void FMKGPHyperparameterOptimization::computePredictiveVarianceApproximateFine (
     //kSelf += pf->f(dim,*it);
   }
   // ---------------- compute the approximation of the second term --------------------
-//    t.stop();  
-//   std::cerr << "ApproxFine -- time for first term: "  << t.getLast()  << std::endl;
-
-//   t.start();
   NICE::Vector kStar;
   fmk->hikComputeKernelVector ( x, kStar );
-/*  t.stop();
-  std::cerr << "ApproxFine -- time for kernel vector: "  << t.getLast()  << std::endl;*/
 
-    
-//     NICE::Vector multiplicationResults; // will contain nrOfEigenvaluesToConsiderForVarApprox many entries
-//     multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
-    NICE::Vector multiplicationResults( nrOfEigenvaluesToConsiderForVarApprox-1, 0.0 );
+
     //ok, there seems to be a nasty thing in computing multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
     //wherefor it takes aeons...
     //so we compute it by ourselves
-    
-    
-//     for ( uint tmpI = 0; tmpI < kStar.size(); tmpI++)
+//     NICE::Vector multiplicationResults; // will contain nrOfEigenvaluesToConsiderForVarApprox many entries
+//     multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
+
+    NICE::Vector multiplicationResults( nrOfEigenvaluesToConsiderForVarApprox-1, 0.0 );
     NICE::Matrix::const_iterator eigenVecIt = eigenMaxVectors.begin();
-//       double kStarI ( kStar[tmpI] );
     for ( int tmpJ = 0; tmpJ < nrOfEigenvaluesToConsiderForVarApprox-1; tmpJ++)
     {
       for ( NICE::Vector::const_iterator kStarIt = kStar.begin(); kStarIt != kStar.end(); kStarIt++,eigenVecIt++)
@@ -1508,9 +1283,7 @@ void FMKGPHyperparameterOptimization::computePredictiveVarianceExact ( const NIC
   {
     fthrow ( Exception, "ikmsum is empty... have you trained this classifer? Aborting..." );
   }  
-  
-    Timer t;
-//   t.start();
+
   // ---------------- compute the first term --------------------
   double kSelf ( 0.0 );
   int dim ( 0 );
@@ -1522,21 +1295,13 @@ void FMKGPHyperparameterOptimization::computePredictiveVarianceExact ( const NIC
   }
 
   // ---------------- compute the second term --------------------
-//     t.stop();  
-//   std::cerr << "ApproxExact -- time for first term: "  << t.getLast()  << std::endl;
-
-//   t.start();  
   NICE::Vector kStar;
   fmk->hikComputeKernelVector ( x, kStar );
-//  t.stop();
-//   std::cerr << "ApproxExact -- time for kernel vector: "  << t.getLast()  << std::endl;
-//   
 
   //now run the ILS method
   NICE::Vector diagonalElements;
   ikmsum->getDiagonalElements ( diagonalElements );
 
-//     t.start();
   // init simple jacobi pre-conditioning
   ILSConjugateGradients *linsolver_cg = dynamic_cast<ILSConjugateGradients *> ( linsolver );
 
@@ -1560,16 +1325,7 @@ void FMKGPHyperparameterOptimization::computePredictiveVarianceExact ( const NIC
       *  This reduces the number of iterations by 5 or 8
       */  
   beta = (kStar * (1.0 / eigenMax[0]) );
-/*    t.stop();
-std::cerr << "ApproxExact -- time for preconditioning etc: "  << t.getLast()  << std::endl;    
-  
-t.start();*/
-  //   t.start();
   linsolver->solveLin ( *ikmsum, kStar, beta );
-  //   t.stop();
-//     t.stop();
-//         t.stop();
-//   std::cerr << "ApproxExact -- time for lin solve: "  << t.getLast()  << std::endl;
 
   beta *= kStar;
   
@@ -1600,7 +1356,7 @@ void FMKGPHyperparameterOptimization::restore ( std::istream & is, int format )
     if ( ! this->isStartTag( tmp, "FMKGPHyperparameterOptimization" ) )
     {
         std::cerr << " WARNING - attempt to restore FMKGPHyperparameterOptimization, but start flag " << tmp << " does not match! Aborting... " << std::endl;
-	throw;
+      throw;
     } 
 
     if (fmk != NULL)
@@ -1636,15 +1392,15 @@ void FMKGPHyperparameterOptimization::restore ( std::istream & is, int format )
       tmp = this->removeStartTag ( tmp );
       
       if ( b_restoreVerbose )
-	std::cerr << " currently restore section " << tmp << " in FMKGPHyperparameterOptimization" << std::endl;
+        std::cerr << " currently restore section " << tmp << " in FMKGPHyperparameterOptimization" << std::endl;
       
       if ( tmp.compare("fmk") == 0 )
       {
-	fmk = new FastMinKernel;
+        fmk = new FastMinKernel;
         fmk->restore( is, format );
 
-	is >> tmp; // end of block 
-	tmp = this->removeEndTag ( tmp );
+        is >> tmp; // end of block 
+        tmp = this->removeEndTag ( tmp );
       }      
       else if ( tmp.compare("precomputedA") == 0 )
       {
@@ -1665,8 +1421,8 @@ void FMKGPHyperparameterOptimization::restore ( std::istream & is, int format )
           precomputedA.insert ( std::pair<int, PrecomputedType> ( nr, pct ) );
         }
         
-	is >> tmp; // end of block 
-	tmp = this->removeEndTag ( tmp );
+        is >> tmp; // end of block 
+        tmp = this->removeEndTag ( tmp );
       }        
       else if ( tmp.compare("precomputedB") == 0 )
       {
@@ -1687,8 +1443,8 @@ void FMKGPHyperparameterOptimization::restore ( std::istream & is, int format )
           precomputedB.insert ( std::pair<int, PrecomputedType> ( nr, pct ) );
         }    
         
-	is >> tmp; // end of block 
-	tmp = this->removeEndTag ( tmp );
+        is >> tmp; // end of block 
+        tmp = this->removeEndTag ( tmp );
       }       
       else if ( tmp.compare("precomputedT") == 0 )
       {
@@ -2077,16 +1833,15 @@ void FMKGPHyperparameterOptimization::addExample( const NICE::SparseVector * exa
   if ( this->verboseTime)
     std::cerr << "Time used for adding the data to the fmk object: " << t.getLast() << std::endl;
 
-  //TODO update the matrix for variance computations as well!!!
   
   // add examples to all implicite kernel matrices we currently use
   this->ikmsum->addExample ( example, label, performOptimizationAfterIncrement );
   
-  std::cerr << "call updateAfterSingleIncrement " << std::endl;
+  std::cerr << "call updateAfterIncrement " << std::endl;
   
   // update the corresponding matrices A, B and lookup tables T  
   // optional: do the optimization again using the previously known solutions as initialization
-  this->updateAfterSingleIncrement ( *example, newClasses, performOptimizationAfterIncrement );
+  this->updateAfterIncrement ( newClasses, performOptimizationAfterIncrement );
   
   //clean up
   newClasses.clear();
@@ -2099,92 +1854,44 @@ void FMKGPHyperparameterOptimization::addMultipleExamples( const std::vector< co
 				      const bool & performOptimizationAfterIncrement
 				    )
 {
-// // //   //TODO check whether this set is actually needed
-// // //   std::set< int > newClasses;
-// // // 
-// // //   if (this->knownClasses.size() == 1) //binary setting
-// // //   {
-// // //     int oldSize ( this->labels.size() );
-// // //     this->labels.resize ( this->labels.size() + newLabels.size() );
-// // //     for ( uint i = 0; i < newLabels.size(); i++ )
-// // //     {
-// // //       this->labels[i+oldSize] = newLabels[i];
-// // //       //have we seen this class already?
-// // //       if ( (newLabels[i]  != this->binaryLabelPositive) && (newLabels[i]  != this->binaryLabelNegative) )
-// // //       {
-// // //         fthrow(Exception, "Binary setting does not allow adding new classes so far");
-// // // //         knownClasses.insert( newLabels[i] );
-// // // //         newClasses.insert( newLabels[i] );
-// // //       }
-// // //     }      
-// // //   }
-// // //   else //multi-class setting
-// // //   {
-// // //     int oldSize ( this->labels.size() );
-// // //     this->labels.resize ( this->labels.size() + newLabels.size() );
-// // //     for ( uint i = 0; i < newLabels.size(); i++ )
-// // //     {
-// // //       this->labels[i+oldSize] = newLabels[i];
-// // //       //have we seen this class already?
-// // //       if (knownClasses.find( newLabels[i] ) == knownClasses.end() )
-// // //       {
-// // //         knownClasses.insert( newLabels[i] );
-// // //         newClasses.insert( newLabels[i] );
-// // //       }
-// // //     }    
-// // //   }
-// // //   
-// // // 
-// // // 
-// // //   // add the new example to our data structure
-// // //   // It is necessary to do this already here and not lateron for internal reasons (see GMHIKernel for more details)
-// // //   Timer t;
-// // //   t.start();
-// // //   
-// // //   fmk->addMultipleExamples ( newExamples, pf );  
-// // //   t.stop();
-// // //   if (verboseTime)
-// // //     std::cerr << "Time used for adding the data to the fmk object: " << t.getLast() << std::endl;
-// // //   
-// // //     // add examples to all implicite kernel matrices we currently use
-// // //   this->ikmsum->addExample ( example, label, performOptimizationAfterIncrement );
-// // //   
-// // //   Timer tVar;
-// // //   tVar.start();  
-// // //   //do we need  to update our matrices?
-// // //   if ( precomputedAForVarEst.size() != 0)
-// // //   {
-// // //     std::cerr << "update the variance matrices " << std::endl;
-// // //     //this computes everything from the scratch
-// // //     this->prepareVarianceApproximation();
-// // //     //this would perform a more sophisticated update
-// // //     //unfortunately, there is a bug somewhere
-// // //     //TODO fixme!
-// // // //     std::cerr << "update the LUTs needed for variance computation" << std::endl;
-// // // //     for ( std::vector<const NICE::SparseVector*>::const_iterator exampleIt = newExamples.begin(); exampleIt != newExamples.end(); exampleIt++ )
-// // // //     {  
-// // // //       std::cerr << "new example: " << std::endl;
-// // // //       (**exampleIt).store(std::cerr);
-// // // //       std::cerr << "now update the LUT for var est" << std::endl;
-// // // //       fmk->updatePreparationForKVNApproximation( **exampleIt, precomputedAForVarEst, pf );  
-// // // //       if ( q != NULL )
-// // // //       {
-// // // //         fmk->updateLookupTableForKVNApproximation( **exampleIt, precomputedTForVarEst, *q, pf );
-// // // //       }
-// // // //     }
-// // // //     std::cerr << "update of LUTs for variance compuation done" << std::endl;
-// // //   }
-// // //   tVar.stop();
-// // //   if (verboseTime)
-// // //     std::cerr << "Time used for computing the Variance Matrix and LUT: " << tVar.getLast() << std::endl;  
-// // //   
-// // // 
-// // // 
-// // //   // update the corresponding matrices A, B and lookup tables T
-// // //   // optional: do the optimization again using the previously known solutions as initialization
-// // //   updateAfterMultipleIncrements ( newExamples, newClasses, performOptimizationAfterIncrement );
-// // //   
-// // //   //clean up
-// // //   newClasses.clear();
+  std::cerr << " --- FMKGPHyperparameterOptimization::addMultipleExamples --- " << std::endl;  
+
+  std::set< int > newClasses;
+  
+  this->labels.append ( newLabels );
+  //have we seen this class already?
+  for ( NICE::Vector::const_iterator vecIt = newLabels.begin(); 
+       vecIt != newLabels.end(); vecIt++
+      )
+  {  
+      if ( this->knownClasses.find( *vecIt ) == this->knownClasses.end() )
+    {
+      this->knownClasses.insert( *vecIt );
+      newClasses.insert( *vecIt );
+    } 
+  }
+  
+  std::cerr << "call addExample of fmk object " << std::endl;
+  // add the new example to our data structure
+  // It is necessary to do this already here and not lateron for internal reasons (see GMHIKernel for more details)
+  NICE::Timer t;
+  t.start();
+  this->fmk->addMultipleExamples ( newExamples, pf );
+  t.stop();
+  if ( this->verboseTime)
+    std::cerr << "Time used for adding the data to the fmk object: " << t.getLast() << std::endl;
+  
+  // add examples to all implicite kernel matrices we currently use
+  this->ikmsum->addMultipleExamples ( newExamples, newLabels, performOptimizationAfterIncrement );
+  
+  std::cerr << "call updateAfterIncrement " << std::endl;
+  
+  // update the corresponding matrices A, B and lookup tables T  
+  // optional: do the optimization again using the previously known solutions as initialization
+  this->updateAfterIncrement ( newClasses, performOptimizationAfterIncrement );
+
+  //clean up
+  newClasses.clear();
   
+  std::cerr << " --- FMKGPHyperparameterOptimization::addMultipleExamples done --- " << std::endl;    
 }

FMKGPHyperparameterOptimization.h

@@ -154,18 +154,12 @@ class FMKGPHyperparameterOptimization : public NICE::Persistent, public NICE::On
     //! we store the alpha vectors for good initializations in the IL setting
     std::map<int, NICE::Vector> lastAlphas;  
 
-    //! Update matrices (A, B, LUTs) and optionally find optimal parameters after adding a new example.  
-    void updateAfterSingleIncrement (
-      const NICE::SparseVector & x, 
+    //! Update matrices (A, B, LUTs) and optionally find optimal parameters after adding (a) new example(s).  
+    void updateAfterIncrement (
       const std::set<int> newClasses,
       const bool & performOptimizationAfterIncrement = false
     );    
-    //! Update matrices (A, B, LUTs) and optionally find optimal parameters after adding multiple examples.  
-    void updateAfterMultipleIncrements (
-      const std::vector<const NICE::SparseVector*> & x, 
-      const std::set<int> newClasses,
-      const bool & performOptimizationAfterIncrement = false
-    );     
+  
 
     
   public:  

FastMinKernel.cpp

@@ -57,7 +57,7 @@ FastMinKernel::FastMinKernel ( const sparse_t & X, const double noise, const std
 }
 #endif
 
-FastMinKernel::FastMinKernel ( const vector< SparseVector * > & X, const double noise, const bool _debug, const bool & dimensionsOverExamples, const int & _dim)
+FastMinKernel::FastMinKernel ( const std::vector< const NICE::SparseVector * > & X, const double noise, const bool _debug, const bool & dimensionsOverExamples, const int & _dim)
 {
   this->setDebug(_debug);
   this->hik_prepare_kernel_multiplications ( X, this->X_sorted, dimensionsOverExamples, _dim);
@@ -112,7 +112,7 @@ void FastMinKernel::hik_prepare_kernel_multiplications(const std::vector<std::ve
   X_sorted.set_features(X, _dim);
 }
 
-void FastMinKernel::hik_prepare_kernel_multiplications(const std::vector< NICE::SparseVector * > & X, NICE::FeatureMatrixT<double> & X_sorted, const bool & dimensionsOverExamples, const int & _dim)
+void FastMinKernel::hik_prepare_kernel_multiplications(const std::vector< const NICE::SparseVector * > & X, NICE::FeatureMatrixT<double> & X_sorted, const bool & dimensionsOverExamples, const int & _dim)
 {
   X_sorted.set_features(X, dimensionsOverExamples, _dim);
 }

FastMinKernel.h

@@ -79,7 +79,7 @@ namespace NICE {
       */
       void hik_prepare_kernel_multiplications(const std::vector<std::vector<double> > & X, NICE::FeatureMatrixT<double> & X_sorted, const int & _dim = -1);
       
-      void hik_prepare_kernel_multiplications ( const std::vector< NICE::SparseVector * > & X, NICE::FeatureMatrixT<double> & X_sorted, const bool & dimensionsOverExamples, const int & _dim = -1);
+      void hik_prepare_kernel_multiplications ( const std::vector< const NICE::SparseVector * > & X, NICE::FeatureMatrixT<double> & X_sorted, const bool & dimensionsOverExamples, const int & _dim = -1);
       
       void randomPermutation(NICE::Vector & permutation, const std::vector<int> & oldIndices, const int & newSize) const;
       
@@ -113,7 +113,7 @@ namespace NICE {
       * @param X vector of sparse vector pointers
       * @param noise GP noise
       */
-      FastMinKernel( const std::vector< SparseVector * > & X, const double noise, const bool _debug = false, const bool & dimensionsOverExamples=false, const int & _dim = -1);
+      FastMinKernel( const std::vector< const NICE::SparseVector * > & X, const double noise, const bool _debug = false, const bool & dimensionsOverExamples=false, const int & _dim = -1);
 
 #ifdef NICE_USELIB_MATIO
       /**

FeatureMatrixT.h

@@ -97,7 +97,7 @@ template<class T> class FeatureMatrixT : NICE::Persistent
 #endif
 
     /** just another constructor for sparse features */
-    FeatureMatrixT(const std::vector< SparseVector * > & X, const bool dimensionsOverExamples = false, const int & _dim = -1);
+    FeatureMatrixT(const std::vector< const NICE::SparseVector * > & X, const bool dimensionsOverExamples = false, const int & _dim = -1);
     
 #ifdef NICE_USELIB_MATIO
     /**
@@ -303,7 +303,7 @@ template<class T> class FeatureMatrixT : NICE::Persistent
     void set_features(const std::vector<std::vector<T> > & _features, std::vector<std::vector<int> > & permutations, const int & _dim = -1);
     void set_features(const std::vector<std::vector<T> > & _features, std::vector<std::map<int,int> > & permutations, const int & _dim = -1);
     void set_features(const std::vector<std::vector<T> > & _features, const int & _dim = -1);
-    void set_features(const std::vector< NICE::SparseVector * > & _features, const bool dimensionsOverExamples = false, const int & _dim = -1);
+    void set_features(const std::vector< const NICE::SparseVector * > & _features, const bool dimensionsOverExamples = false, const int & _dim = -1);
     
     /**
     * @brief get a permutation vector for each dimension

FeatureMatrixT.tcc

@@ -51,7 +51,7 @@ namespace NICE {
     //Constructor reading data from a vector of sparse vector pointers
     template <typename T>
     FeatureMatrixT<T>::
-    FeatureMatrixT(const std::vector< SparseVector * > & X, const bool dimensionsOverExamples, const int & _dim)
+    FeatureMatrixT(const std::vector< const NICE::SparseVector * > & X, const bool dimensionsOverExamples, const int & _dim)
     {
       features.clear();
       
@@ -630,7 +630,7 @@ namespace NICE {
     }
     
     template <typename T>
-    void FeatureMatrixT<T>::set_features(const std::vector< NICE::SparseVector * > & _features, const bool dimensionsOverExamples, const int & _dim)
+    void FeatureMatrixT<T>::set_features(const std::vector< const NICE::SparseVector * > & _features, const bool dimensionsOverExamples, const int & _dim)
     {   
       features.clear();
       if (_features.size() == 0)

GPHIKClassifier.cpp

@@ -142,19 +142,19 @@ std::set<int> GPHIKClassifier::getKnownClassNumbers ( ) const
 //                      CLASSIFIER STUFF
 ///////////////////// ///////////////////// /////////////////////
 
-void GPHIKClassifier::classify ( const SparseVector * example,  int & result, SparseVector & scores )
+void GPHIKClassifier::classify ( const SparseVector * example,  int & result, SparseVector & scores ) const
 {
   double tmpUncertainty;
   this->classify( example, result, scores, tmpUncertainty );
 }
 
-void GPHIKClassifier::classify ( const NICE::Vector * example,  int & result, SparseVector & scores )
+void GPHIKClassifier::classify ( const NICE::Vector * example,  int & result, SparseVector & scores ) const
 {
   double tmpUncertainty;
   this->classify( example, result, scores, tmpUncertainty );
 }
 
-void GPHIKClassifier::classify ( const SparseVector * example,  int & result, SparseVector & scores, double & uncertainty )
+void GPHIKClassifier::classify ( const SparseVector * example,  int & result, SparseVector & scores, double & uncertainty ) const
 {
   if (gphyper == NULL)
      fthrow(Exception, "Classifier not trained yet -- aborting!" );
@@ -188,7 +188,7 @@ void GPHIKClassifier::classify ( const SparseVector * example,  int & result, Sp
   }    
 }
 
-void GPHIKClassifier::classify ( const NICE::Vector * example,  int & result, SparseVector & scores, double & uncertainty )
+void GPHIKClassifier::classify ( const NICE::Vector * example,  int & result, SparseVector & scores, double & uncertainty ) const
 {
   if (gphyper == NULL)
      fthrow(Exception, "Classifier not trained yet -- aborting!" );  
@@ -223,7 +223,7 @@ void GPHIKClassifier::classify ( const NICE::Vector * example,  int & result, Sp
 }
 
 /** training process */
-void GPHIKClassifier::train ( const std::vector< NICE::SparseVector *> & examples, const NICE::Vector & labels )
+void GPHIKClassifier::train ( const std::vector< const NICE::SparseVector *> & examples, const NICE::Vector & labels )
 {
   if (verbose)
   {
@@ -297,7 +297,7 @@ void GPHIKClassifier::train ( const std::vector< NICE::SparseVector *> & example
 }
 
 /** training process */
-void GPHIKClassifier::train ( const std::vector< SparseVector *> & examples, std::map<int, NICE::Vector> & binLabels )
+void GPHIKClassifier::train ( const std::vector< const NICE::SparseVector *> & examples, std::map<int, NICE::Vector> & binLabels )
 { 
   if (verbose)
     std::cerr << "GPHIKClassifier::train" << std::endl;
@@ -367,7 +367,7 @@ GPHIKClassifier *GPHIKClassifier::clone () const
   return NULL;
 }
   
-void GPHIKClassifier::predictUncertainty( const NICE::SparseVector * example, double & uncertainty )
+void GPHIKClassifier::predictUncertainty( const NICE::SparseVector * example, double & uncertainty ) const
 {  
   if (gphyper == NULL)
      fthrow(Exception, "Classifier not trained yet -- aborting!" );  
@@ -400,7 +400,7 @@ void GPHIKClassifier::predictUncertainty( const NICE::SparseVector * example, do
   }
 }
 
-void GPHIKClassifier::predictUncertainty( const NICE::Vector * example, double & uncertainty )
+void GPHIKClassifier::predictUncertainty( const NICE::Vector * example, double & uncertainty ) const
 {  
   if (gphyper == NULL)
      fthrow(Exception, "Classifier not trained yet -- aborting!" );  
@@ -651,19 +651,33 @@ void GPHIKClassifier::addExample( const NICE::SparseVector * example,
 			     const bool & performOptimizationAfterIncrement
 			   )
 {
-  if ( this->gphyper == NULL )
-     fthrow(Exception, "Classifier not initially trained yet -- aborting!" );     
-  //TODO add option for starting with empty classifier!
-    // -> call train() with converted input here
-  //***done*** // TODO add option to go from 2 to 3 classes!  ***done***
-  // TODO add option going from 1 to 2 classes without adding new alpha vector
+  //***done*** //TODO add option for starting with empty classifier!
+  //***done***   // -> call train() with converted input here
+  //***done***  // TODO add option to go from 2 to 3 classes!  ***done***
+  //***done*** // TODO add option going from 1 to 2 classes without adding new alpha vector
   //***done*** // TODO check variance matrices in update ***done***
   // TODO add check option for variance update
-  // TODO adapt code for addMultipleExamples
-
-  this->gphyper->addExample( example, label, performOptimizationAfterIncrement );
+  //***done*** // TODO adapt code for addMultipleExamples  
+  
+  
+  if ( this->gphyper == NULL )
+  {
+    //call train method instead
+     std::cerr << "Classifier not initially trained yet -- run initial training instead of incremental extension!"  << std::endl;
+     
+    std::vector< const NICE::SparseVector *> examplesVec;
+    examplesVec.push_back ( example );
+    
+    NICE::Vector labelsVec ( 1 , label );
+    
+    this->train ( examplesVec, labelsVec );
+  }
+  else
+  {
+    this->gphyper->addExample( example, label, performOptimizationAfterIncrement );
 
-  std::cerr << " --- GPHIKClassifierIL::addExample done --- " << std::endl;  
+    std::cerr << " --- GPHIKClassifierIL::addExample done --- " << std::endl;   
+  }
 }
 
 void GPHIKClassifier::addMultipleExamples( const std::vector< const NICE::SparseVector * > & newExamples,
@@ -676,8 +690,15 @@ void GPHIKClassifier::addMultipleExamples( const std::vector< const NICE::Sparse
     return;
 
   if ( this->gphyper == NULL )
-     fthrow(Exception, "Classifier not initially trained yet -- aborting!" );     
-  //TODO add option for starting with empty classifier!
-  
-  this->gphyper->addMultipleExamples( newExamples, newLabels, performOptimizationAfterIncrement );  
+  {
+    //call train method instead
+    
+    this->train ( newExamples, newLabels );    
+    
+    std::cerr << "train method successfully called in add multiple examples" << std::endl;
+  }
+  else
+  {
+    this->gphyper->addMultipleExamples( newExamples, newLabels, performOptimizationAfterIncrement );     
+  }
 }

GPHIKClassifier.h

@@ -92,7 +92,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param result (int) class number of most likely class
      * @param scores (SparseVector) classification scores for known classes
      */        
-    void classify ( const NICE::SparseVector * example,  int & result, NICE::SparseVector & scores );
+    void classify ( const NICE::SparseVector * example,  int & result, NICE::SparseVector & scores ) const;
     
     /** 
      * @brief classify a given example with the previously learnt model
@@ -103,7 +103,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param scores (SparseVector) classification scores for known classes
      * @param uncertainty (double*) predictive variance of the classification result, if computed
      */    
-    void classify ( const NICE::SparseVector * example,  int & result, NICE::SparseVector & scores, double & uncertainty );
+    void classify ( const NICE::SparseVector * example,  int & result, NICE::SparseVector & scores, double & uncertainty ) const;
     
     /** 
      * @brief classify a given example with the previously learnt model
@@ -114,7 +114,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param result (int) class number of most likely class
      * @param scores (SparseVector) classification scores for known classes
      */        
-    void classify ( const NICE::Vector * example,  int & result, NICE::SparseVector & scores );
+    void classify ( const NICE::Vector * example,  int & result, NICE::SparseVector & scores ) const;
     
     /** 
      * @brief classify a given example with the previously learnt model
@@ -126,7 +126,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param scores (SparseVector) classification scores for known classes
      * @param uncertainty (double) predictive variance of the classification result, if computed
      */    
-    void classify ( const NICE::Vector * example,  int & result, NICE::SparseVector & scores, double & uncertainty );    
+    void classify ( const NICE::Vector * example,  int & result, NICE::SparseVector & scores, double & uncertainty ) const;    
 
     /**
      * @brief train this classifier using a given set of examples and a given set of binary label vectors 
@@ -135,7 +135,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param examples (std::vector< NICE::SparseVector *>) training data given in a sparse representation
      * @param labels (Vector) class labels (multi-class)
      */
-    void train ( const std::vector< NICE::SparseVector *> & examples, const NICE::Vector & labels );
+    void train ( const std::vector< const NICE::SparseVector *> & examples, const NICE::Vector & labels );
     
     /** 
      * @brief train this classifier using a given set of examples and a given set of binary label vectors 
@@ -144,7 +144,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param examples examples to use given in a sparse data structure
      * @param binLabels corresponding binary labels with class no. There is no need here that every examples has only on positive entry in this set (1,-1)
      */
-    void train ( const std::vector< NICE::SparseVector *> & examples, std::map<int, NICE::Vector> & binLabels );
+    void train ( const std::vector< const NICE::SparseVector *> & examples, std::map<int, NICE::Vector> & binLabels );
     
     GPHIKClassifier *clone () const;
 
@@ -155,7 +155,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param examples example for which the classification uncertainty shall be predicted, given in a sparse representation
      * @param uncertainty contains the resulting classification uncertainty
      */       
-    void predictUncertainty( const NICE::SparseVector * example, double & uncertainty );
+    void predictUncertainty( const NICE::SparseVector * example, double & uncertainty ) const;
     
     /** 
      * @brief prediction of classification uncertainty
@@ -164,7 +164,7 @@ class GPHIKClassifier : public NICE::Persistent, public NICE::OnlineLearnable
      * @param examples example for which the classification uncertainty shall be predicted, given in a non-sparse representation
      * @param uncertainty contains the resulting classification uncertainty
      */       
-    void predictUncertainty( const NICE::Vector * example, double & uncertainty );    
+    void predictUncertainty( const NICE::Vector * example, double & uncertainty ) const;    
     
 
 

progs/classifyDatasetGPHIK.cpp

@@ -19,7 +19,7 @@
 #include "gp-hik-core/GPHIKClassifier.h"
 
 
-void readSparseExamples ( const std::string & fn,  std::vector< NICE::SparseVector * > & examples, NICE::Vector & labels )
+void readSparseExamples ( const std::string & fn,  std::vector< const NICE::SparseVector * > & examples, NICE::Vector & labels )
 {
   // initially cleaning of variables
   examples.clear();
@@ -103,7 +103,7 @@ int main (int argc, char* argv[])
   // ========================================================================  
    
   // read training data
-  std::vector< NICE::SparseVector * > examplesTrain;
+  std::vector< const NICE::SparseVector * > examplesTrain;
   NICE::Vector labelsTrain;
   
   std::string s_fn_trainingSet = conf.gS("main", "trainset");
@@ -125,7 +125,7 @@ int main (int argc, char* argv[])
   // ========================================================================
   
   // read test data
-  std::vector< NICE::SparseVector * > examplesTest;
+  std::vector< const NICE::SparseVector * > examplesTest;
   NICE::Vector labelsTest;
   
   std::string s_fn_testSet = conf.gS("main", "testset");
@@ -145,7 +145,7 @@ int main (int argc, char* argv[])
   
   NICE::Matrix confusion ( i_noClassesTest, i_noClassesTrain, 0.0 );
   
-  for (std::vector< NICE::SparseVector *>::const_iterator itTestExamples = examplesTest.begin(); itTestExamples != examplesTest.end(); itTestExamples++, idx++)
+  for (std::vector< const NICE::SparseVector *>::const_iterator itTestExamples = examplesTest.begin(); itTestExamples != examplesTest.end(); itTestExamples++, idx++)
   {
     int classno_groundtruth = labelsTest( idx );
     int classno_predicted;
@@ -171,13 +171,13 @@ int main (int argc, char* argv[])
   // ========================================================================
   
   // release memore of feature vectors from training set
-  for (std::vector< NICE::SparseVector *>::const_iterator itTrainExamples = examplesTrain.begin(); itTrainExamples != examplesTrain.end(); itTrainExamples++ )
+  for (std::vector< const NICE::SparseVector *>::const_iterator itTrainExamples = examplesTrain.begin(); itTrainExamples != examplesTrain.end(); itTrainExamples++ )
   {
     delete *itTrainExamples;
   }
   
   // release memore of feature vectors from test set
-  for (std::vector< NICE::SparseVector *>::const_iterator itTestExamples = examplesTest.begin(); itTestExamples != examplesTest.end(); itTestExamples++ )
+  for (std::vector< const NICE::SparseVector *>::const_iterator itTestExamples = examplesTest.begin(); itTestExamples != examplesTest.end(); itTestExamples++ )
   {
     delete *itTestExamples;
   }

progs/toyExample.cpp

@@ -70,10 +70,10 @@ int main (int argc, char* argv[])
   }
   
   //----------------- convert data to sparse data structures ---------
-  std::vector< NICE::SparseVector *> examplesTrain;
+  std::vector< const NICE::SparseVector *> examplesTrain;
   examplesTrain.resize( dataTrain.rows() );
   
-  std::vector< NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
+  std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
   for (int i = 0; i < (int)dataTrain.rows(); i++, exTrainIt++)
   {
     *exTrainIt =  new NICE::SparseVector( dataTrain.getRow(i) );

progs/toyExampleStoreRestore.cpp

@@ -48,10 +48,10 @@ int main (int argc, char* argv[])
   } 
   
   //----------------- convert data to sparse data structures ---------
-  std::vector< NICE::SparseVector *> examplesTrain;
+  std::vector< const NICE::SparseVector *> examplesTrain;
   examplesTrain.resize( dataTrain.rows() );
   
-  std::vector< NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
+  std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
   for (int i = 0; i < (int)dataTrain.rows(); i++, exTrainIt++)
   {
     *exTrainIt =  new NICE::SparseVector( dataTrain.getRow(i) );

tests/TestGPHIKOnlineLearnable.cpp

@@ -33,63 +33,248 @@ void TestGPHIKOnlineLearnable::setUp() {
 
 void TestGPHIKOnlineLearnable::tearDown() {
 }
-void TestGPHIKOnlineLearnable::testOnlineLearningMethods()
+
+
+
+void readData ( const std::string filename, NICE::Matrix & data, NICE::Vector & yBin, NICE::Vector & yMulti )
+{
+ std::ifstream ifs ( filename.c_str() , ios::in );
+
+  if ( ifs.good() )
+  {
+    ifs >> data;
+    ifs >> yBin;
+    ifs >> yMulti;
+    ifs.close();  
+  }
+  else 
+  {
+    std::cerr << "Unable to read data from file " << filename << " -- aborting." << std::endl;
+    CPPUNIT_ASSERT ( ifs.good() );
+  }    
+}
+
+void prepareLabelMappings (std::map<int,int> mapClNoToIdxTrain, const GPHIKClassifier * classifier, std::map<int,int> mapClNoToIdxTest, const NICE::Vector & yMultiTest)
+{
+  // determine classes known during training and corresponding mapping
+  // thereby allow for non-continous class labels
+  std::set<int> classesKnownTraining = classifier->getKnownClassNumbers();
+  
+  int noClassesKnownTraining ( classesKnownTraining.size() );
+  std::set<int>::const_iterator clTrIt = classesKnownTraining.begin();
+  for ( int i=0; i < noClassesKnownTraining; i++, clTrIt++ )
+      mapClNoToIdxTrain.insert ( std::pair<int,int> ( *clTrIt, i )  );
+  
+  // determine classes known during testing and corresponding mapping
+  // thereby allow for non-continous class labels
+  std::set<int> classesKnownTest;
+  classesKnownTest.clear();
+  
+
+  // determine which classes we have in our label vector
+  // -> MATLAB: myClasses = unique(y);
+  for ( NICE::Vector::const_iterator it = yMultiTest.begin(); it != yMultiTest.end(); it++ )
+  {
+    if ( classesKnownTest.find ( *it ) == classesKnownTest.end() )
+    {
+      classesKnownTest.insert ( *it );
+    }
+  }          
+  
+  int noClassesKnownTest ( classesKnownTest.size() );  
+  std::set<int>::const_iterator clTestIt = classesKnownTest.begin();
+  for ( int i=0; i < noClassesKnownTest; i++, clTestIt++ )
+      mapClNoToIdxTest.insert ( std::pair<int,int> ( *clTestIt, i )  );   
+}
+
+void evaluateClassifier ( NICE::Matrix & confusionMatrix, 
+                          const NICE::GPHIKClassifier * classifier, 
+                          const NICE::Matrix & data,
+                          const NICE::Vector & yMulti,
+                          const std::map<int,int> & mapClNoToIdxTrain,
+                          const std::map<int,int> & mapClNoToIdxTest
+                        ) 
 {
+  int i_loopEnd  ( (int)data.rows() );  
   
+  for (int i = 0; i < i_loopEnd ; i++)
+  {
+    NICE::Vector example ( data.getRow(i) );
+    NICE::SparseVector scores;
+    int result;    
+    
+    // classify with incrementally trained classifier 
+    classifier->classify( &example, result, scores );
+    
+    confusionMatrix( mapClNoToIdxTrain.find(result)->second, mapClNoToIdxTest.find(yMulti[i])->second ) += 1.0;
+  }
+}
+
+void TestGPHIKOnlineLearnable::testOnlineLearningStartEmpty()
+{
   if (verboseStartEnd)
-    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningMethods ===================== " << std::endl;  
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningStartEmpty ===================== " << std::endl;  
   
   NICE::Config conf;
   
   conf.sB ( "GPHIKClassifier", "eig_verbose", false);
   conf.sS ( "GPHIKClassifier", "optimization_method", "downhillsimplex");
   
-  std::string trainData = conf.gS( "main", "trainData", "toyExampleSmallScaleTrain.data" );
-  NICE::GPHIKClassifier * classifier;  
+  std::string s_trainData = conf.gS( "main", "trainData", "toyExampleSmallScaleTrain.data" );
   
   //------------- read the training data --------------
   
   NICE::Matrix dataTrain;
   NICE::Vector yBinTrain;
   NICE::Vector yMultiTrain; 
+  
+  readData ( s_trainData, dataTrain, yBinTrain, yMultiTrain );
+  
+  //----------------- convert data to sparse data structures ---------
+  std::vector< const NICE::SparseVector *> examplesTrain;
+  examplesTrain.resize( dataTrain.rows() );
+  
+  std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
+  for (int i = 0; i < (int)dataTrain.rows(); i++, exTrainIt++)
+  {
+    *exTrainIt =  new NICE::SparseVector( dataTrain.getRow(i) );
+  }
+  
+  //create classifier object
+  NICE::GPHIKClassifier * classifier;
+  classifier = new NICE::GPHIKClassifier ( &conf );  
+  bool performOptimizationAfterIncrement ( false );
 
-  std::ifstream ifsTrain ( trainData.c_str() , ios::in );
+  // add training samples, but without running training method first
+  classifier->addMultipleExamples ( examplesTrain,yMultiTrain, performOptimizationAfterIncrement );  
+  
+  // create second object trained in the standard way
+  NICE::GPHIKClassifier * classifierScratch = new NICE::GPHIKClassifier ( &conf );
+  classifierScratch->train ( examplesTrain, yMultiTrain );
+  
+    
+  // TEST both classifiers to produce equal results
+  
+  //------------- read the test data --------------
+  
+  
+  NICE::Matrix dataTest;
+  NICE::Vector yBinTest;
+  NICE::Vector yMultiTest; 
+  
+  std::string s_testData = conf.gS( "main", "testData", "toyExampleTest.data" );  
+  
+  readData ( s_testData, dataTest, yBinTest, yMultiTest );
+
+    
+  // ------------------------------------------
+  // ------------- PREPARATION --------------
+  // ------------------------------------------   
+  
+  // determine classes known during training/testing and corresponding mapping
+  // thereby allow for non-continous class labels  
+  std::map<int,int> mapClNoToIdxTrain;
+  std::map<int,int> mapClNoToIdxTest;
+  prepareLabelMappings (mapClNoToIdxTrain, classifier, mapClNoToIdxTest, yMultiTest);
+  
+  
+  NICE::Matrix confusionMatrix         ( mapClNoToIdxTrain.size(), mapClNoToIdxTest.size(), 0.0);
+  NICE::Matrix confusionMatrixScratch  ( mapClNoToIdxTrain.size(), mapClNoToIdxTest.size(), 0.0);
+  
+    
+  // ------------------------------------------
+  // ------------- CLASSIFICATION --------------
+  // ------------------------------------------  
+  evaluateClassifier ( confusionMatrix, classifier, dataTest, yMultiTest,
+                          mapClNoToIdxTrain,mapClNoToIdxTest ); 
+  
+  evaluateClassifier ( confusionMatrixScratch, classifierScratch, dataTest, yMultiTest,
+                          mapClNoToIdxTrain,mapClNoToIdxTest );  
+  
+    
+  // post-process confusion matrices
+  confusionMatrix.normalizeColumnsL1();
+  double arr ( confusionMatrix.trace()/confusionMatrix.cols() );
+
+  confusionMatrixScratch.normalizeColumnsL1();
+  double arrScratch ( confusionMatrixScratch.trace()/confusionMatrixScratch.cols() );
 
-  if ( ifsTrain.good() )
+  
+  CPPUNIT_ASSERT_DOUBLES_EQUAL( arr, arrScratch, 1e-8);
+  
+  // don't waste memory
+  
+  delete classifier;
+  delete classifierScratch;  
+  
+  for (std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin(); exTrainIt != examplesTrain.end(); exTrainIt++)
   {
-    ifsTrain >> dataTrain;
-    ifsTrain >> yBinTrain;
-    ifsTrain >> yMultiTrain;
-    ifsTrain.close();  
+    delete *exTrainIt;
   }
-  else 
-  {
-    std::cerr << "Unable to read training data from file " << trainData << " -- aborting." << std::endl;
-    CPPUNIT_ASSERT ( ifsTrain.good() );
-  } 
   
+  
+  if (verboseStartEnd)
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningStartEmpty done ===================== " << std::endl;   
+}
+
+void TestGPHIKOnlineLearnable::testOnlineLearningOCCtoBinary()
+{
+  if (verboseStartEnd)
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningOCCtoBinary ===================== " << std::endl;  
+  
+  if (verboseStartEnd)
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningOCCtoBinary done ===================== " << std::endl;   
+}
+
+void TestGPHIKOnlineLearnable::testOnlineLearningBinarytoMultiClass()
+{
+  if (verboseStartEnd)
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningBinarytoMultiClass ===================== " << std::endl;   
+  
+  if (verboseStartEnd)
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningBinarytoMultiClass done ===================== " << std::endl;   
+}
+
+void TestGPHIKOnlineLearnable::testOnlineLearningMultiClass()
+{
+  
+  if (verboseStartEnd)
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningMultiClass ===================== " << std::endl;  
+  
+  NICE::Config conf;
+  
+  conf.sB ( "GPHIKClassifier", "eig_verbose", false);
+  conf.sS ( "GPHIKClassifier", "optimization_method", "downhillsimplex");
+  
+  std::string s_trainData = conf.gS( "main", "trainData", "toyExampleSmallScaleTrain.data" );
+  
+  //------------- read the training data --------------
+  
+  NICE::Matrix dataTrain;
+  NICE::Vector yBinTrain;
+  NICE::Vector yMultiTrain; 
+  
+  readData ( s_trainData, dataTrain, yBinTrain, yMultiTrain );
+
   //----------------- convert data to sparse data structures ---------
-  std::vector< NICE::SparseVector *> examplesTrain;
+  std::vector< const NICE::SparseVector *> examplesTrain;
   examplesTrain.resize( dataTrain.rows()-1 );
   
-  std::vector< NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
+  std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
   for (int i = 0; i < (int)dataTrain.rows()-1; i++, exTrainIt++)
   {
     *exTrainIt =  new NICE::SparseVector( dataTrain.getRow(i) );
   }  
   
   // TRAIN INITIAL CLASSIFIER FROM SCRATCH
-  
+  NICE::GPHIKClassifier * classifier;
   classifier = new NICE::GPHIKClassifier ( &conf );
 
   //use all but the first example for training and add the first one lateron
   NICE::Vector yMultiRelevantTrain  ( yMultiTrain.getRangeRef( 0, yMultiTrain.size()-2  ) );
-
-  std::cerr << "yMultiRelevantTrain: " << yMultiRelevantTrain << std::endl;
   
   classifier->train ( examplesTrain , yMultiRelevantTrain );
   
-  std::cerr << "Training done -- start incremental learning " << std::endl;
   
   // RUN INCREMENTAL LEARNING
   
@@ -139,94 +324,37 @@ void TestGPHIKOnlineLearnable::testOnlineLearningMethods()
   NICE::Matrix dataTest;
   NICE::Vector yBinTest;
   NICE::Vector yMultiTest; 
-
-  std::string testData = conf.gS( "main", "testData", "toyExampleTest.data" );  
-  std::ifstream ifsTest ( testData.c_str(), ios::in );
-  if ( ifsTest.good() )
-  {
-    ifsTest >> dataTest;
-    ifsTest >> yBinTest;
-    ifsTest >> yMultiTest;
-    ifsTest.close();  
-  }
-  else 
-  {
-    std::cerr << "Unable to read test data, aborting." << std::endl;
-    CPPUNIT_ASSERT ( ifsTest.good() );
-  }
   
+  std::string s_testData = conf.gS( "main", "testData", "toyExampleTest.data" );  
+  
+  readData ( s_testData, dataTest, yBinTest, yMultiTest );
+
+    
   // ------------------------------------------
   // ------------- PREPARATION --------------
   // ------------------------------------------   
   
-  // determine classes known during training and corresponding mapping
-  // thereby allow for non-continous class labels
-  std::set<int> classesKnownTraining = classifier->getKnownClassNumbers();
-  
-  int noClassesKnownTraining ( classesKnownTraining.size() );
+  // determine classes known during training/testing and corresponding mapping
+  // thereby allow for non-continous class labels  
   std::map<int,int> mapClNoToIdxTrain;
-  std::set<int>::const_iterator clTrIt = classesKnownTraining.begin();
-  for ( int i=0; i < noClassesKnownTraining; i++, clTrIt++ )
-      mapClNoToIdxTrain.insert ( std::pair<int,int> ( *clTrIt, i )  );
-  
-  // determine classes known during testing and corresponding mapping
-  // thereby allow for non-continous class labels
-  std::set<int> classesKnownTest;
-  classesKnownTest.clear();
-  
-
-  // determine which classes we have in our label vector
-  // -> MATLAB: myClasses = unique(y);
-  for ( NICE::Vector::const_iterator it = yMultiTest.begin(); it != yMultiTest.end(); it++ )
-  {
-    if ( classesKnownTest.find ( *it ) == classesKnownTest.end() )
-    {
-      classesKnownTest.insert ( *it );
-    }
-  }          
-  
-  int noClassesKnownTest ( classesKnownTest.size() );  
   std::map<int,int> mapClNoToIdxTest;
-  std::set<int>::const_iterator clTestIt = classesKnownTest.begin();
-  for ( int i=0; i < noClassesKnownTest; i++, clTestIt++ )
-      mapClNoToIdxTest.insert ( std::pair<int,int> ( *clTestIt, i )  ); 
-          
-  
-  NICE::Matrix confusionMatrix         ( noClassesKnownTraining, noClassesKnownTest, 0.0);
-  NICE::Matrix confusionMatrixScratch    ( noClassesKnownTraining, noClassesKnownTest, 0.0);
-  
-  std::cerr << "data preparation for testing is done "<< std::endl;
+  prepareLabelMappings (mapClNoToIdxTrain, classifier, mapClNoToIdxTest, yMultiTest);
   
-  int i_loopEnd  ( (int)dataTest.rows() );
   
-  
-  for (int i = 0; i < i_loopEnd ; i++)
-  {
-    NICE::Vector example ( dataTest.getRow(i) );
-    NICE::SparseVector scores;
-    int result;
-    
-    
-    // classify with incrementally trained classifier 
-    classifier->classify( &example, result, scores );
-    std::cerr << "results with IL classifier: " << std::endl;
-    scores.store ( std::cerr );   
-    
-    confusionMatrix( mapClNoToIdxTrain.find(result)->second, mapClNoToIdxTest.find(yMultiTest[i])->second ) += 1.0;
-
-    // classify with classifier learned from scratch
-    scores.clear();
-    classifierScratch->classify( &example, result, scores );
-    std::cerr << "Results with scratch classifier: " << std::endl;
-    scores.store( std::cerr );
-    std::cerr << std::endl;
+  NICE::Matrix confusionMatrix         ( mapClNoToIdxTrain.size(), mapClNoToIdxTest.size(), 0.0);
+  NICE::Matrix confusionMatrixScratch    ( mapClNoToIdxTrain.size(), mapClNoToIdxTest.size(), 0.0);
     
-    confusionMatrixScratch( mapClNoToIdxTrain.find(result)->second, mapClNoToIdxTest.find(yMultiTest[i])->second ) += 1.0;
-  }  
+  // ------------------------------------------
+  // ------------- CLASSIFICATION --------------
+  // ------------------------------------------  
+  evaluateClassifier ( confusionMatrix, classifier, dataTest, yMultiTest,
+                          mapClNoToIdxTrain,mapClNoToIdxTest ); 
+  
+  evaluateClassifier ( confusionMatrixScratch, classifierScratch, dataTest, yMultiTest,
+                          mapClNoToIdxTrain,mapClNoToIdxTest );  
   
-  //TODO also check that both classifiers result in the same store-files
     
-  std::cerr <<  "postprocess confusion matrices " << std::endl;
+  // post-process confusion matrices
   confusionMatrix.normalizeColumnsL1();
   double arr ( confusionMatrix.trace()/confusionMatrix.cols() );
 
@@ -237,13 +365,17 @@ void TestGPHIKOnlineLearnable::testOnlineLearningMethods()
   CPPUNIT_ASSERT_DOUBLES_EQUAL( arr, arrScratch, 1e-8);
   
   // don't waste memory
-  //TODO clean up of training data, also in TestGPHIKPersistent
   
   delete classifier;
   delete classifierScratch;
   
+  for (std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin(); exTrainIt != examplesTrain.end(); exTrainIt++)
+  {
+    delete *exTrainIt;
+  } 
+  
   if (verboseStartEnd)
-    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningMethods done ===================== " << std::endl;  
+    std::cerr << "================== TestGPHIKOnlineLearnable::testOnlineLearningMultiClass done ===================== " << std::endl;  
   
 }
 

tests/TestGPHIKOnlineLearnable.h

@@ -13,7 +13,10 @@
 class TestGPHIKOnlineLearnable : public CppUnit::TestFixture {
 
     CPPUNIT_TEST_SUITE( TestGPHIKOnlineLearnable );
-	 CPPUNIT_TEST(testOnlineLearningMethods);
+      CPPUNIT_TEST(testOnlineLearningStartEmpty);
+      CPPUNIT_TEST(testOnlineLearningOCCtoBinary);
+      CPPUNIT_TEST(testOnlineLearningBinarytoMultiClass);
+      CPPUNIT_TEST(testOnlineLearningMultiClass);
       
     CPPUNIT_TEST_SUITE_END();
   
@@ -23,8 +26,13 @@ class TestGPHIKOnlineLearnable : public CppUnit::TestFixture {
     void setUp();
     void tearDown();
 
+    void testOnlineLearningStartEmpty();    
+    
+    void testOnlineLearningOCCtoBinary();
+    
+    void testOnlineLearningBinarytoMultiClass();
 
-    void testOnlineLearningMethods();
+    void testOnlineLearningMultiClass();
 };
 
 #endif // _TESTGPHIKONLINELEARNABLE_H

tests/TestGPHIKPersistent.cpp

@@ -65,10 +65,10 @@ void TestGPHIKPersistent::testPersistentMethods()
   } 
   
   //----------------- convert data to sparse data structures ---------
-  std::vector< NICE::SparseVector *> examplesTrain;
+  std::vector< const NICE::SparseVector *> examplesTrain;
   examplesTrain.resize( dataTrain.rows() );
   
-  std::vector< NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
+  std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin();
   for (int i = 0; i < (int)dataTrain.rows(); i++, exTrainIt++)
   {
     *exTrainIt =  new NICE::SparseVector( dataTrain.getRow(i) );
@@ -206,6 +206,17 @@ void TestGPHIKPersistent::testPersistentMethods()
   
   CPPUNIT_ASSERT_DOUBLES_EQUAL( arr, arrRestored, 1e-8);
   
+  // don't waste memory
+  //TODO clean up of training data, also in TestGPHIKPersistent
+  
+  delete classifier;
+  delete classifierRestored;
+  
+  for (std::vector< const NICE::SparseVector *>::iterator exTrainIt = examplesTrain.begin(); exTrainIt != examplesTrain.end(); exTrainIt++)
+  {
+    delete *exTrainIt;
+  } 
+  
   if (verboseStartEnd)
     std::cerr << "================== TestGPHIKPersistent::testPersistentMethods done ===================== " << std::endl;