NICE_GP_HIK_Exp/progs/activeLearningCheckerBoard.cpp

/**
* @file ActiveLearningCheckerBoard.cpp
* @brief Incrementally train the GP HIK classifier using the predictive variance and its approximations to select new samples, perform binary tests. We do not use the fast-hik implementations but perform the computations manually
* @author Alexander Freytag
* @date 11-06-2012
*/
#include <vector>
#include <iostream>
#include <stdlib.h>
#include <time.h>
#include <set>


#include <core/basics/Config.h>
#include <core/basics/StringTools.h>
#include <core/basics/Timer.h>

#include <core/image/ImageT.h>
#include <core/image/ColorImageT.h>
#include <core/image/CircleT.h>
#include <core/image/LineT.h>
// QT Interface for image display
// We only use the simple function showImage in this example, but there is far more
// to explore in this area.
#include <core/imagedisplay/ImageDisplay.h>

#include "core/algebra/CholeskyRobust.h"

#include "core/vector/Algorithms.h"
#include <core/vector/SparseVectorT.h>
#include <core/vector/VectorT.h>

//----------

#include "vislearning/baselib/ProgressBar.h"
#include <vislearning/baselib/Globals.h>

#include <vislearning/classifier/kernelclassifier/KCGPRegOneVsAll.h>
#include <vislearning/classifier/fpclassifier/gphik/FPCGPHIK.h>
#include "vislearning/cbaselib/MultiDataset.h"
#include <vislearning/cbaselib/LabeledSet.h>
#include "vislearning/cbaselib/ClassificationResults.h"


#include <vislearning/math/kernels/KernelData.h>

//----------

#include "gp-hik-exp/progs/datatools.h"

//----------

using namespace std;
using namespace NICE;
using namespace OBJREC;

enum QueryStrategy{
      RANDOM = 0,
      GPMEAN,
      GPPREDVAR,
      GPHEURISTIC,
      GPHEURISTICPLUS,
      GPBALANCE
    }; 
    
std::string convertInt(int number)
{
   stringstream ss;//create a stringstream
   ss << number;//add number to the stream
   return ss.str();//return a string with the contents of the stream
}

void sampleFromCheckerboard( const int & nrOfSectorsProDim, const int & sizeOfSector, const int & examplesPerSector, std::vector<NICE::Vector> & examples, NICE::Vector & labels)
{
  int nrOfSectorsTotal ( nrOfSectorsProDim * nrOfSectorsProDim );
  //set the labels
  labels.resize( nrOfSectorsTotal * examplesPerSector );
  for ( int ex = 0; ex < examplesPerSector; ex++)
  {
    for ( int i = 0; i < nrOfSectorsProDim; i++)
    {
      for ( int j = 0; j < nrOfSectorsProDim; j++)
      {      
        labels[ (i*nrOfSectorsProDim+j)*examplesPerSector + ex ] = ( i + j ) % 2;
      }
    }
  }

  for ( int i = 0; i < nrOfSectorsProDim; i++)
  {
    for ( int j = 0; j < nrOfSectorsProDim; j++)
    {  
      for ( int ex = 0; ex < examplesPerSector; ex++)
      {
        NICE::Vector example( 3 );
        double xi ( rand() % sizeOfSector  + i * sizeOfSector ) ;
        double yi ( rand() % sizeOfSector  + j * sizeOfSector );
        //compute normalized histograms
        example[0] = xi / (nrOfSectorsTotal*sizeOfSector);
        example[1] = yi / (nrOfSectorsTotal*sizeOfSector);
        example[2] = 1.0 - example[0] - example[1];
        examples.push_back( example );
      }
    }
  }
}

void paintImageBorders( NICE::ColorImage & img, const int & nrOfSectorsProDim, const int & sizeOfSector )
{
  std::cerr << "img.width(): " << img.width() << " img.height(): " << img.height() << std::endl;
  std::cerr << "nrOfSectorsProDim*sizeOfSector-1: " << nrOfSectorsProDim*sizeOfSector-1 << std::endl;
  
  NICE::Line l1 ( NICE::Coord( 0, 0 ) , NICE::Coord ( 0, nrOfSectorsProDim*sizeOfSector-1) );
  NICE::Line l2 ( NICE::Coord( 0, nrOfSectorsProDim*sizeOfSector-1 ) , NICE::Coord ( nrOfSectorsProDim*sizeOfSector-1, nrOfSectorsProDim*sizeOfSector-1) );
  NICE::Line l3 ( NICE::Coord( nrOfSectorsProDim*sizeOfSector-1, nrOfSectorsProDim*sizeOfSector-1 ) , NICE::Coord ( nrOfSectorsProDim*sizeOfSector-1, 0) );
  NICE::Line l4 ( NICE::Coord( nrOfSectorsProDim*sizeOfSector-1, 0 ) , NICE::Coord ( 0, 0 ) );
  
  l1.draw( img, Color ( 0, 0, 0 ) );
  l2.draw( img, Color ( 0, 0, 0 ) ); 
  l3.draw( img, Color ( 0, 0, 0 ) );
  l4.draw( img, Color ( 0, 0, 0 ) );   
}

void paintSectorsInImage( NICE::ColorImage & img, const int & nrOfSectorsProDim, const int & sizeOfSector )
{
  for ( int i = 1; i < nrOfSectorsProDim; i++ )
  {
    NICE::Line lHor ( NICE::Coord( 0, i*sizeOfSector ) , NICE::Coord ( nrOfSectorsProDim*sizeOfSector, i*sizeOfSector) );
    NICE::Line lVer ( NICE::Coord( i*sizeOfSector, 0 ) , NICE::Coord ( i*sizeOfSector, nrOfSectorsProDim*sizeOfSector) );
    lHor.draw( img, Color ( 0, 0, 0 ) );
    lVer.draw( img, Color ( 0, 0, 0 ) );     
  }
}

void paintLabeledExamples( NICE::ColorImage & img, const NICE::Vector & y, const Examples & examples, const int & nrOfSectorsProDim, const int & sizeOfSector, const int & diameter )
{
  int nrOfSectorsTotal ( nrOfSectorsProDim * nrOfSectorsProDim );
  for ( uint lE = 0; lE < examples.size(); lE++)
  {
//     if ( y[lE] != 1)
//     {
//       NICE::Circle circ   (  NICE::Coord( (int) ( (* examples[lE].second.svec) [0] *nrOfSectorsTotal *sizeOfSector) ,
//                                                 (int) (( (* examples[lE].second.svec) [1]) * nrOfSectorsTotal *sizeOfSector) ), diameter );
//       circ.draw ( img, Color ( 255, 0, 0 ) );
//     }
//     else
//     {      
//       NICE::Circle circ   (  NICE::Coord( (int) ( (* examples[lE].second.svec) [0] * nrOfSectorsTotal *sizeOfSector) ,
//                                                 (int) ( (* examples[lE].second.svec) [1] * nrOfSectorsTotal *sizeOfSector) ), diameter );
//       circ.draw ( img, Color ( 0, 0, 255 ) );      
//     }
    int thickness (2);
    for ( int i = 0; i < thickness; i++)
    {
      NICE::Circle circ   (  NICE::Coord( (int) ( (* examples[lE].second.svec) [0] * nrOfSectorsTotal *sizeOfSector) ,
                                  (int) ( (* examples[lE].second.svec) [1] * nrOfSectorsTotal *sizeOfSector) ), diameter-i );
      circ.draw ( img, Color ( 0, 0, 0 ) );   //old: ( 0, 255, 0 )
    }
  }  
}

void paintUnlabeledExamples( NICE::ColorImage & img, const vector< NICE::Vector > & trainDataOrig, const NICE::Vector & y, const std::vector<int> & unlabeledExamples, const int & nrOfSectorsProDim, const int & sizeOfSector, const int & diameter )
{
  int nrOfSectorsTotal ( nrOfSectorsProDim * nrOfSectorsProDim );
  for ( uint uE = 0; uE < unlabeledExamples.size(); uE++)
  {
    if ( y[ unlabeledExamples[uE] ] == 0)
    {    
      NICE::Circle circ   (  NICE::Coord( (int) (trainDataOrig[ unlabeledExamples[uE] ] [0] * nrOfSectorsTotal *sizeOfSector),
                                                (int) (trainDataOrig[ unlabeledExamples[uE] ] [1] * nrOfSectorsTotal *sizeOfSector) ) , diameter );
      circ.draw ( img, Color ( 255, 0, 0 ) );
    }
    else
    {
      NICE::Circle circ   (  NICE::Coord( (int) (trainDataOrig[ unlabeledExamples[uE] ] [0] * nrOfSectorsTotal *sizeOfSector) , 
                            (int) (trainDataOrig[ unlabeledExamples[uE] ] [1] * nrOfSectorsTotal *sizeOfSector) ) , diameter );
      circ.draw ( img, Color ( 0, 0, 255 ) );      
    }
  }  
}

void paintClassificationResult( NICE::ColorImage & img, const NICE::Vector& xstar, const int & diameter, const ClassificationResult & result, const int & nrOfSectorsProDim, const int & sizeOfSector )
{
  int nrOfSectorsTotal ( nrOfSectorsProDim * nrOfSectorsProDim );
  NICE::Circle circ   (  NICE::Coord( (int) ( xstar[0] * nrOfSectorsTotal *sizeOfSector) ,
                                                    (int) ( xstar[1] * nrOfSectorsTotal *sizeOfSector) ), diameter ); 
  if (result.classno == 1) // classified as negative
  {
    circ.draw ( img, Color ( 0, 0, 255 ) );
  }
  else // classified as positive
  {
    circ.draw ( img, Color ( 255, 0, 0 ) );
  }
}

void paintQueriedExamples( NICE::ColorImage & img, const NICE::Vector& xstar, const int & diameter, const int & nrOfSectorsProDim, const int & sizeOfSector )
{
  int nrOfSectorsTotal ( nrOfSectorsProDim * nrOfSectorsProDim );
  int thickness (2);
  for ( int i = 0; i < thickness; i++)
  {
    NICE::Circle circ   (  NICE::Coord( (int) ( xstar[0] * nrOfSectorsTotal *sizeOfSector) ,
                                            (int) ( xstar[1] * nrOfSectorsTotal *sizeOfSector) ), diameter-i );
    circ.draw ( img, Color ( 0, 0, 0 ) );   //old: ( 0, 255, 0 )
  }
}

/**
    Computes from randomly or deterministically choosen trainimages kernelmatrizes and evaluates their performance, using ROI-optimization
*/
int main ( int argc, char **argv )
{
  std::cout.precision ( 10 );
  std::cerr.precision ( 10 );

  NICE::Config conf ( argc, argv );
  int trainExPerClass = conf.gI ( "main", "trainExPerClass", 10 );
  int incrementalAddSize = conf.gI("main", "incrementalAddSize", 1);
  int nrOfIncrements = conf.gI("main", "nrOfIncrements", 9);
  
  int num_runs = conf.gI ( "main", "num_runs", 10 ); 
  bool do_classification = conf.gB ( "main", "do_classification", true );
  
  double noise = conf.gD("GPHIKClassifier", "noise", 0.01);
  double squaredNoise = pow( noise, 2);
  
  int sizeOfSector = conf.gI( "main", "sizeOfSector", 250 );
  int nrOfSectorsProDim = conf.gI( "main", "nrOfSectorsProDim", 2 );
  int examplesPerSector = conf.gI( "main", "examplesPerSector", 5 );
  int examplesPerSectorTest = conf.gI( "main", "examplesPerSectorTest", 50 );
  
  bool visualizationOfResults = conf.gB( "main", "visualizationOfResults", true );
  bool paintSectorBorders = conf.gB( "main", "paintSectorBorders" , true );
  bool saveImages = conf.gB( "main", "saveImages", false );
  std::string destinationForImages = conf.gS( "main", "destinationForImages", "" );
  
  string queryStrategyString = conf.gS( "main", "queryStrategy", "random");
  QueryStrategy queryStrategy;
  if (queryStrategyString.compare("gpMean") == 0)
  {
    queryStrategy = GPMEAN;
  }
  else if (queryStrategyString.compare("gpPredVar") == 0)
  {
    queryStrategy = GPPREDVAR;
  }
  else if (queryStrategyString.compare("gpHeuristic") == 0)
  {
    queryStrategy = GPHEURISTIC;
  }
  else if (queryStrategyString.compare("gpHeuristicPlus") == 0)
  {
    queryStrategy = GPHEURISTICPLUS;
  }  
  else if (queryStrategyString.compare("gpBalance") == 0)
  {
    queryStrategy = GPBALANCE;
  }    
  else
  {
    queryStrategy = RANDOM;
  }
 
  
  bool verbose = conf.gB ( "main", "verbose", false );

  /* initialize random seed: */
//   srand ( time ( NULL ) ); //with 0 for reproductive results
//    srand ( 0 ); //with 0 for reproductive results
  
  // ===========================  INIT =========================== 
  
  std::vector<std::vector<double> > recognitions_rates(nrOfIncrements+1);
  std::vector<std::vector<double> > AUC_scores(nrOfIncrements+1);
  std::vector<std::vector<float> > classification_times(nrOfIncrements+1);
  std::vector<std::vector<float> > IL_training_times(nrOfIncrements);
  
  for ( int run = 0; run < num_runs; run++ )
  {
    std::cerr << "run: " << run << std::endl;    
    srand ( run * 100000 ); //with 0 for reproductive results
    
    // read training set
    vector< NICE::Vector > trainDataOrig;
    Vector y;
    sampleFromCheckerboard( nrOfSectorsProDim, sizeOfSector, examplesPerSector, trainDataOrig, y );  
    
    // ------------------ TESTING
    std::vector<NICE::Vector> testData;
    Vector yTest;
    sampleFromCheckerboard( nrOfSectorsProDim, sizeOfSector, examplesPerSectorTest, testData, yTest );     

    if ( verbose )
    {
      for (uint i = 0; i < trainDataOrig.size(); i++ )
      {
        std::cerr << i << " : " << trainDataOrig[i] << std::endl;
      }    
            
      std::cerr << "resulting binary label vector:" << y << std::endl;
    }
    
    std::set<int> classesAvailable;
    classesAvailable.insert( 0 ); //we have a single negative class
    classesAvailable.insert( 1 ); //and we have a single positive class
    
    std::map<int,int> nrExamplesPerClassInDataset; //simply count how many examples for every class are available
    std::map<int,std::vector<int> > examplesPerClassInDataset;  //as well as their corresponding indices in the dataset
    
    //initialize this storage
    for (std::set<int>::const_iterator it = classesAvailable.begin(); it != classesAvailable.end(); it++)
    {
      nrExamplesPerClassInDataset.insert(std::pair<int,int>(*it,0));
      examplesPerClassInDataset.insert(std::pair<int,std::vector<int> >(*it,std::vector<int>(0)));
    }

    //store the indices of the examples
    for ( uint i = 0; i < y.size(); i++ )
    {
      (examplesPerClassInDataset.find( y[i] )->second).push_back(i);
    }
    
    //and count how many examples are in every class
    for (std::map<int,std::vector<int> >::const_iterator it = examplesPerClassInDataset.begin(); it != examplesPerClassInDataset.end(); it++)
    {
      nrExamplesPerClassInDataset.find(it->first)->second = it->second.size();
    }
    
    //simple output to tell how many examples we have for every class
    for ( std::map<int,int>::const_iterator it =  nrExamplesPerClassInDataset.begin(); it != nrExamplesPerClassInDataset.end(); it++)
    {
      cerr << it->first << ": " << it->second << endl;
    }    
      
    Examples examples;   
    
    //count how many examples of every class we have while actively selecting new examples
    //NOTE works only if we have subsequent class numbers
    NICE::Vector pickedExamplesPerClass( classesAvailable.size(), trainExPerClass);
    
    std::map<int,std::vector<int> > examplesPerClassInDatasetTmp (examplesPerClassInDataset);
    
    //chose examples for every class used for training
    //we will always use the first examples from each class, since the dataset comes already randomly ordered
    for (std::set<int>::const_iterator clIt = classesAvailable.begin(); clIt != classesAvailable.end(); clIt++)
    {
      std::map<int,std::vector<int> >::iterator exIt = examplesPerClassInDatasetTmp.find(*clIt);
      if ( verbose )
        std::cerr << "pick training examples for class " << *clIt << std::endl;
      
      for (int i = 0; i < trainExPerClass; i++)
      {
        if ( verbose )        
          std::cerr << "i: " << i << std::endl;
        int exampleIndex ( rand() % ( exIt->second.size() ) );
        if ( verbose )        
          std::cerr << "pick example " << exIt->second[exampleIndex] << " - " << y[exIt->second[exampleIndex] ]  << std::endl;
        
        Example example;
        NICE::Vector & xTrain = trainDataOrig[exIt->second[exampleIndex]];
        example.svec = new SparseVector(xTrain);
        //let's take this example and its corresponding label (which should be *clIt)
        examples.push_back ( pair<int, Example> ( y[exIt->second[exampleIndex] ], example ) ); 
        //
        exIt->second.erase(exIt->second.begin()+exampleIndex);
      }
    }    
    
    for (uint i = 0; i < examples.size(); i++ )
    {
      std::cerr << i << " : ";
      examples[i].second.svec->store(std::cerr);
    }
    
    
    //which examples are left to be actively chosen lateron?
    std::vector<int> unlabeledExamples( y.size() - trainExPerClass*classesAvailable.size() );
    int exCnt( 0 );
    for (std::set<int>::const_iterator clIt = classesAvailable.begin(); clIt != classesAvailable.end(); clIt++ )
    {
      std::map<int,std::vector<int> >::iterator exIt = examplesPerClassInDatasetTmp.find(*clIt);
      //list all examples of this specific class
      for (std::vector<int>::const_iterator it = exIt->second.begin(); it != exIt->second.end(); it++)
      {
        unlabeledExamples[exCnt] = *it;
        exCnt++;     
      }
    }
    
    //Fast-HIK
    FPCGPHIK * classifier  = new FPCGPHIK( &conf );
      
    time_t  prep_start_time = clock();
    FeaturePool fp; // will be ignored
    classifier->train ( fp, examples );
    float time_preparation = ( float ) ( clock() - prep_start_time ) ;
    std::cerr << "Time for training: " << time_preparation / CLOCKS_PER_SEC << std::endl;      
    
    //this is only needed for the visualization
    NICE::Vector yBinGP ( examples.size(), -1 );   

    for ( uint i = 0; i < examples.size(); i++ )
    {
      if ( examples[i].first == 1)
        yBinGP[i] = 1;
    }  
    std::cerr << "yBinGP: " << yBinGP << std::endl;
    
    int nrOfClassesUsed = classesAvailable.size();

    if ( visualizationOfResults )
    {
      NICE::ColorImage img ( nrOfSectorsProDim*sizeOfSector, nrOfSectorsProDim*sizeOfSector );
      img.set( 255, 255, 255 );
      
      if ( paintSectorBorders )
        paintSectorsInImage( img, nrOfSectorsProDim, sizeOfSector );
      paintImageBorders( img, nrOfSectorsProDim, sizeOfSector );
      //paint the example that we can query
      paintLabeledExamples( img, yBinGP, examples, nrOfSectorsProDim, sizeOfSector, 10 );     
      //and those that we already know
      paintUnlabeledExamples( img, trainDataOrig, y,  unlabeledExamples, nrOfSectorsProDim, sizeOfSector, 2 );
      
      if ( saveImages )
      {
        img.writePPM ( destinationForImages + "imgAL_run"+convertInt(run)+"_000_initialBoard.ppm" );
      }
      else
        showImage(img, "Initial Checkerboard");
    }
    
/*    // ------------------ TESTING
    std::vector<NICE::Vector> testData;
    Vector yTest;
    sampleFromCheckerboard( nrOfSectorsProDim, sizeOfSector, examplesPerSectorTest, testData, yTest ); */   
        
    NICE::Matrix confusionMatrix ( 2, 2 );
    confusionMatrix.set ( 0.0 );      
    
    time_t  start_time = clock();

    std::vector<int> chosen_examples_per_class ( nrOfClassesUsed );
    
    std::cerr << "Current statistic about picked examples per class: " << pickedExamplesPerClass << std::endl;

    if ( do_classification  )
    {
      NICE::ColorImage imgTest;
      if ( visualizationOfResults )
      {
        imgTest.resize ( nrOfSectorsProDim*sizeOfSector, nrOfSectorsProDim*sizeOfSector );
        imgTest.set( 255, 255, 255 );
        
        if ( paintSectorBorders )
          paintSectorsInImage( imgTest, nrOfSectorsProDim, sizeOfSector );  
        paintImageBorders( imgTest, nrOfSectorsProDim, sizeOfSector );  
        //again paint our labeled training images used so far
        paintLabeledExamples( imgTest, yBinGP, examples, nrOfSectorsProDim, sizeOfSector, 10 ); 
      }
      
      ClassificationResults results;
      ClassificationResult result;
      for ( uint i = 0 ; i < testData.size(); i++ )
      {
        const Vector & xstar = testData[i];
        SparseVector xstar_sparse ( xstar );
        Example example;
        example.svec = &xstar_sparse;        
        result = classifier->classify( example );
        
        if ( visualizationOfResults )
          paintClassificationResult( imgTest, xstar, 2, result, nrOfSectorsProDim, sizeOfSector );
        
        result.classno_groundtruth = ( yTest[i] == 1 ) ? 1 : 0;
        confusionMatrix ( result.classno_groundtruth , result.classno ) ++;
        results.push_back( result );        
      }
      
      if ( visualizationOfResults )
      {
        if ( saveImages )
        {
          imgTest.writePPM ( destinationForImages + "imgAL_run"+convertInt(run)+"_incStep_"+convertInt(0)+"ClassifResult.ppm" );
        }       
        else
          showImage(imgTest, "Classification Result");
      }

      float time_classification = ( float ) ( clock() - start_time ) ;
      if ( verbose )
        cerr << "Time for Classification with " << nrOfClassesUsed*trainExPerClass << " training-examples: " << time_classification / CLOCKS_PER_SEC << " [s]" << endl;
      ( classification_times[0] ).push_back ( time_classification / CLOCKS_PER_SEC );
      
      confusionMatrix.normalizeRowsL1();
      std::cerr << confusionMatrix;
      double avg_recognition_rate = 0.0;
      for ( int i = 0 ; i < ( int ) confusionMatrix.rows(); i++ )
      {
        avg_recognition_rate += confusionMatrix ( i, i );
      }        
      avg_recognition_rate /= confusionMatrix.rows();
      std::cerr << " run: " << run << " avg recognition rate: " <<  avg_recognition_rate*100 << " % -- " << examples.size() << " training examples used" << std::endl;

      recognitions_rates[0].push_back ( avg_recognition_rate*100 );        
      std::cerr << "number of classified examples: " << results.size() << std::endl;

      std::cerr << "perform auc evaluation "<< std::endl;
      double aucScore = results.getBinaryClassPerformance( ClassificationResults::PERF_AUC );
      
      std::cerr << " run: " << run << " AUC-score: " <<  aucScore << " % -- " << examples.size() << " training examples used" << std::endl << std::endl;

      AUC_scores[0].push_back ( aucScore*100 );
    }

    //Now start the Incremental-Learning-Part
    
    for (int incrementationStep = 0; incrementationStep < nrOfIncrements; incrementationStep++)
    {
      //simply count how many possible example we have 
      int nrOfPossibleExamples(  unlabeledExamples.size() );
      
      //chose examples for every class used for training
      Examples newExamples;      
      
      NICE::ColorImage imgAL;
      
      if ( visualizationOfResults )
      {     
        imgAL.resize ( nrOfSectorsProDim*sizeOfSector, nrOfSectorsProDim*sizeOfSector );
        imgAL.set( 255, 255, 255 );
  
        if ( paintSectorBorders )
          paintSectorsInImage( imgAL, nrOfSectorsProDim, sizeOfSector ); 
        paintImageBorders( imgAL, nrOfSectorsProDim, sizeOfSector );
        //again paint our labeled training images used so far
        paintLabeledExamples( imgAL, yBinGP, examples, nrOfSectorsProDim, sizeOfSector, 10 );
        //and paint the unlabeled examples that are available to query
        paintUnlabeledExamples( imgAL, trainDataOrig, y,  unlabeledExamples, nrOfSectorsProDim, sizeOfSector, 2 );
      }
            
      if (queryStrategy == RANDOM)
      {
        if ( verbose ) 
          std::cerr << "print chosen examples: " << std::endl;           
        for (int i = 0; i < incrementalAddSize; i++)
        {        
          int exampleIndex ( rand() % ( unlabeledExamples.size() ) );
          
          Example newExample;
          NICE::Vector & xTrain = trainDataOrig[ unlabeledExamples[exampleIndex] ];
          newExample.svec = new SparseVector( xTrain ); 
          int label( y[ unlabeledExamples[exampleIndex] ] );
          //store this example for the visualization
          examples.push_back ( pair<int, Example> ( label, newExample ) );
          //and store it to add it to the classifier
          newExamples.push_back ( pair<int, Example> ( label, newExample ) );
          unlabeledExamples.erase( unlabeledExamples.begin()+exampleIndex );
          if ( verbose ) 
            std::cerr << exampleIndex+1 << " / " << incrementalAddSize << std::endl;          
           
          pickedExamplesPerClass[label]++;
          yBinGP.append(label);
          
          if ( visualizationOfResults )
            paintQueriedExamples( imgAL, xTrain, 10, nrOfSectorsProDim, sizeOfSector );
        }
      }// end computation for RANDOM
      else if ( (queryStrategy == GPMEAN) || (queryStrategy == GPPREDVAR) || (queryStrategy == GPHEURISTIC) || (queryStrategy == GPHEURISTICPLUS) || GPBALANCE)
      {
        //compute uncertainty values for all examples according to the query strategy
        std::vector<std::pair<int,double> > scores;
        scores.clear();
        time_t  unc_pred_start_time = clock();
        for (uint exIndex = 0; exIndex < unlabeledExamples.size(); exIndex++)
        {
            NICE::Vector & xTrain = trainDataOrig[ unlabeledExamples[exIndex] ];
            SparseVector xTrainSparse ( xTrain );
            Example example;
            example.svec = &xTrainSparse;        

            if (queryStrategy == GPMEAN)
            {              
              //compute the resulting score
              ClassificationResult r = classifier->classify( example );
              //we only have two classes with "inverse" outputs
              scores.push_back( std::pair<int,double> ( exIndex, fabs(r.scores[0]) ) );
            }
            else if (queryStrategy == GPPREDVAR)
            {
              double uncertainty;
              //use the pred variance computation specified in the config file
              classifier->predictUncertainty( example, uncertainty );
              //take the maximum of the scores for the predictive variance
              scores.push_back( std::pair<int,double> ( exIndex, uncertainty) );
            }
            else if (queryStrategy == GPHEURISTIC)
            {
              double uncertainty;
              //use the pred variance computation specified in the config file
              classifier->predictUncertainty( example, uncertainty );
              //compute the mean values for every class
              ClassificationResult r = classifier->classify( example );
              //take the minimum of the scores for the heuristic measure
              scores.push_back( std::pair<int,double> ( exIndex, fabs(r.scores[0]) / sqrt( squaredNoise + uncertainty )) );             
            }
            else if (queryStrategy == GPHEURISTICPLUS)
            {
              double uncertainty;
              //use the pred variance computation specified in the config file
              classifier->predictUncertainty( example, uncertainty );
              //compute the mean values for every class
              ClassificationResult r = classifier->classify( example );
              //take the minimum of the scores for the heuristic measure
              scores.push_back( std::pair<int,double> ( exIndex, fabs(r.scores[0]) + sqrt( squaredNoise + uncertainty )) );             
            }
            else if (queryStrategy == GPBALANCE)
            {
              double uncertainty;
              //use the pred variance computation specified in the config file
              classifier->predictUncertainty( example, uncertainty );
              //compute the mean values for every class
              ClassificationResult r = classifier->classify( example );
              double scorePositive (fabs (r.scores[0] - 1.0 ));
              double scoreNegative (fabs (r.scores[0] + 1.0 ));
              double score = scorePositive < scoreNegative ? scorePositive : scoreNegative;
              //take the minimum of the scores for the heuristic measure
              scores.push_back( std::pair<int,double> ( exIndex, score / ( squaredNoise + uncertainty )) );             
            }            
        }
        float time_score_computation = ( float ) ( clock() - unc_pred_start_time ) ;
          
        //pick the ones with best score
        //we could speed this up using a more sophisticated search method
        
        if ( (queryStrategy == GPPREDVAR) || (queryStrategy == GPHEURISTICPLUS) )//take the maximum of the scores for the predictive variance or the new weight
        {
          std::set<int> chosenExamplesForThisRun;
          chosenExamplesForThisRun.clear();          
          for (int i = 0; i < incrementalAddSize; i++)
          {
            std::vector<std::pair<int,double> >::iterator bestExample = scores.begin();
            std::vector<std::pair<int,double> >::iterator worstExample = scores.begin();
            
            for (std::vector<std::pair<int,double> >::iterator jIt = scores.begin(); jIt !=scores.end(); jIt++)
            {
              if (jIt->second > bestExample->second)
                bestExample = jIt;
              if (jIt->second < worstExample->second)
                worstExample = jIt;                
            }
            if ( verbose ) 
              std::cerr << "i: " << i << " bestExample: " << bestExample->second << " worstExample: " << worstExample->second << std::endl;
            
            Example newExample;    
            NICE::Vector & xTrain = trainDataOrig[ unlabeledExamples[bestExample->first] ]; 
            newExample.svec = new SparseVector( xTrain ); 
            //actually this is the ACTIVE LEARNING step (query a label)
            int label( y[ unlabeledExamples[bestExample->first] ] );
            //store this example for the visualization
            examples.push_back ( pair<int, Example> ( label, newExample ) );    
            //and store it to add it to the classifier
            newExamples.push_back ( pair<int, Example> ( label, newExample ) );    
            //remember the index, to safely remove this example afterwards from unlabeledExamples
            chosenExamplesForThisRun.insert(bestExample->first);
            scores.erase(bestExample);
            pickedExamplesPerClass[label]++;
            yBinGP.append(label);
            
            if ( visualizationOfResults )
              paintQueriedExamples( imgAL, xTrain, 10, nrOfSectorsProDim, sizeOfSector );
          }
          
          //delete the queried examples from the set of unlabeled ones
          //do this in an decreasing order in terms of indices to ensure valid access
          for (std::set<int>::const_reverse_iterator it = chosenExamplesForThisRun.rbegin(); it != chosenExamplesForThisRun.rend(); it++)
          {
            unlabeledExamples.erase( unlabeledExamples.begin()+(*it) );             
          }          
        }
        else //take the minimum of the scores for the heuristic, heuristicPlus and the gp mean (minimum margin)
        {
          std::set<int> chosenExamplesForThisRun;
          chosenExamplesForThisRun.clear();
          for (int i = 0; i < incrementalAddSize; i++)
          {
            std::vector<std::pair<int,double> >::iterator bestExample = scores.begin();
            std::vector<std::pair<int,double> >::iterator worstExample = scores.begin();
            
            for (std::vector<std::pair<int,double> >::iterator jIt = scores.begin(); jIt !=scores.end(); jIt++)
            {
              if (jIt->second < bestExample->second)
                bestExample = jIt;
              if (jIt->second > worstExample->second)
                worstExample = jIt;               
            }
            if ( verbose )
              std::cerr << "i: " << i << " bestExample: " << bestExample->second << " worstExample: " << worstExample->second << std::endl;
            Example newExample;    
            NICE::Vector & xTrain = trainDataOrig[ unlabeledExamples[bestExample->first] ];
            newExample.svec = new SparseVector( xTrain ); 
            //actually this is the ACTIVE LEARNING step (query a label)
            int label( y[ unlabeledExamples[bestExample->first] ] );
            //store this example for the visualization
            examples.push_back ( pair<int, Example> ( label, newExample ) );
            //and store it to add it to the classifier
            newExamples.push_back ( pair<int, Example> ( label, newExample ) );
            //remember the index, to safely remove this example afterwards from unlabeledExamples
            chosenExamplesForThisRun.insert(bestExample->first);
            scores.erase(bestExample);
            pickedExamplesPerClass[label]++;
            yBinGP.append(label);
            
            if ( visualizationOfResults )
              paintQueriedExamples( imgAL, xTrain, 10, nrOfSectorsProDim, sizeOfSector );
          }  
                    
          //delete the queried example from the set of unlabeled ones
          //do this in an decreasing order in terms of indices to ensure valid access
          for (std::set<int>::const_reverse_iterator it = chosenExamplesForThisRun.rbegin(); it != chosenExamplesForThisRun.rend(); it++)
          {
            unlabeledExamples.erase( unlabeledExamples.begin()+(*it) );             
          }

        }
      
        std::cerr << "Time used to compute query-scores for " <<  nrOfPossibleExamples << " examples: " << time_score_computation / CLOCKS_PER_SEC << " [s]" << std::endl;
      } // end computation for GPMEAN, GPPREDVAR, GPHEURISTIC, GPHEURISTICPLUS
      
      if ( visualizationOfResults )
      {      
        if ( saveImages )
        {
          imgAL.writePPM ( destinationForImages + "imgAL_run"+convertInt(run)+"_incStep_"+convertInt(incrementationStep+1)+"_queries.ppm" );
        }
        else
          showImage(imgAL, "Old and new queried example");
      }
      
      std::cerr << "Current statistic about picked examples per class: " << pickedExamplesPerClass << std::endl;

      //incremental learning
      classifier->addMultipleExamples( newExamples );      
      
      //do the classification for evaluating the benefit of new examples
      if ( do_classification )
      {
        
        NICE::ColorImage imgTest;
        if ( visualizationOfResults )
        {
          imgTest.resize( nrOfSectorsProDim*sizeOfSector, nrOfSectorsProDim*sizeOfSector );
          imgTest.set( 255, 255, 255 );
          
          if ( paintSectorBorders )
            paintSectorsInImage( imgTest, nrOfSectorsProDim, sizeOfSector ); 
          paintImageBorders( imgTest, nrOfSectorsProDim, sizeOfSector ); 
          //again paint our labeled training images used so far
          paintLabeledExamples( imgTest, yBinGP, examples, nrOfSectorsProDim, sizeOfSector, 10 ); 
        }
        
        time_t  start_time = clock();
        ClassificationResults results;
        confusionMatrix.set( 0.0 );
        ClassificationResult result;
        for ( uint i = 0 ; i < testData.size(); i++ )
        {
          const Vector & xstar = testData[i];
          SparseVector xstar_sparse ( xstar );
          
          Example example;
          example.svec = &xstar_sparse;        
          result = classifier->classify( example );          
          
          if ( visualizationOfResults )
            paintClassificationResult( imgTest, xstar, 2, result, nrOfSectorsProDim, sizeOfSector );
                   
          result.classno_groundtruth = ( yTest[i] == 1 ) ? 1 : 0; 
          results.push_back( result );      
          confusionMatrix ( result.classno_groundtruth , result.classno ) ++;            
        }     

        float time_classification = ( float ) ( clock() - start_time ) ;
        if ( verbose )
          std::cerr << "Time for Classification with " << nrOfClassesUsed*trainExPerClass+incrementalAddSize*(incrementationStep+1) << " training-examples: " << time_classification / CLOCKS_PER_SEC << " [s]" << std::endl;
        ( classification_times[incrementationStep+1] ).push_back ( time_classification / CLOCKS_PER_SEC );
        
        confusionMatrix.normalizeRowsL1();
        std::cerr << confusionMatrix;
        double avg_recognition_rate ( 0.0 );
        for ( int i = 0 ; i < ( int ) confusionMatrix.rows(); i++ )
        {
          avg_recognition_rate += confusionMatrix ( i, i );
        }
        avg_recognition_rate /= confusionMatrix.rows();          
        
        std::cerr << " run: " << run << " avg recognition rate: " <<  avg_recognition_rate*100 << " % -- " << nrOfClassesUsed*trainExPerClass+incrementalAddSize*(incrementationStep+1) << " training examples used" << std::endl;

        recognitions_rates[incrementationStep+1].push_back ( avg_recognition_rate*100 );           

        
        double score = results.getBinaryClassPerformance( ClassificationResults::PERF_AUC );
        std::cerr << " run: " << run << " AUC-score: " <<  score << " % -- " << nrOfClassesUsed*trainExPerClass+incrementalAddSize*(incrementationStep+1) << " training examples used" << std::endl << std::endl;          

        AUC_scores[incrementationStep+1].push_back ( score*100 );
        
        if ( visualizationOfResults )
        {        
          if ( saveImages )
          {
            imgTest.writePPM ( destinationForImages + "imgAL_run"+convertInt(run)+"_incStep_"+convertInt(incrementationStep+1)+"ClassifResult.ppm" );
          } 
          else
            showImage(imgTest, "Classification result after inc step " + convertInt(incrementationStep+1));  
        }
      } //classification after IL adding */
    } //IL adding of different classes
    std::cerr << "Final statistic about picked examples per class: " << pickedExamplesPerClass << std::endl;
    
    //don't waste memory!
    for ( uint tmp = 0; tmp < examples.size(); tmp++ )
    {
      delete examples[tmp].second.svec;
      examples[tmp].second.svec = NULL;
    }
  }//runs 
      
  // ================= EVALUATION =========================
  
  int nrOfClassesUsed ( 2 ); //binary setting

  if ( do_classification )
  {
    std::cerr << "========================" << std::endl;
    std::cerr << "recognition_rates" << std::endl;
    for ( std::vector<std::vector<double> >::const_iterator it = recognitions_rates.begin(); it != recognitions_rates.end(); it++ )
    {
      for ( std::vector<double> ::const_iterator jt = ( *it ).begin(); jt != ( *it ).end(); jt++ )
      {
        std::cerr << *jt << " ";
      }
      std::cerr << std::endl;
    }

    std::vector<double> mean_recs;
    std::vector<double> std_dev_recs;
    for (std::vector<std::vector<double> >::const_iterator it = recognitions_rates.begin(); it != recognitions_rates.end(); it++ )
    {
      double mean_rec ( 0.0 );
      for ( std::vector<double>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        mean_rec += *itRun;
      }
      mean_rec /= it->size();
      mean_recs.push_back ( mean_rec );

      double std_dev_rec ( 0.0 );
      for ( std::vector<double>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        std_dev_rec += pow ( *itRun - mean_rec, 2 );
      }
      std_dev_rec /= it->size();
      std_dev_rec = sqrt ( std_dev_rec );
      std_dev_recs.push_back ( std_dev_rec );
    }

    int datasize ( nrOfClassesUsed*trainExPerClass );
    for ( uint i = 0; i < recognitions_rates.size(); i++)
    {
      std::cerr << "size: " << datasize << " meanRR: " << mean_recs[i] << " stdDevRR: " << std_dev_recs[i] << std::endl;
      datasize += incrementalAddSize ;
    }
    
    std::cerr << "========================" << std::endl;
    std::cerr << "AUC_scores" << std::endl;
    for ( std::vector<std::vector<double> >::const_iterator it = AUC_scores.begin(); it != AUC_scores.end(); it++ )
    {
      for ( std::vector<double> ::const_iterator jt = ( *it ).begin(); jt != ( *it ).end(); jt++ )
      {
        std::cerr << *jt << " ";
      }
      std::cerr << std::endl;
    }

    std::vector<double> mean_aucs;
    std::vector<double> std_dev_aucs;
    for (std::vector<std::vector<double> >::const_iterator it = AUC_scores.begin(); it != AUC_scores.end(); it++ )
    {
      double mean_auc ( 0.0 );
      for ( std::vector<double>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        mean_auc += *itRun;
      }
      mean_auc /= it->size();
      mean_aucs.push_back ( mean_auc );

      double std_dev_auc ( 0.0 );
      for ( std::vector<double>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        std_dev_auc += pow ( *itRun - mean_auc, 2 );
      }
      std_dev_auc /= it->size();
      std_dev_auc = sqrt ( std_dev_auc );
      std_dev_aucs.push_back ( std_dev_auc );
    }

    datasize  = nrOfClassesUsed*trainExPerClass;
    for ( uint i = 0; i < recognitions_rates.size(); i++)
    {
      std::cerr << "size: " << datasize << " meanAUC: " << mean_aucs[i] << " stdDevAUC: " << std_dev_aucs[i] << std::endl;
      datasize += incrementalAddSize ;
    }      
  }
  else
  {
    std::cerr << "========================" << std::endl;
    std::cerr << "No classification done therefor no classification times available." << std::endl;
  } 

  return 0;
}