NICE_GP_HIK_Exp/progs/IL_NewExamples.cpp

/**
* @file IL_NewExamples.cpp
* @brief Large GP-IL-Testsetup
* @author Alexander Freytag
* @date 09-05-2012
*/
#include <vector>
#include <stdlib.h>
#include <time.h>
#include <set>


#include <core/basics/Config.h>
#include <core/basics/StringTools.h>

#include <core/vector/SparseVectorT.h>
#include <core/vector/VectorT.h>

//----------

#include "vislearning/baselib/ProgressBar.h"

#include <vislearning/classifier/kernelclassifier/KCGPRegOneVsAll.h>
#include "vislearning/cbaselib/MultiDataset.h"
#include <vislearning/cbaselib/LabeledSet.h>
#include "vislearning/cbaselib/ClassificationResults.h"
#include <vislearning/baselib/Globals.h>

#include <vislearning/math/kernels/KernelData.h>

//----------

#include "gp-hik-exp/progs/datatools.h"
#include "gp-hik-exp/GPHIKClassifierNICE.h"

//----------

// #include <incrementallearning/IL_Framework_Generic.h>


//
using namespace std;
using namespace NICE;
using namespace OBJREC;

enum verbose_level {NONE = 0, LOW = 1,  MEDIUM = 2, EVERYTHING = 3};

/**
    Computes from randomly or deterministically choosen trainimages kernelmatrizes and evaluates their performance, using ROI-optimization
*/
int main ( int argc, char **argv )
{
  std::cout.precision ( 5 );
  std::cerr.precision ( 5 );

  NICE::Config conf ( argc, argv );
  int trainExPerClass = conf.gI ( "GP_IL", "trainExPerClass", 10 );
  int incrementalAddSize = conf.gI("GP_IL", "incrementalAddSize", 1);
  int nrOfIncrements = conf.gI("GP_IL", "nrOfIncrements", 9);
  int num_runs = conf.gI ( "GP_IL", "num_runs", 10 );  
  bool do_classification = conf.gB ( "GP_IL", "do_classification", true );
  bool incrementalNotBatch = conf.gB( "GP_IL", "incrementalNotBatch", true );
  
  string featureLocation = conf.gS( "GP_IL", "featureLocation", "toyExampleLargeLargeScale.data");
  
  int verbose_int = conf.gI ( "GP_IL", "verbose", 0 );
  verbose_level verbose ( NONE );
  switch ( verbose_int )
  {
    case 0:
      verbose = NONE;
      break;
    case 1:
      verbose = LOW;
      break;
    case 2:
      verbose = MEDIUM;
      break;
    case 3:
      verbose = EVERYTHING;
      break;
  }



  /* initialize random seed: */
  srand ( time ( NULL ) ); //with 0 for reproductive results
//    srand ( 0 ); //with 0 for reproductive results

  // ===========================  INIT =========================== 
  
  //these classes are the basic knowledge we have at the beginning
  set<int> classesForTraining;
  classesForTraining.insert(0);
  classesForTraining.insert(1);
  classesForTraining.insert(2);
  classesForTraining.insert(3);
  classesForTraining.insert(4);
  classesForTraining.insert(5);
  classesForTraining.insert(6);
  classesForTraining.insert(7);
  classesForTraining.insert(8);
  classesForTraining.insert(9);
  classesForTraining.insert(10);
  classesForTraining.insert(11);
  classesForTraining.insert(12);
  classesForTraining.insert(13);
  classesForTraining.insert(14);
  
//   //these classes will be added iteratively to our training set
//   std::set<int> classesForIncrementalTraining;
  
  std::vector<std::vector<double> > recognitions_rates(nrOfIncrements+1);
  std::vector<std::vector<float> > classification_times(nrOfIncrements+1);
  std::vector<std::vector<float> > IL_training_times(nrOfIncrements);
  
  for ( int run = 0; run < num_runs; run++ )
  {
    std::cerr << "run: " << run << std::endl;    
    
    //15-scenes settings
    std::string ext = conf.gS("main", "ext", ".txt"); 
    std::cerr << "Using cache extension: " << ext << std::endl;

    OBJREC::MultiDataset md ( &conf );
    const ClassNames & classNamesTrain = md.getClassNames("train");
    
    // read training set
    vector< NICE::Vector > trainDataOrig;
    Vector y;
    const LabeledSet *train = md["train"];

    readData< std::vector< NICE::Vector >, NICE::Vector >  ( conf, *train, trainDataOrig, y, ext );

    std::vector<double> labelsStd;
    int datasize_all ( trainDataOrig.size() );
    
    std::set<int> classesAvailable;
    for ( uint i = 0; i < y.size(); i++)
    {
      //automatically check for duplicates
      classesAvailable.insert(y[i]);
    }
    
    int numberOfClasses =  classesAvailable.size();
    
    std::map<int,int> nrExamplesPerClassInDataset;
    std::map<int,std::vector<int> > examplesPerClassInDataset;
    
    for (std::set<int>::const_iterator it = classesAvailable.begin(); it != classesAvailable.end(); it++)
    {
      nrExamplesPerClassInDataset.insert(std::pair<int,int>(*it,0));
      examplesPerClassInDataset.insert(std::pair<int,std::vector<int> >(*it,std::vector<int>(0)));
    }
    
    for ( uint i = 0; i < y.size(); i++ )
    {
      (examplesPerClassInDataset.find(y[i])->second).push_back(i);
    }
    
    for (std::map<int,std::vector<int> >::const_iterator it = examplesPerClassInDataset.begin(); it != examplesPerClassInDataset.end(); it++)
    {
      nrExamplesPerClassInDataset.find(it->first)->second = it->second.size();
    }
    
    for ( std::map<int,int>::const_iterator it =  nrExamplesPerClassInDataset.begin(); it != nrExamplesPerClassInDataset.end(); it++)
    {
      cerr << it->first << ": " << it->second << endl;
    }    
       
    Examples examples;
    
    
    std::map<int,std::vector<int> > examplesPerClassInDatasetTmp (examplesPerClassInDataset);
    //chose examples for every class used for training
    for (std::set<int>::const_iterator clIt = classesForTraining.begin(); clIt != classesForTraining.end(); clIt++)
    {
      std::map<int,std::vector<int> >::iterator exIt = examplesPerClassInDatasetTmp.find(*clIt);
      std::cerr << "pick training examples for class " << *clIt << std::endl;
      
      for (int i = 0; i < trainExPerClass; i++)
      {
        std::cerr << "i: " << i << std::endl;
         int exampleIndex ( rand() % ( exIt->second.size() ) );
         std::cerr << "exampleIndex: " << exampleIndex << std::endl;
         
        Example example;
        NICE::Vector & xTrain = trainDataOrig[exIt->second[exampleIndex]];
        example.svec = new SparseVector(xTrain);
        examples.push_back ( pair<int, Example> ( y[exIt->second[exampleIndex] ], example ) );         
         
        exIt->second.erase(exIt->second.begin()+exampleIndex);
      }
    }    
    std::cerr << "start training " << std::endl;
    time_t  prep_start_time = clock();

    GPHIKClassifierNICE * classifier  = new GPHIKClassifierNICE( &conf );
    
    FeaturePool fp; // will be ignored
    classifier->train ( fp, examples );

    float time_preparation = ( float ) ( clock() - prep_start_time ) ;
    
    int classesUsed(classesForTraining.size());
    
    std::cerr << "training done " << std::endl;
    
      // ------------------ TESTING
    const LabeledSet *test = md["test"];
    VVector testData;
    Vector yTest;
    readData< VVector, Vector > ( conf, *test, testData, yTest, ext );
    

    NICE::Matrix confusionMatrix ( numberOfClasses, numberOfClasses );
    confusionMatrix.set ( 0.0 );

    time_t  start_time = clock();

    std::vector<int> chosen_examples_per_class ( numberOfClasses );

    if ( do_classification )
    {
      for ( uint i = 0 ; i < testData.size(); i++ )
      {
        Example example;
        const Vector & xstar = testData[i];
        SparseVector xstar_sparse ( xstar );
        OBJREC::ClassificationResult result;
        example.svec = &xstar_sparse;
        
        result = classifier->classify( example );
        cerr << "[" << i << " / " << testData.size() << "] " << result.classno << " " << yTest[i] << std::endl;
        
        result.classno_groundtruth = yTest[i];
        confusionMatrix ( result.classno_groundtruth , result.classno ) ++;
      }


      float time_classification = ( float ) ( clock() - start_time ) ;
      if ( verbose >= LOW )
        cerr << "Time for Classification with " << classesUsed*trainExPerClass << " training-examples: " << time_classification / CLOCKS_PER_SEC << " [s]" << endl;
      ( classification_times[0] ).push_back ( time_classification / CLOCKS_PER_SEC );

      confusionMatrix.normalizeRowsL1();
      double avg_recognition_rate = 0.0;
      for ( int i = 0 ; i < ( int ) confusionMatrix.rows(); i++ )
      {
        if ( verbose >= MEDIUM )
        {
          cerr << "Class no: " <<  i  << " : " << confusionMatrix ( i, i ) << endl;
        }
        avg_recognition_rate += confusionMatrix ( i, i );
      }

      avg_recognition_rate /= confusionMatrix.rows();

      cerr << confusionMatrix << endl;
      cerr << "avg recognition rate " << avg_recognition_rate*100 << " %" << endl;

      recognitions_rates[0].push_back ( avg_recognition_rate*100 );
    }

    //Now start the Incremental-Learning-Part
    
    for (int incrementationStep = 0; incrementationStep < nrOfIncrements; incrementationStep++)
    {
      
      //iteratively add 1 example
      if (incrementalNotBatch)
      {
      uint oldSize = examples.size();
      //chose examples for every class used for training
      int cnt(0);
      Examples newExamples;
      for (std::set<int>::const_iterator clIt = classesForTraining.begin(); clIt != classesForTraining.end(); clIt++)
      {
        std::map<int,std::vector<int> >::iterator exIt = examplesPerClassInDatasetTmp.find(*clIt);
        
        for (int i = 0; i < incrementalAddSize; i++)
        {
          std::cerr << "i: " << cnt << std::endl;
          Example example;    
          
          int exampleIndex ( rand() % ( exIt->second.size() ) );
          NICE::Vector & xTrain = trainDataOrig[exIt->second[exampleIndex] ];
          example.svec = new SparseVector(xTrain);
//           examples.push_back ( pair<int, Example> ( y[exIt->second[exampleIndex] ], example ) );           
          newExamples.push_back ( pair<int, Example> ( y[exIt->second[exampleIndex] ], example ) );
          exIt->second.erase(exIt->second.begin()+exampleIndex);
          cnt++;
        }
      }        
           
        std::cerr << "Incremental, but not batch" << std::endl;
        time_t  IL_add_start_time = clock();

        classifier->addMultipleExamples( newExamples );
        float time_IL_add = ( float ) ( clock() - IL_add_start_time ) ;
        std::cerr << "Time for IL-adding of " << incrementalAddSize*classesForTraining.size() << " examples to already " <<  classesUsed*trainExPerClass+classesUsed*incrementalAddSize*incrementationStep << "  training-examples: " << time_IL_add / CLOCKS_PER_SEC << " [s]" << std::endl;
        IL_training_times[incrementationStep].push_back(time_IL_add / CLOCKS_PER_SEC);
      }
      else
      {       
        std::cerr << "batch retraining -- add new data to currently known training examples" << std::endl;
        //chose examples for every class used for training
        int cnt(0);
        for (std::set<int>::const_iterator clIt = classesForTraining.begin(); clIt != classesForTraining.end(); clIt++)
        {
          std::map<int,std::vector<int> >::iterator exIt = examplesPerClassInDatasetTmp.find(*clIt);
          
          for (int i = 0; i < incrementalAddSize; i++)
          {
            std::cerr << "i: " << cnt << std::endl;
            Example example;    
            
            int exampleIndex ( rand() % ( exIt->second.size() ) );
            NICE::Vector & xTrain = trainDataOrig[exIt->second[exampleIndex] ];
            example.svec = new SparseVector(xTrain);
            examples.push_back ( pair<int, Example> ( y[exIt->second[exampleIndex] ], example ) );           

            exIt->second.erase(exIt->second.begin()+exampleIndex);
            cnt++;
          }
        }          

        std::cerr <<  "start batch retraining" << std::endl;
        time_t  batch_add_start_time = clock();        
        //
        if (classifier != NULL)
          delete classifier;
        classifier = new GPHIKClassifierNICE( &conf );
        classifier->train( fp, examples );
        //
        float time_batch_add = ( float ) ( clock() - batch_add_start_time ) ;
        std::cerr << "Time for batch relearning after adding of " << incrementalAddSize*classesForTraining.size() << " examples to already " <<  classesUsed*trainExPerClass+classesUsed*incrementalAddSize*incrementationStep << "  training-examples: " << time_batch_add / CLOCKS_PER_SEC << " [s]" << std::endl;
        IL_training_times[incrementationStep].push_back(time_batch_add / CLOCKS_PER_SEC);        
      }
           
      //do the classification for evaluating the benefit of new examples
      if ( do_classification )
      {
        std::cerr << "do classification" << std::endl;
        for ( uint i = 0 ; i < testData.size(); i++ )
        {
          Example example;
          const Vector & xstar = testData[i];
          SparseVector xstar_sparse ( xstar );
          example.svec = &xstar_sparse;
          OBJREC::ClassificationResult result;
          
          result = classifier->classify( example );
          
          std::cerr << "[" << i << " / " << testData.size() << "] " << result.classno << " " << yTest[i] << std::endl; 
          
          result.classno_groundtruth = yTest[i];
          confusionMatrix ( result.classno_groundtruth , result.classno ) ++;
        }     


        float time_classification = ( float ) ( clock() - start_time ) ;
        if ( verbose >= LOW )
          std::cerr << "Time for Classification with " << classesUsed*trainExPerClass+classesUsed*incrementalAddSize*(incrementationStep+1) << " training-examples: " << time_classification / CLOCKS_PER_SEC << " [s]" << std::endl;
        ( classification_times[incrementationStep+1] ).push_back ( time_classification / CLOCKS_PER_SEC );

        confusionMatrix.normalizeRowsL1();
        double avg_recognition_rate = 0.0;
        for ( int i = 0 ; i < ( int ) confusionMatrix.rows(); i++ )
        {
          if ( verbose >= MEDIUM )
          {
            std::cerr << "Class no: " <<  i  << " : " << confusionMatrix ( i, i ) << std::endl;
          }
          avg_recognition_rate += confusionMatrix ( i, i );
        }

        avg_recognition_rate /= confusionMatrix.rows();

        cerr << confusionMatrix << endl;
        cerr << "avg recognition rate " << avg_recognition_rate*100 << " %" << endl;

        recognitions_rates[incrementationStep+1].push_back ( avg_recognition_rate*100 );
      } //classification after IL adding
    } //IL adding of different classes
  }//runs 


  int classesUsed(classesForTraining.size());
  std::cerr << "classes used: " << classesUsed << " incrementalAddSize: " << incrementalAddSize << std::endl;

  if ( do_classification )
  {
    std::cerr << "========================" << std::endl;
    std::cerr << "content of classification_times: " << std::endl;
    for ( std::vector<std::vector<float> >::const_iterator it = classification_times.begin(); it != classification_times.end(); it++ )
    {
      for ( std::vector<float> ::const_iterator jt = ( *it ).begin(); jt != ( *it ).end(); jt++ )
      {
        std::cerr << *jt << " ";
      }
      std::cerr << std::endl;
    }

    std::vector<float> mean_classification_times;
    std::vector<float> std_dev_classification_times;
    for ( std::vector<std::vector<float> >::const_iterator it = classification_times.begin(); it != classification_times.end(); it++ )
    {
      float mean_classification_time ( 0.0 );
      for ( std::vector<float>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        mean_classification_time += *itRun;
      }
      mean_classification_time /= it->size();
      mean_classification_times.push_back ( mean_classification_time );

      double std_dev_classification_time ( 0.0 );
      for ( std::vector<float>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        std_dev_classification_time += pow ( *itRun - mean_classification_time, 2 );
      }
      std_dev_classification_time /= it->size();
      std_dev_classification_time = sqrt ( std_dev_classification_time );
      std_dev_classification_times.push_back ( std_dev_classification_time );
    }
    
    int datasize ( classesUsed*trainExPerClass );
    for ( uint i = 0; i < mean_classification_times.size(); i++)
    {
      std::cerr << "size: " << datasize << " mean classification time: " << mean_classification_times[i] << " std_dev classification time: " << std_dev_classification_times[i] << std::endl;
      datasize += classesUsed*incrementalAddSize ;
    }
  }
  else
  {
    std::cerr << "========================" << std::endl;
    std::cerr << "No classification done therefor no classification times available." << std::endl;
  }

  std::cerr << "========================" << std::endl;
  std::cerr << "content of IL_training_times: " << std::endl;
  for ( std::vector<std::vector<float> >::const_iterator it = IL_training_times.begin(); it != IL_training_times.end(); it++ )
  {
    for ( std::vector<float> ::const_iterator jt = ( *it ).begin(); jt != ( *it ).end(); jt++ )
    {
      std::cerr << *jt << " ";
    }
    std::cerr << std::endl;
  }

  std::vector<float> mean_IL_training_times;
  std::vector<float> std_dev_IL_training_times;
  for ( std::vector<std::vector<float> >::const_iterator it = IL_training_times.begin(); it != IL_training_times.end(); it++ )
  {  
    float mean_IL_training_time ( 0.0 );
    for ( std::vector<float>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
    {
      mean_IL_training_time += *itRun;
    }
    mean_IL_training_time /= it->size();
    mean_IL_training_times.push_back ( mean_IL_training_time );

    double std_dev_IL_training_time ( 0.0 );
    for ( std::vector<float>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
    {
      std_dev_IL_training_time += pow ( *itRun - mean_IL_training_time, 2 );
    }
    std_dev_IL_training_time /= it->size();
    std_dev_IL_training_time = sqrt ( std_dev_IL_training_time );
    std_dev_IL_training_times.push_back ( std_dev_IL_training_time );
  }

  int datasize ( classesUsed*trainExPerClass );
  for ( uint i = 0; i < mean_IL_training_times.size(); i++)
  {
    cerr << "size: " << datasize << " and adding " << classesUsed*incrementalAddSize << " mean IL_training time: " << mean_IL_training_times[i] << " std_dev IL_training time: " << std_dev_IL_training_times[i] << endl;
    datasize += classesUsed*incrementalAddSize ;
  }

  if ( do_classification )
  {
    std::cerr << "========================" << std::endl;
    std::cerr << "content of recognition_rates: " << std::endl;
    for ( std::vector<std::vector<double> >::const_iterator it = recognitions_rates.begin(); it != recognitions_rates.end(); it++ )
    {
      for ( std::vector<double> ::const_iterator jt = ( *it ).begin(); jt != ( *it ).end(); jt++ )
      {
        std::cerr << *jt << " ";
      }
      std::cerr << std::endl;
    }

    std::cerr << "calculating final results " << std::endl;
    std::vector<double> mean_recs;
    std::vector<double> std_dev_recs;
    for (std::vector<std::vector<double> >::const_iterator it = recognitions_rates.begin(); it != recognitions_rates.end(); it++ )
    {
      double mean_rec ( 0.0 );
      for ( std::vector<double>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        mean_rec += *itRun;
      }
      mean_rec /= it->size();
      mean_recs.push_back ( mean_rec );

      double std_dev_rec ( 0.0 );
      for ( std::vector<double>::const_iterator itRun = it->begin(); itRun != it->end(); itRun++ )
      {
        std_dev_rec += pow ( *itRun - mean_rec, 2 );
      }
      std_dev_rec /= it->size();
      std_dev_rec = sqrt ( std_dev_rec );
      std_dev_recs.push_back ( std_dev_rec );
    }

    int datasize ( classesUsed*trainExPerClass + classesUsed*incrementalAddSize);
    for ( uint i = 0; i < recognitions_rates.size(); i++)
    {
      std::cerr << "size: " << datasize << " mean_IL: " << mean_recs[i] << " std_dev_IL: " << std_dev_recs[i] << std::endl;
      datasize += classesUsed*incrementalAddSize ;
    }
  }
  else
  {
    std::cerr << "========================" << std::endl;
    std::cerr << "No classification done therefor no classification times available." << std::endl;
  }

  return 0;
}