NICE_VisLearning/progs/testNullSpaceNovelty.cpp

/** 
* @file testNullSpaceNovelty.cpp
* @brief test function for class KCNullSpaceNovelty
* @author Paul Bodesheim
* @date 28-11-2012 (dd-mm-yyyy)
*/

#include <ctime>
#include <time.h>

#include "core/basics/Config.h"
#include "core/basics/Timer.h"
#include "core/vector/Algorithms.h"
#include "core/vector/SparseVectorT.h"

#include "vislearning/classifier/kernelclassifier/KCNullSpaceNovelty.h"
#include "vislearning/math/kernels/KernelData.h"
#include "vislearning/cbaselib/ClassificationResults.h"
#include "vislearning/baselib/ProgressBar.h"

#include "core/matlabAccess/MatFileIO.h"
#include "vislearning/matlabAccessHighLevel/ImageNetData.h"

using namespace std;
using namespace NICE;
using namespace OBJREC;

// --------------- THE KERNEL FUNCTION ( exponential kernel with euclidian distance ) ----------------------
double measureDistance ( const NICE::SparseVector & a, const NICE::SparseVector & b, const double & sigma = 2.0)
{
  double inner_sum(0.0);

  double d;      
  
  //new version, where we needed on average 0.001707 s for each test sample
  NICE::SparseVector::const_iterator aIt = a.begin();
  NICE::SparseVector::const_iterator bIt = b.begin();
   
  //compute the euclidian distance between both feature vectores (given as SparseVectors)
  while ( (aIt != a.end()) && (bIt != b.end()) )
  {
    if (aIt->first == bIt->first)
    {
      d = ( aIt->second - bIt->second );      
      inner_sum += d * d;
      aIt++;
      bIt++;
    }
    else if ( aIt->first < bIt->first)
    {
      inner_sum += aIt->second * aIt->second;
      aIt++;      
    }
    else
    {
      inner_sum += bIt->second * bIt->second;
      bIt++;       
    }
  }
  
  //compute remaining values, if b reached the end but not a
  while (aIt != a.end())
  {
    inner_sum += aIt->second * aIt->second;
    aIt++; 
  }
  //compute remaining values, if a reached the end but not b
  while (bIt != b.end())
  {
    inner_sum += bIt->second * bIt->second;
    bIt++; 
  }  

  //normalization of the exponent
  inner_sum /= (2.0*sigma*sigma);
  
  //finally, compute the RBF-kernel score (RBF = radial basis function)
  return exp(-inner_sum);
}

// --------------- THE KERNEL FUNCTION ( HIK ) ----------------------
double minimumDistance ( const NICE::SparseVector & a, const NICE::SparseVector & b )
{
  double inner_sum(0.0);
  
  NICE::SparseVector::const_iterator aIt = a.begin();
  NICE::SparseVector::const_iterator bIt = b.begin();
  
  //compute the minimum distance between both feature vectores (given as SparseVectors)
  while ( (aIt != a.end()) && (bIt != b.end()) )
  {
    if (aIt->first == bIt->first)
    {
      inner_sum += std::min( aIt->second , bIt->second );
      aIt++;
      bIt++;
    }
    else if ( aIt->first < bIt->first)
    {
      aIt++;      
    }
    else
    {
      bIt++;       
    }
  }
  
  return inner_sum;
  
}

/** 
    test the basic functionality of fast-hik hyperparameter optimization 
*/
int main (int argc, char **argv)
{   
  std::set_terminate(__gnu_cxx::__verbose_terminate_handler);

  Config conf ( argc, argv );
  string resultsfile = conf.gS("main", "results", "results.txt" );
  int nrOfExamplesPerClass = conf.gI("main", "nrOfExamplesPerClass", 100);
  nrOfExamplesPerClass = std::min(nrOfExamplesPerClass, 100); // we do not have more than 100 examples per class
    
  // -------- read ImageNet data --------------  
  std::vector<SparseVector> trainingData;
  NICE::Vector y;
  
  std::cerr << "Reading ImageNet data ..." << std::endl;
  bool imageNetLocal = conf.gB("main", "imageNetLocal" , false);
  string imageNetPath;
  if (imageNetLocal)
    imageNetPath = "/users2/rodner/data/imagenet/devkit-1.0/";
  else
    imageNetPath = "/home/dbv/bilder/imagenet/devkit-1.0/";

  ImageNetData imageNetTrain ( imageNetPath + "demo/" );

  imageNetTrain.preloadData( "train", "training" );
  trainingData = imageNetTrain.getPreloadedData();
  y = imageNetTrain.getPreloadedLabels();
    
  std::cerr << "Reading of training data finished" << std::endl;
  std::cerr << "trainingData.size(): " << trainingData.size() << std::endl;
  std::cerr << "y.size(): " << y.size() << std::endl;
  
  std::cerr << "Reading ImageNet test data files (takes some seconds)..." << std::endl;
  ImageNetData imageNetTest ( imageNetPath + "demo/" );
  imageNetTest.preloadData ( "val", "testing" );
  imageNetTest.loadExternalLabels ( imageNetPath + "data/ILSVRC2010_validation_ground_truth.txt" );  
  
  // -------- select training set -------------
  NICE::Vector knownClassLabels(5,0.0);
  for (int k=1; k<6; k++)
    knownClassLabels(k-1) = k;
  
  std::vector<SparseVector> currentTrainingData;
  currentTrainingData.clear();
  NICE::Vector currentTrainingLabels(nrOfExamplesPerClass*knownClassLabels.size(),0);
  
  int k(0);
  for (size_t i = 0; i < y.size(); i++)
  {
    for (size_t j=0; j<knownClassLabels.size(); j++)
    {
      if ( y[i] == knownClassLabels[j] )
      {
        currentTrainingLabels(k) = knownClassLabels[j];
        currentTrainingData.push_back(trainingData[i]);
        k++;
        break;
      }  
    }
    
  }
  
  Timer tTrain;
  tTrain.start();
    
  //compute the kernel matrix
  NICE::Matrix kernelMatrix(nrOfExamplesPerClass*knownClassLabels.size(), nrOfExamplesPerClass*knownClassLabels.size(), 0.0);
  double kernelScore(0.0);
  int cl(0);
  
  for (size_t i = 0; i < kernelMatrix.rows(); i++)
  {
    for (size_t j = i; j < kernelMatrix.cols(); j++)
    {
      kernelScore = minimumDistance(currentTrainingData[i],currentTrainingData[j]);
      kernelMatrix(i-cl*100,j-cl*100) = kernelScore;
        
      if (i != j)
          kernelMatrix(j-cl*100,i-cl*100) = kernelScore;
    }
  }    
  
  KernelData kernelData( &conf, kernelMatrix, "Kernel", false );
  KCNullSpaceNovelty knfst( &conf);
  
  knfst.teach(&kernelData, currentTrainingLabels);
  
  tTrain.stop();
  std::cerr << "Time used for training " << cl << ": " << tTrain.getLast() << std::endl;  
  
  std::cerr << "training set statistic: " << std::endl;
  std::map<int,int>::iterator itt;
  for (itt = ( (std::map<int,int>) knfst.getTrainingSetStatistic() ).begin(); itt != knfst.getTrainingSetStatistic().end(); itt++)
    std::cerr << (*itt).first << "  " << (*itt).second << std::endl;
  std::cerr << "one-class setting?: " << knfst.isOneClass() << std::endl;
  std::cerr << "null space dimension: "<< knfst.getNullSpaceDimension() << std::endl;
  std::cerr << "target points: " << std::endl;
  for (size_t k=0; k<knfst.getTargetPoints().size(); k++)
    std::cerr << knfst.getTargetPoints()[k] << std::endl;    
       
  std::cerr << "training done - now perform the evaluation" << std::endl;

  // ------------------------------ TESTING ------------------------------
   
  std::cerr << "Classification step ... with " << imageNetTest.getNumPreloadedExamples() << " examples" << std::endl;
    
  ClassificationResults results;
     
  ProgressBar pb;
  Timer tTest;
  tTest.start();    
    
  for ( uint i = 0 ; i < (uint)imageNetTest.getNumPreloadedExamples(); i++ )
  {
    pb.update ( imageNetTest.getNumPreloadedExamples() );

    const SparseVector & svec = imageNetTest.getPreloadedExample ( i );
     
    //compute (self) similarities
    double kernelSelf (minimumDistance(svec,svec) );
    NICE::Vector kernelVector (nrOfExamplesPerClass, 0.0);
      
    for (int j = 0; j < nrOfExamplesPerClass; j++)
    {
      kernelVector[j] = minimumDistance(currentTrainingData[j],svec);
    }     
      
    ClassificationResult r;      
    r = knfst.classifyKernel( kernelVector, kernelSelf);  
      
    // set ground truth label
    r.classno_groundtruth = 0;
    for (size_t j=0; j<knownClassLabels.size(); j++)
    {
      if ( ((int)imageNetTest.getPreloadedLabel ( i )) == knownClassLabels[j] )
      {
        r.classno_groundtruth = 1;
        break;
      }
    }

    //remember the results for the evaluation lateron
    results.push_back ( r );
  }
    
  tTest.stop();
  std::cerr << "Time used for evaluation: " << tTest.getLast() << std::endl;       
    
  double timeForSingleExample(0.0);
  timeForSingleExample = tTest.getLast()/imageNetTest.getNumPreloadedExamples();
  std::cerr.precision(10);
  std::cerr << "time used for evaluation single elements: " << timeForSingleExample << std::endl;      

  // run the AUC-evaluation
  double perfvalue( 0.0 );   
  perfvalue = results.getBinaryClassPerformance( ClassificationResults::PERF_AUC );

  std::cerr << " novelty detection performance: " << perfvalue << std::endl;
  
  return 0;
}