added SemSegNovelty

semseg/SemSegNovelty.cpp

@@ -0,0 +1,319 @@
+#include <sstream>
+#include <iostream>
+
+#include "SemSegNovelty.h"
+
+#include "fast-hik/GPHIKClassifier.h"
+#include "vislearning/baselib/ICETools.h"
+#include "vislearning/baselib/Globals.h"
+#include "vislearning/features/fpfeatures/SparseVectorFeature.h"
+#include "core/basics/StringTools.h"
+#include "core/basics/Timer.h"
+
+using namespace std;
+using namespace NICE;
+using namespace OBJREC;
+
+SemSegNovelty::SemSegNovelty ( const Config *conf,
+                               const MultiDataset *md )
+    : SemanticSegmentation ( conf, & ( md->getClassNames ( "train" ) ) )
+{
+  this->conf = conf;
+
+  featExtract = new LFColorWeijer ( conf );
+
+  save_cache = conf->gB ( "FPCPixel", "save_cache", true );
+  read_cache = conf->gB ( "FPCPixel", "read_cache", false );
+  uncertdir = conf->gS("debug", "uncertainty","uncertainty");
+  cache = conf->gS ( "cache", "root", "" );
+
+  classifier = new GPHIKClassifier ( conf, "ClassiferGPHIK" );;
+
+  whs = conf->gI ( "SemSegNovelty", "window_size", 10 );
+  featdist = conf->gI ( "SemSegNovelty", "grid", 10 );
+
+  cn = md->getClassNames ( "train" );
+
+  if ( read_cache )
+  {
+    string classifierdst = "/classifier.data";
+    fprintf ( stderr, "SemSegNovelty:: Reading classifier data from %s\n", ( cache + classifierdst ).c_str() );
+
+    try
+    {
+      if ( classifier != NULL )
+      {
+        classifier->read ( cache + classifierdst );
+      }
+
+      fprintf ( stderr, "SemSegNovelty:: successfully read\n" );
+    }
+    catch ( char *str )
+    {
+      cerr << "error reading data: " << str << endl;
+    }
+  }
+  else
+  {
+    train ( md );
+  }
+}
+
+SemSegNovelty::~SemSegNovelty()
+{
+  // clean-up
+  if ( classifier != NULL )
+    delete classifier;
+  if ( featExtract != NULL )
+    delete featExtract;
+}
+
+void SemSegNovelty::train ( const MultiDataset *md )
+{
+  const LabeledSet train = * ( *md ) ["train"];
+  const LabeledSet *trainp = &train;
+
+  ////////////////////////
+  // feature extraction //
+  ////////////////////////
+
+  std::string forbidden_classes_s = conf->gS ( "analysis", "donttrain", "" );
+  if ( forbidden_classes_s == "" )
+  {
+    forbidden_classes_s = conf->gS ( "analysis", "forbidden_classes", "" );
+  }
+  cn.getSelection ( forbidden_classes_s, forbidden_classes );
+  cerr << "forbidden: " << forbidden_classes_s << endl;
+
+  ProgressBar pb ( "Local Feature Extraction" );
+  pb.show();
+
+  int imgnb = 0;
+
+  Examples examples;
+  examples.filename = "training";
+
+  int featdim = -1;
+
+  LOOP_ALL_S ( *trainp )
+  {
+    //EACH_S(classno, currentFile);
+    EACH_INFO ( classno, info );
+
+    std::string currentFile = info.img();
+
+    CachedExample *ce = new CachedExample ( currentFile );
+
+    const LocalizationResult *locResult = info.localization();
+    if ( locResult->size() <= 0 )
+    {
+      fprintf ( stderr, "WARNING: NO ground truth polygons found for %s !\n",
+                currentFile.c_str() );
+      continue;
+    }
+
+    int xsize, ysize;
+    ce->getImageSize ( xsize, ysize );
+
+    Image labels ( xsize, ysize );
+    labels.set ( 0 );
+    locResult->calcLabeledImage ( labels, ( *classNames ).getBackgroundClass() );
+
+    NICE::ColorImage img;
+    try {
+      img = ColorImage ( currentFile );
+    } catch ( Exception ) {
+      cerr << "SemSegNovelty: error opening image file <" << currentFile << ">" << endl;
+      continue;
+    }
+
+    Globals::setCurrentImgFN ( currentFile );
+
+    MultiChannelImageT<double> feats;
+
+    // extract features
+    featExtract->getFeats ( img, feats );
+    featdim = feats.channels();
+
+    // compute integral images
+    for ( int c = 0; c < featdim; c++ )
+    {
+      feats.calcIntegral ( c );
+    }
+
+    for ( int y = 0; y < ysize; y += featdist )
+    {
+      for ( int x = 0; x < xsize; x += featdist )
+      {
+        int classno = labels ( x, y );
+
+        if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
+          continue;
+
+       
+        Example example;
+        example.vec = NULL;
+        example.svec = new SparseVector ( featdim );
+        for ( int f = 0; f < featdim; f++ )
+        {
+          double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
+          if ( val > 1e-10 )
+            ( *example.svec ) [f] = val;
+        }
+        
+        example.svec->normalize();
+        
+        example.position = imgnb;
+        examples.push_back ( pair<int, Example> ( classno, example ) );
+      }
+    }
+
+    delete ce;
+    imgnb++;
+    pb.update ( trainp->count() );
+  }
+
+  pb.hide();
+
+
+  //////////////////////
+  // train classifier //
+  //////////////////////
+  FeaturePool fp;
+
+  Feature *f = new SparseVectorFeature ( featdim );
+
+  f->explode ( fp );
+  delete f;
+
+  if ( classifier != NULL )
+    classifier->train ( fp, examples );
+  else
+  {
+    cerr << "no classifier selected?!" << endl;
+    exit ( -1 );
+  }
+
+  fp.destroy();
+
+  if ( save_cache )
+  {
+    if ( classifier != NULL )
+      classifier->save ( cache + "/classifier.data" );
+  }
+
+  ////////////
+  //clean up//
+  ////////////
+  for ( int i = 0; i < ( int ) examples.size(); i++ )
+  {
+    examples[i].second.clean();
+  }
+  examples.clear();
+
+  cerr << "SemSeg training finished" << endl;
+}
+
+void SemSegNovelty::semanticseg ( CachedExample *ce, NICE::Image & segresult, NICE::MultiChannelImageT<double> & probabilities )
+{
+  Timer timer;
+  timer.start();
+  
+  Examples examples;
+  examples.filename = "testing";
+
+  segresult.set ( 0 );
+
+  int featdim = -1;
+
+  std::string currentFile = Globals::getCurrentImgFN();
+
+
+  int xsize, ysize;
+  ce->getImageSize ( xsize, ysize );
+
+  probabilities.reInit( xsize, ysize, cn.getMaxClassno()+1);
+  probabilities.set ( 0.0 );
+  
+  NICE::ColorImage img;
+  try {
+    img = ColorImage ( currentFile );
+  } catch ( Exception ) {
+    cerr << "SemSegNovelty: error opening image file <" << currentFile << ">" << endl;
+    return;
+  }
+
+  MultiChannelImageT<double> feats;
+
+  // extract features
+  featExtract->getFeats ( img, feats );
+  featdim = feats.channels();
+  
+  // compute integral images
+  for ( int c = 0; c < featdim; c++ )
+  {
+    feats.calcIntegral ( c );
+  }
+
+  FloatImage uncert ( xsize, ysize );
+  uncert.set ( 0.0 );
+
+  double maxunc = -numeric_limits<double>::max();
+  timer.stop();
+  cout << "first: " << timer.getLastAbsolute() << endl;
+  timer.start();
+#pragma omp parallel for
+  for ( int y = 0; y < ysize; y++ )
+  {
+    Example example;
+    example.vec = NULL;
+    example.svec = new SparseVector ( featdim );
+    for ( int x = 0; x < xsize; x++ )
+    {
+      for ( int f = 0; f < featdim; f++ )
+      {
+        double val = feats.getIntegralValue ( x - whs, y - whs, x + whs, y + whs, f );
+        if ( val > 1e-10 )
+          ( *example.svec ) [f] = val;
+      }
+      example.svec->normalize();
+      
+      ClassificationResult cr = classifier->classify ( example );
+
+      for ( int j = 0 ; j < cr.scores.size(); j++ )
+      {
+        probabilities ( x, y, j ) = cr.scores[j];
+      }
+      segresult ( x, y ) = cr.classno;
+      if(maxunc < cr.uncertainty)
+        maxunc = cr.uncertainty;
+      uncert ( x, y ) = cr.uncertainty;
+      example.svec->clear();
+    }
+    delete example.svec;
+    example.svec = NULL;
+  }
+  
+  cout << "maxunertainty: " << maxunc << endl;
+
+  timer.stop();
+  cout << "second: " << timer.getLastAbsolute() << endl;
+  timer.start();
+  
+  ColorImage imgrgb ( xsize, ysize );
+
+  std::stringstream out;
+  std::vector< std::string > list2;
+  StringTools::split ( Globals::getCurrentImgFN (), '/', list2 );
+  out << uncertdir << "/" << list2.back();
+
+  uncert.writeRaw(out.str()+".rawfloat");
+  uncert(0,0) = 0.0;
+  uncert(0,1) = 1.0;
+  ICETools::convertToRGB ( uncert, imgrgb );
+  imgrgb.write ( out.str() + "rough.png" );
+
+  
+  timer.stop();
+  cout << "last: " << timer.getLastAbsolute() << endl;
+}

semseg/SemSegNovelty.h

@@ -0,0 +1,86 @@
+/**
+ * @file SemSegNovelty.h
+ * @brief semantic segmentation using the method from Csurka08
+ * @author Björn Fröhlich
+ * @date 04/24/2009
+ */
+#ifndef SemSegNoveltyINCLUDE
+#define SemSegNoveltyINCLUDE
+
+#include "SemanticSegmentation.h"
+
+#include "SemSegTools.h"
+#include "vislearning/classifier/classifierbase/FeaturePoolClassifier.h"
+#include "vislearning/features/localfeatures/LFColorWeijer.h"
+
+
+/** @brief pixelwise labeling systems */
+
+namespace OBJREC {
+
+class SemSegNovelty : public SemanticSegmentation
+{
+
+  protected:
+    //! boolean whether to save the cache or not
+    bool save_cache;
+
+    //! boolean whether to read the cache or not, if read_cache is false, everything will be trained
+    bool read_cache;
+
+    //! The cached Data
+    std::string cache;
+    
+    //! Classifier
+    FeaturePoolClassifier *classifier;
+    
+    //! feature extraction
+    LFColorWeijer *featExtract;
+    
+    //! Configuration File
+    const NICE::Config *conf;
+    
+    //! distance between features for training
+    int featdist;
+    
+    //! half of the window size for local features
+    int whs;
+    
+    //! name of all classes
+    ClassNames cn;
+    
+    //! set of forbidden/background classes
+    std::set<int> forbidden_classes;
+    
+    //! where to save the uncertainty
+    std::string uncertdir;
+   
+  public:
+
+    /** constructor
+      *  @param conf needs a configfile
+      *  @param md and a MultiDataset (contains images and other things)
+      */
+    SemSegNovelty ( const NICE::Config *conf, const MultiDataset *md );
+
+    /** simple destructor */
+    virtual ~SemSegNovelty();
+
+    /** The trainingstep
+      *  @param md and a MultiDataset (contains images and other things)
+      */
+    void train ( const MultiDataset *md );
+
+    /** The main procedure. Input: Image, Output: Segmented Image with pixelwise labeles and the probabilities
+      * @param ce image data
+      * @param segresult result of the semantic segmentation with a label for each pixel
+      * @param probabilities multi-channel image with one channel for each class and corresponding probabilities for each pixel
+      */
+    void semanticseg ( CachedExample *ce,
+                       NICE::Image & segresult,
+                       NICE::MultiChannelImageT<double> & probabilities );
+};
+
+} //namespace
+
+#endif