NICE_SemSeg/semseg/SemSegNovelty.h

/**
 * @file SemSegNovelty.h
 * @brief semantic segmentation using the method from Csurka08
 * @author Björn Fröhlich, Alexander Freytag
 * @date 04/24/2009
 */
#ifndef _NICE_SEMSEGNOVELTYINCLUDE
#define _NICE_SEMSEGNOVELTYINCLUDE


// nice-core includes
#include <core/basics/Persistent.h>

// nice-vislearning includes
#include <vislearning/classifier/classifierbase/FeaturePoolClassifier.h>
#include <vislearning/classifier/genericClassifierSelection.h>
#include <vislearning/features/localfeatures/LocalFeatureColorWeijer.h>

// nice-segmentation includes
#include <segmentation/RegionSegmentationMethod.h>

// nice-semseg includes
#include "SemanticSegmentation.h"
#include "SemSegTools.h"



/** @brief pixelwise labeling systems */

namespace OBJREC {

class SemSegNovelty : public SemanticSegmentation
{

  protected:
    
    /////////////////////////
    /////////////////////////
    // PROTECTED VARIABLES //
    /////////////////////////
    /////////////////////////    
    
    ////////////////////////////////////////
    // variables only setable via configfile
    ////////////////////////////////////////
  
    
    ///////////////////////////////
    //     FEATURE EXTRACTION    //
    ///////////////////////////////  

    //! feature extraction
    LocalFeatureColorWeijer *featExtract;     
    
    //! distance between features for training
    int trainWsize;
    
    //! half of the window size for local features
    int whs;
    
    //! rectangle size for classification, 1 means pixelwise
    int testWSize;
    
    
    ///////////////////////////////
    //     NOVELTY COMPUTATION   //
    ///////////////////////////////
    
    enum NoveltyMethod{
      GPVARIANCE, // novel = large variance
      GPUNCERTAINTY, //novel = small uncertainty (mean / var)
      GPMINMEAN,  //novel = small mean
      GPMEANRATIO,  //novel = small difference between mean of most plausible class and mean of snd
                   //        most plausible class (not useful in binary settings)
      GPWEIGHTALL, // novel = large weight in alpha vector after updating the model (can be predicted exactly)
      GPWEIGHTRATIO, // novel = small difference between weights for alpha vectors with assumptions of GT label to be the most 
                    //         plausible against the second most plausible class
      RANDOM        // query regions randomly
    }; 
    
    //! specify how "novelty" shall be computed, e.g., using GP-variance, GP-uncertainty, or predicted weight entries
    NoveltyMethod noveltyMethod;
    std::string noveltyMethodString;
    
    //! maximum uncertainty over all images, i.e., the novelty score of the most "novel" region of all test images
    double globalMaxUncert;
    
    //! determine whether a "novelty" method computes large scores for novel objects (e.g., variance), or small scores (e.g., min abs mean)
    bool mostNoveltyWithMaxScores;
    
    //! find the maximum uncertainty or not within the whole test set
    bool findMaximumUncert;
    
    //! image with most uncertain region
    NICE::ColorImage maskedImg;
    
    //! for debugging and visualization: show novelty images with and without region segmentation and the most novel region
    bool b_visualizeALimages;      
    
    
    ///////////////////////////////
    //     CLASSIFICATION STUFF  //
    /////////////////////////////// 
    
    //! just store the name of our classifier
    // Theoretically redundant, but currently makes things easier for store and restore...
    std::string classifierString;
    
    //! Classifier
    FeaturePoolClassifier *classifier;
    VecClassifier *vclassifier; 
    
    //! set of forbidden/background classes for the initial training
    std::set<int> forbidden_classesTrain;
    //! set of forbidden/background classes for the whole process of learning over time
    std::set<int> forbidden_classesActiveLearning;
    //! store the class numbers currently used
    std::set<int> classesInUse;
    
   //! obviously, the number of classes used for training (i.e., classesInUse.size() )
    int numberOfClasses;
    
    //! boolean whether to read the initial classifier from a file. If not, training will be performed
    bool read_classifier;
    
    //! boolean whether to save the final classifier or not
    bool save_classifier;

    //! The cached Data
    std::string cache; 
    
   //! where to save the resulting images (uncertainty and classification results)
    std::string resultdir;    
        
    
    //! current examples for most uncertain region
    Examples newTrainExamples;    
    
    ///////////////////////////////
    //     SEGMENTATION STUFF    //
    /////////////////////////////// 
    
    //! just store the name of our segmentation method. 
    // Theoretically redundant, but currently makes things easier for store and restore...
    std::string s_rsMethode;
    
    //! low level Segmentation method
    RegionSegmentationMethod *regionSeg;

    //! boolean whether to reuse segmentation results for single images in different runs
    bool reuseSegmentation;    

    
    //! contains filenames of images and indices of contained regions, that where already queried, to prevent them from being queried again
    std::map<std::string,std::set<int> > queriedRegions;
    
    std::pair<std::string, int> currentRegionToQuery;
    

    ///////////////////////////////
    //     protected methods
    ///////////////////////////////
    
  
    inline void computeClassificationResults( const NICE::MultiChannelImageT<double> & feats, 
                                                    NICE::Image & segresult,
                                                    NICE::MultiChannelImageT<double> & probabilities,
                                                    const int & xsize,
                                                    const int & ysize,
                                                    const int & featdim );

   void computeNoveltyByRandom(         NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );    
    
   void computeNoveltyByVariance(       NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );
   
   void computeNoveltyByGPUncertainty ( NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );
   
   void computeNoveltyByGPMean        ( NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );  
   void computeNoveltyByGPMeanRatio   ( NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );  
   void computeNoveltyByGPWeightAll   ( NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );  
   void computeNoveltyByGPWeightRatio ( NICE::FloatImage & noveltyImage, 
                                  const NICE::MultiChannelImageT<double> & feats,  
                                        NICE::Image & segresult,
                                        NICE::MultiChannelImageT<double> & probabilities,
                                  const int & xsize, const int & ysize, const int & featdim );     
   
  public:

    /** 
     * @brief default constructor
     * @author Alexander Freytag
     * @date 06-02-2014 ( dd-mm-yyyy )
     */
    SemSegNovelty ( );    
    
    /** 
     * @brief recommended constructor
     * @author Alexander Freytag
     *  @param conf needs a configfile
     *  @param md and a MultiDataset (contains images and other things)
     */
    SemSegNovelty ( const NICE::Config *conf, const MultiDataset *md );

    /** simple destructor */
    virtual ~SemSegNovelty();
    
    void initFromConfig ( const NICE::Config * conf, const std::string _confSection = "SemSegNovelty" );

    /** The trainingstep
      *  @param md and a MultiDataset (contains images and other things)
      */
    void train ( const MultiDataset *md );

    /** The main procedure. Input: Image, Output: Segmented Image with pixelwise labeles and the probabilities
      * @param ce image data
      * @param segresult result of the semantic segmentation with a label for each pixel
      * @param probabilities multi-channel image with one channel for each class and corresponding probabilities for each pixel
      */
    void semanticseg ( CachedExample *ce,
                       NICE::Image & segresult,
                       NICE::MultiChannelImageT<double> & probabilities );
    
    /**
     * @brief visualize a specific region in the original image
     *
     * @param img input image
     * @param regions map of the regions
     * @param region visualize this region
     * @param outimage result
     * @return void
     **/
    void visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix &regions, int region, NICE::ColorImage &outimage);

    /**
     * @brief Add a new example to the known training data
     *
     * @param newExample (NICE::Vector) the feature vector of the new examples
     * @param newClassNo (int) the corresponding GT class number
     * @return void
     **/    
    void addNewExample(const NICE::Vector & newExample, const int & newClassNo);
    
    /**
     * @brief Add those examples, which belong to the most novel region seen so far
     *
     * @return void
     **/    
    virtual void addNovelExamples();    

    /**
     * @brief Get a pointer to the examples extracted from the most novel region seen so far
     *
     * @return Examples *
     **/        
    virtual const Examples * getNovelExamples() const; 
    
    ///////////////////// INTERFACE PERSISTENT /////////////////////
    // interface specific methods for store and restore
    ///////////////////// INTERFACE PERSISTENT /////////////////////   
    
    /** 
     * @brief Load active-segmentation-object from external file (stream)
     * @author Alexander Freytag
     */     
    virtual void restore ( std::istream & is, int format = 0 );
    
    /** 
     * @brief Save active-segmentation-object to external file (stream)
     * @author Alexander Freytag
     */       
    virtual void store( std::ostream & os, int format = 0 ) const;
    
    /** 
     * @brief Clear active-segmentation-object object
     * @author Alexander Freytag
     */    
    virtual void clear ();
    
};

} //namespace

#endif //_NICE_SEMSEGNOVELTYINCLUDE