/**
* @file SemSegNovelty.h
* @brief semantic segmentation using the method from Csurka08
* @author Björn Fröhlich, Alexander Freytag
* @date 04/24/2009
*/
#ifndef _NICE_SEMSEGNOVELTYINCLUDE
#define _NICE_SEMSEGNOVELTYINCLUDE
// nice-core includes
#include <core/basics/Persistent.h>
// nice-vislearning includes
#include <vislearning/classifier/classifierbase/FeaturePoolClassifier.h>
#include <vislearning/classifier/genericClassifierSelection.h>
#include <vislearning/features/localfeatures/LocalFeatureColorWeijer.h>
// nice-segmentation includes
#include <segmentation/RegionSegmentationMethod.h>
// nice-semseg includes
#include "SemanticSegmentation.h"
#include "SemSegTools.h"
/** @brief pixelwise labeling systems */
namespace OBJREC {
class SemSegNovelty : public SemanticSegmentation
{
protected:
/////////////////////////
/////////////////////////
// PROTECTED VARIABLES //
/////////////////////////
/////////////////////////
////////////////////////////////////////
// variables only setable via configfile
////////////////////////////////////////
///////////////////////////////
// FEATURE EXTRACTION //
///////////////////////////////
//! feature extraction
LocalFeatureColorWeijer *featExtract;
//! distance between features for training
int trainWSize;
//! half of the window size for local features
int whs;
//! rectangle size for classification, 1 means pixelwise
int testWSize;
///////////////////////////////
// NOVELTY COMPUTATION //
///////////////////////////////
enum NoveltyMethod{
GPVARIANCE, // novel = large variance
GPUNCERTAINTY, //novel = small uncertainty (mean / var)
GPMINMEAN, //novel = small mean
GPMEANRATIO, //novel = small difference between mean of most plausible class and mean of snd
// most plausible class (not useful in binary settings)
GPWEIGHTALL, // novel = large weight in alpha vector after updating the model (can be predicted exactly)
GPWEIGHTRATIO, // novel = small difference between weights for alpha vectors with assumptions of GT label to be the most
// plausible against the second most plausible class
RANDOM // query regions randomly
};
//! specify how "novelty" shall be computed, e.g., using GP-variance, GP-uncertainty, or predicted weight entries
NoveltyMethod noveltyMethod;
std::string noveltyMethodString;
//! maximum uncertainty over all images, i.e., the novelty score of the most "novel" region of all test images
double globalMaxUncert;
//! determine whether a "novelty" method computes large scores for novel objects (e.g., variance), or small scores (e.g., min abs mean)
bool mostNoveltyWithMaxScores;
//! find the maximum uncertainty or not within the whole test set
bool findMaximumUncert;
//! image with most uncertain region
NICE::ColorImage maskedImg;
//! for debugging and visualization: show novelty images with and without region segmentation and the most novel region
bool b_visualizeALimages;
///////////////////////////////
// CLASSIFICATION STUFF //
///////////////////////////////
//! just store the name of our classifier
// Theoretically redundant, but currently makes things easier for store and restore...
std::string classifierString;
//! Classifier
FeaturePoolClassifier *classifier;
VecClassifier *vclassifier;
//! set of forbidden/background classes for the initial training
std::set<int> forbidden_classesTrain;
//! set of forbidden/background classes for the whole process of learning over time
std::set<int> forbidden_classesActiveLearning;
//! store the class numbers currently used
std::set<int> classesInUse;
//! obviously, the number of classes used for training (i.e., classesInUse.size() )
int numberOfClasses;
//! boolean whether to read the initial classifier from a file. If not, training will be performed
bool read_classifier;
//! boolean whether to save the final classifier or not
bool save_classifier;
//! The cached Data
std::string cache;
//! where to save the resulting images (uncertainty and classification results)
std::string resultdir;
//! current examples for most uncertain region
Examples newTrainExamples;
NICE::MultiChannelImageT<double> m_CurrentImageFeatures;
///////////////////////////////
// SEGMENTATION STUFF //
///////////////////////////////
//! just store the name of our segmentation method.
// Theoretically redundant, but currently makes things easier for store and restore...
std::string s_rsMethode;
//! low level Segmentation method
RegionSegmentationMethod *regionSeg;
//! boolean whether to reuse segmentation results for single images in different runs
bool reuseSegmentation;
//! contains filenames of images and indices of contained regions, that where already queried, to prevent them from being queried again
std::map<std::string,std::set<int> > queriedRegions;
std::pair<std::string, int> currentRegionToQuery;
///////////////////////////////
// protected methods
///////////////////////////////
inline void computeClassificationResults( const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize,
const int & ysize,
const int & featdim );
void computeNoveltyByRandom( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
void computeNoveltyByVariance( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
void computeNoveltyByGPUncertainty ( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
void computeNoveltyByGPMean ( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
void computeNoveltyByGPMeanRatio ( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
void computeNoveltyByGPWeightAll ( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
void computeNoveltyByGPWeightRatio ( NICE::FloatImage & noveltyImage,
const NICE::MultiChannelImageT<double> & feats,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities,
const int & xsize, const int & ysize, const int & featdim );
public:
/**
* @brief default constructor
* @author Alexander Freytag
* @date 06-02-2014 ( dd-mm-yyyy )
*/
SemSegNovelty ( );
/**
* @brief recommended constructor
* @author Alexander Freytag
* @param conf needs a configfile
* @param md and a MultiDataset (contains images and other things)
*/
SemSegNovelty ( const NICE::Config *conf, const MultiDataset *md );
/** simple destructor */
virtual ~SemSegNovelty();
void initFromConfig ( const NICE::Config * conf, const std::string _confSection = "SemSegNovelty" );
/** The trainingstep
* @param md and a MultiDataset (contains images and other things)
*/
void train ( const MultiDataset *md );
/** The main procedure. Input: Image, Output: Segmented Image with pixelwise labeles and the probabilities
* @param ce image data
* @param segresult result of the semantic segmentation with a label for each pixel
* @param probabilities multi-channel image with one channel for each class and corresponding probabilities for each pixel
*/
void semanticseg ( CachedExample *ce,
NICE::ImageT<int> & segresult,
NICE::MultiChannelImageT<double> & probabilities );
void semanticseg ( CachedExample *ce,
NICE::MultiChannelImageT<int> & segresult,
NICE::MultiChannelImage3DT<double> & probabilities )
{}
/**
* @brief visualize a specific region in the original image
*
* @param img input image
* @param regions map of the regions
* @param region visualize this region
* @param outimage result
* @return void
**/
void visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix ®ions, int region, NICE::ColorImage &outimage);
/**
* @brief Add a new example to the known training data
*
* @param newExample (NICE::Vector) the feature vector of the new examples
* @param newClassNo (int) the corresponding GT class number
* @return void
**/
void addNewExample(const NICE::Vector & newExample, const int & newClassNo);
/**
* @brief Add those examples, which belong to the most novel region seen so far
*
* @return void
**/
virtual void addNovelExamples();
/**
* @brief Get a pointer to the examples extracted from the most novel region seen so far
*
* @return Examples *
**/
virtual const Examples * getNovelExamples() const;
///////////////////// INTERFACE PERSISTENT /////////////////////
// interface specific methods for store and restore
///////////////////// INTERFACE PERSISTENT /////////////////////
/**
* @brief Load active-segmentation-object from external file (stream)
* @author Alexander Freytag
*/
virtual void restore ( std::istream & is, int format = 0 );
/**
* @brief Save active-segmentation-object to external file (stream)
* @author Alexander Freytag
*/
virtual void store( std::ostream & os, int format = 0 ) const;
/**
* @brief Clear active-segmentation-object object
* @author Alexander Freytag
*/
virtual void clear ();
};
} //namespace
#endif //_NICE_SEMSEGNOVELTYINCLUDE