/** * @file SemSegNovelty.h * @brief semantic segmentation using the method from Csurka08 * @author Björn Fröhlich, Alexander Freytag * @date 04/24/2009 */ #ifndef _NICE_SEMSEGNOVELTYINCLUDE #define _NICE_SEMSEGNOVELTYINCLUDE // nice-core includes #include // nice-vislearning includes #include #include #include // nice-segmentation includes #include // nice-semseg includes #include "SemanticSegmentation.h" #include "SemSegTools.h" /** @brief pixelwise labeling systems */ namespace OBJREC { class SemSegNovelty : public SemanticSegmentation { protected: ///////////////////////// ///////////////////////// // PROTECTED VARIABLES // ///////////////////////// ///////////////////////// //////////////////////////////////////// // variables only setable via configfile //////////////////////////////////////// /////////////////////////////// // FEATURE EXTRACTION // /////////////////////////////// //! feature extraction LocalFeatureColorWeijer *featExtract; //! distance between features for training int trainWSize; //! half of the window size for local features int whs; //! rectangle size for classification, 1 means pixelwise int testWSize; /////////////////////////////// // NOVELTY COMPUTATION // /////////////////////////////// enum NoveltyMethod{ GPVARIANCE, // novel = large variance GPUNCERTAINTY, //novel = small uncertainty (mean / var) GPMINMEAN, //novel = small mean GPMEANRATIO, //novel = small difference between mean of most plausible class and mean of snd // most plausible class (not useful in binary settings) GPWEIGHTALL, // novel = large weight in alpha vector after updating the model (can be predicted exactly) GPWEIGHTRATIO, // novel = small difference between weights for alpha vectors with assumptions of GT label to be the most // plausible against the second most plausible class RANDOM // query regions randomly }; //! specify how "novelty" shall be computed, e.g., using GP-variance, GP-uncertainty, or predicted weight entries NoveltyMethod noveltyMethod; std::string noveltyMethodString; //! maximum uncertainty over all images, i.e., the novelty score of the most "novel" region of all test images double globalMaxUncert; //! determine whether a "novelty" method computes large scores for novel objects (e.g., variance), or small scores (e.g., min abs mean) bool mostNoveltyWithMaxScores; //! find the maximum uncertainty or not within the whole test set bool findMaximumUncert; //! image with most uncertain region NICE::ColorImage maskedImg; //! for debugging and visualization: show novelty images with and without region segmentation and the most novel region bool b_visualizeALimages; /////////////////////////////// // CLASSIFICATION STUFF // /////////////////////////////// //! just store the name of our classifier // Theoretically redundant, but currently makes things easier for store and restore... std::string classifierString; //! Classifier FeaturePoolClassifier *classifier; VecClassifier *vclassifier; //! set of forbidden/background classes for the initial training std::set forbidden_classesTrain; //! set of forbidden/background classes for the whole process of learning over time std::set forbidden_classesActiveLearning; //! store the class numbers currently used std::set classesInUse; //! obviously, the number of classes used for training (i.e., classesInUse.size() ) int numberOfClasses; //! boolean whether to read the initial classifier from a file. If not, training will be performed bool read_classifier; //! boolean whether to save the final classifier or not bool save_classifier; //! The cached Data std::string cache; //! where to save the resulting images (uncertainty and classification results) std::string resultdir; //! current examples for most uncertain region Examples newTrainExamples; NICE::MultiChannelImageT m_CurrentImageFeatures; /////////////////////////////// // SEGMENTATION STUFF // /////////////////////////////// //! just store the name of our segmentation method. // Theoretically redundant, but currently makes things easier for store and restore... std::string s_rsMethode; //! low level Segmentation method RegionSegmentationMethod *regionSeg; //! boolean whether to reuse segmentation results for single images in different runs bool reuseSegmentation; //! contains filenames of images and indices of contained regions, that where already queried, to prevent them from being queried again std::map > queriedRegions; std::pair currentRegionToQuery; /////////////////////////////// // protected methods /////////////////////////////// inline void computeClassificationResults( const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByRandom( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByVariance( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByGPUncertainty ( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByGPMean ( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByGPMeanRatio ( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByGPWeightAll ( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); void computeNoveltyByGPWeightRatio ( NICE::FloatImage & noveltyImage, const NICE::MultiChannelImageT & feats, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities, const int & xsize, const int & ysize, const int & featdim ); public: /** * @brief default constructor * @author Alexander Freytag * @date 06-02-2014 ( dd-mm-yyyy ) */ SemSegNovelty ( ); /** * @brief recommended constructor * @author Alexander Freytag * @param conf needs a configfile * @param md and a MultiDataset (contains images and other things) */ SemSegNovelty ( const NICE::Config *conf, const MultiDataset *md ); /** simple destructor */ virtual ~SemSegNovelty(); void initFromConfig ( const NICE::Config * conf, const std::string _confSection = "SemSegNovelty" ); /** The trainingstep * @param md and a MultiDataset (contains images and other things) */ void train ( const MultiDataset *md ); /** The main procedure. Input: Image, Output: Segmented Image with pixelwise labeles and the probabilities * @param ce image data * @param segresult result of the semantic segmentation with a label for each pixel * @param probabilities multi-channel image with one channel for each class and corresponding probabilities for each pixel */ void semanticseg ( CachedExample *ce, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities ); /** * @brief visualize a specific region in the original image * * @param img input image * @param regions map of the regions * @param region visualize this region * @param outimage result * @return void **/ void visualizeRegion(const NICE::ColorImage &img, const NICE::Matrix ®ions, int region, NICE::ColorImage &outimage); /** * @brief Add a new example to the known training data * * @param newExample (NICE::Vector) the feature vector of the new examples * @param newClassNo (int) the corresponding GT class number * @return void **/ void addNewExample(const NICE::Vector & newExample, const int & newClassNo); /** * @brief Add those examples, which belong to the most novel region seen so far * * @return void **/ virtual void addNovelExamples(); /** * @brief Get a pointer to the examples extracted from the most novel region seen so far * * @return Examples * **/ virtual const Examples * getNovelExamples() const; ///////////////////// INTERFACE PERSISTENT ///////////////////// // interface specific methods for store and restore ///////////////////// INTERFACE PERSISTENT ///////////////////// /** * @brief Load active-segmentation-object from external file (stream) * @author Alexander Freytag */ virtual void restore ( std::istream & is, int format = 0 ); /** * @brief Save active-segmentation-object to external file (stream) * @author Alexander Freytag */ virtual void store( std::ostream & os, int format = 0 ) const; /** * @brief Clear active-segmentation-object object * @author Alexander Freytag */ virtual void clear (); }; } //namespace #endif //_NICE_SEMSEGNOVELTYINCLUDE