#include "FeatureLearningRegionBased.h" //STL #include //core #include #include #include #include #include //vislearning #include using namespace std; using namespace NICE; using namespace OBJREC; //********************************************** // // PROTECTED METHODS // //********************************************** //********************************************** // // PUBLIC METHODS // //********************************************** FeatureLearningRegionBased::FeatureLearningRegionBased ( const Config *_conf, const MultiDataset *_md, const std::string & _section ) : FeatureLearningPrototypes ( _conf, _md, _section ) { //save and read segmentation results from files this->reuseSegmentation = conf->gB ( "FPCPixel", "reuseSegmentation", true ); // select your segmentation method here // currently, the following options are supported: "MarkovCluster", "GraphBased", "MeanShift", "SLIC" string rsMethode = conf->gS ( section, "segmentationMethod", "MeanShift" ); OBJREC::RegionSegmentationMethod *tmpRegionSeg = OBJREC::GenericRegionSegmentationMethodSelection::selectRegionSegmentationMethod(conf, rsMethode); if ( reuseSegmentation ) this->segmentationAlgo = new RSCache ( conf, tmpRegionSeg ); else this->segmentationAlgo = tmpRegionSeg; this->i_gridSize = conf->gI( "LFColorSandeTest" , "grid" , 5 ); } FeatureLearningRegionBased::~FeatureLearningRegionBased() { // clean-up } void FeatureLearningRegionBased::learnNewFeatures ( const std::string & _filename ) { NICE::ColorImage img( _filename ); int xsize ( img.width() ); int ysize ( img.height() ); //variables to store feature information NICE::VVector newFeatures; NICE::VVector cfeatures; NICE::VVector positions; //compute features std::cerr << " EXTRACT FEATURES FROM UNSEEN IMAGE" << std::endl; Globals::setCurrentImgFN ( _filename ); featureExtractor->extractFeatures ( img, newFeatures, positions ); //normalization :) for ( NICE::VVector::iterator i = newFeatures.begin(); i != newFeatures.end(); i++) { i->normalizeL1(); } //compute region segmentation std::cerr << " COMPUTE REGION SEGMENTATION" << std::endl; NICE::Matrix mask; int amountRegions = segmentationAlgo->segRegions ( img, mask ); //compute novelty scores on a feature level std::cerr << " COMPUTE NOVELTY SCORES ON A FEATURE LEVEL" << std::endl; FloatImage noveltyImageGaussFiltered ( xsize, ysize ); this->evaluateCurrentCodebookForGivenFeatures( newFeatures, positions, noveltyImageGaussFiltered ); // compute scores for every region std::cerr << " COMPUTE SCORES FOR EVERY REGION" << std::endl; std::vector regionNoveltyMeasure (amountRegions, 0.0); std::vector regionSize (amountRegions, 0); for ( int y = 0; y < ysize; y += i_gridSize) //y++) { for (int x = 0; x < xsize; x += i_gridSize) //x++) { int r = mask(x,y); regionSize[r]++; //count the amount of "novelty" for the corresponding region regionNoveltyMeasure[r] += noveltyImageGaussFiltered(x,y); } } //loop over all regions and compute averaged novelty scores //NOTE this might be unuseful, since lateron we combine novelty score and region size (e.g. by multiplying) // however, we do not want to settle the combination in adavance for(int r = 0; r < amountRegions; r++) { regionNoveltyMeasure[r] /= regionSize[r]; } //a new region should cover at least 3% of the image for not being penalized double d_minimalImageAmountForAcceptableRegions ( 0.03 ); int minimalImageAmountForAcceptableRegions ( round( d_minimalImageAmountForAcceptableRegions * (xsize/i_gridSize) * (ysize/i_gridSize) ) ); std::cerr << "minimalImageAmountForAcceptableRegions: " << minimalImageAmountForAcceptableRegions << std::endl; //compute final scores for all regions NICE::Vector regionScores ( amountRegions, 0.0 ); std::cerr << "used region sizes for computation: " << std::endl; for(int r = 0; r < amountRegions; r++) { regionScores[r] = std::min( regionSize[r], minimalImageAmountForAcceptableRegions ) * regionNoveltyMeasure[r]; std::cerr << " " << std::min( regionSize[r], minimalImageAmountForAcceptableRegions ); } std::cerr << std::endl << std::endl; int indexOfBestRegion ( regionScores.MaxIndex() ); //////////////////////////////////// // // VISUALIZE REGION SCORES // //////////////////////////////////// NICE::FloatImage regionScoreImage ( xsize, ysize ); NICE::FloatImage regionNoveltyImage ( xsize, ysize ); //contains novelty score averaged over region NICE::FloatImage regionRelevanceImage ( xsize, ysize ); //contains higher scores for larger regions (but with upper limit) regionScoreImage.set( 0.0 ); regionNoveltyImage.set( 0.0 ); regionRelevanceImage.set( 0.0 ); for ( int y = 0; y < ysize; y++) { for (int x = 0; x < xsize; x++) { int r = mask(x,y); regionNoveltyImage(x,y) = regionNoveltyMeasure[r]; regionRelevanceImage(x,y) = regionSize[r]; regionScoreImage(x,y) = regionScores[r]; } } std::cerr << "highest region score: " << regionScoreImage.max()<< " -- smallest region score: " << regionScoreImage.min() << std::endl; std::cerr << "highest region novelty score: " << regionNoveltyImage.max() << " -- smallest region novelty score: " << regionNoveltyImage.min() << std::endl; NICE::ColorImage regionScoreImageRGB ( xsize, ysize ); NICE::ColorImage regionNoveltyScoreImageRGB ( xsize, ysize ); NICE::ColorImage regionRelevanceScoreImageRGB ( xsize, ysize ); //TODO properly specify the maximum value for score visualization here :) imageToPseudoColorWithRangeSpecification( regionScoreImage, regionScoreImageRGB, 0 /* min */, 2.2 /* max */ ); imageToPseudoColorWithRangeSpecification( regionNoveltyImage, regionNoveltyScoreImageRGB, 0 /* min */, 0.012 /* max */ ); imageToPseudoColorWithRangeSpecification( regionRelevanceImage, regionRelevanceScoreImageRGB, 0 /* min */, minimalImageAmountForAcceptableRegions /* max */ ); if ( b_showResults ) showImage(regionScoreImageRGB, "Current (new) image with region scores"); else { std::vector< std::string > list2; StringTools::split ( _filename, '/', list2 ); //write novelty score image std::string destinationNovelty ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_1_regionNoveltyScores.ppm"); regionNoveltyScoreImageRGB.writePPM( destinationNovelty ); //write relevance score image std::string destinationRelevance ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_2_regionRelevanceScores.ppm"); regionRelevanceScoreImageRGB.writePPM( destinationRelevance ); //write image with final scores for region std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_1_3_regionScores.ppm"); regionScoreImageRGB.writePPM( destination ); } //compute representative for best region NICE::Vector representative ( newFeatures.begin()->size(), 0.0 ); // //first try: average feature vectors of the "best" region // NICE::VVector::const_iterator posIt = positions.begin(); // for ( NICE::VVector::const_iterator featIt = newFeatures.begin(); // featIt != newFeatures.end(); // featIt++, posIt++) // { // // //only considere features that actually belong to the best region // if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion ) // continue; // // representative += *featIt; // } // // //simple mean feature vector // representative /= regionSize[indexOfBestRegion]; // //normalization // representative.normalizeL1(); // //second try: simply take the first feature vector of the "best" region (although this one should lay on the border, and so on...) // NICE::VVector::const_iterator posIt = positions.begin(); // for ( NICE::VVector::const_iterator featIt = newFeatures.begin(); // featIt != newFeatures.end(); // featIt++, posIt++) // { // // //only considere features that actually belong to the best region // if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion ) // continue; // // representative = *featIt; // i_posXOfNewPrototype = (*posIt)[0]; // i_posYOfNewPrototype = (*posIt)[1]; // //break after the first positive feature // break; // } //third try: simply take the feature vector of the "best" region with largest novelty score within this region // ... (hopefully, this is no outlier wrt to this region...) double maxNovelty ( 0.0 ); NICE::VVector::const_iterator mostNovelFeature = newFeatures.begin() ; NICE::VVector::const_iterator posIt = positions.begin(); for ( NICE::VVector::const_iterator featIt = newFeatures.begin(); featIt != newFeatures.end(); featIt++, posIt++) { //only considere features that actually belong to the best region if ( mask( (*posIt)[0], (*posIt)[1] ) != indexOfBestRegion ) continue; //did we found a feature of the "best"region with larger novelty score then the current most novel one? if ( noveltyImageGaussFiltered( (*posIt)[0], (*posIt)[1] ) > maxNovelty ) { maxNovelty = noveltyImageGaussFiltered( (*posIt)[0], (*posIt)[1] ); mostNovelFeature = featIt; i_posXOfNewPrototype = (*posIt)[0]; i_posYOfNewPrototype = (*posIt)[1]; } } representative = *mostNovelFeature; std::cerr << " New representative: " << std::endl << representative << std::endl; //include the chosen information into the currently used prototypes prototypes.push_back( representative ); if ( b_evaluationWhileFeatureLearning ) { NICE::ColorImage imgTmp( _filename ); double distToNewCluster ( std::numeric_limits::max() ); int indexOfMostSimFeat( 0 ); double tmpDist; int tmpCnt ( 0 ); for ( NICE::VVector::iterator i = newFeatures.begin(); i != newFeatures.end(); i++, tmpCnt++) { tmpDist = this->distFunction->calculate( *i, representative ); if ( tmpDist < distToNewCluster ) { distToNewCluster = tmpDist; indexOfMostSimFeat = tmpCnt; } } std::cerr << "** minDist to new cluster: " < list2; StringTools::split ( _filename, '/', list2 ); std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_2_bestNewCluster.ppm"); imgTmp.writePPM( destination ); } } //this was a new image, so we increase our internal counter (this->newImageCounter)++; }