123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707 |
- #include "FeatureLearningPrototypes.h"
- #include <iostream>
- #include <core/image/FilterT.h>
- #include <core/image/CircleT.h>
- #include <core/image/Convert.h>
- #include <core/imagedisplay/ImageDisplay.h>
- #include <core/vector/VectorT.h>
- #include <vislearning/features/localfeatures/LFonHSG.h>
- #include <vislearning/features/localfeatures/LFColorSande.h>
- #include <vislearning/features/localfeatures/LocalFeatureColorWeijer.h>
- #include <vislearning/features/localfeatures/LFReadCache.h>
- #include <vislearning/features/localfeatures/LFWriteCache.h>
- #include <vislearning/math/cluster/KMeans.h>
- #include <vislearning/math/cluster/KMedian.h>
- #include <vislearning/math/cluster/GMM.h>
- using namespace std;
- using namespace NICE;
- using namespace OBJREC;
-
-
-
-
-
- void FeatureLearningPrototypes::setClusterAlgo( const std::string & _clusterAlgoString)
- {
-
- if (this->clusterAlgo != NULL)
- delete clusterAlgo;
-
- if (_clusterAlgoString.compare("kmeans") == 0)
- {
- this->clusterAlgo = new OBJREC::KMeans(this->initialNumberOfClusters);
- }
- else if (_clusterAlgoString.compare("kmedian") == 0)
- {
- this->clusterAlgo = new OBJREC::KMedian(this->initialNumberOfClusters);
- }
- else if (_clusterAlgoString.compare("GMM") == 0)
- {
- this->clusterAlgo = new OBJREC::GMM(this->conf, this->initialNumberOfClusters);
- }
- else
- {
- std::cerr << "Unknown cluster algorithm selected, use k-means instead" << std::endl;
- this->clusterAlgo = new OBJREC::KMeans(this->initialNumberOfClusters);
- }
- }
- void FeatureLearningPrototypes::setFeatureExtractor( const bool & _setForTraining )
- {
-
- if (this->featureExtractor != NULL)
- delete featureExtractor;
-
-
-
- std::string opSiftImpl;
- opSiftImpl = this->conf->gS ( "Descriptor", "implementation", "VANDESANDE" );
-
- bool readfeat;
- readfeat = this->conf->gB ( "Descriptor", "read", true );
-
- bool writefeat;
- writefeat = this->conf->gB ( "Descriptor", "write", true );
-
-
- LocalFeatureRepresentation *cSIFT = NULL;
- LocalFeatureRepresentation *writeFeats = NULL;
- LocalFeatureRepresentation *readFeats = NULL;
- this->featureExtractor = NULL;
- if ( opSiftImpl == "NICE" )
- {
- if ( _setForTraining )
- cSIFT = new OBJREC::LFonHSG ( this->conf, "HSGTrain" );
- else
- cSIFT = new OBJREC::LFonHSG ( this->conf, "HSGTest" );
- }
- else if ( opSiftImpl == "VANDESANDE" )
- {
- if ( _setForTraining )
- cSIFT = new OBJREC::LFColorSande ( this->conf, "LFColorSandeTrain" );
- else
- cSIFT = new OBJREC::LFColorSande ( this->conf, "LFColorSandeTest" );
- }
- else
- {
- fthrow ( Exception, "feattype: %s not yet supported" << opSiftImpl );
- }
- this->featureExtractor = cSIFT;
-
- if ( writefeat )
- {
-
- writeFeats = new LFWriteCache ( this->conf, cSIFT );
- this->featureExtractor = writeFeats;
- }
- if ( readfeat )
- {
-
- if ( writefeat )
- {
- readFeats = new LFReadCache ( this->conf, writeFeats, -1 );
- }
- else
- {
- readFeats = new LFReadCache ( this->conf, cSIFT, -1 );
- }
- this->featureExtractor = readFeats;
- }
-
-
- if ( cSIFT != NULL )
- cSIFT = NULL;
- if ( writeFeats != NULL )
- writeFeats = NULL;
- if ( readFeats != NULL )
- readFeats = NULL ;
- }
- void FeatureLearningPrototypes::extractFeaturesFromTrainingImages( const OBJREC::MultiDataset *_md, NICE::VVector & examplesTraining )
- {
- examplesTraining.clear();
-
- int numberOfTrainImage ( 0 );
-
- const LabeledSet *trainFiles = (*_md)["train"];
-
-
- LOOP_ALL_S( *trainFiles )
- {
- EACH_INFO( classno, info );
- std::string filename = info.img();
-
- NICE::ColorImage img( filename );
- if ( b_showTrainingImages )
- {
- showImage( img, "Input" );
- }
-
-
- NICE::VVector features;
- NICE::VVector cfeatures;
- NICE::VVector positions;
-
- Globals::setCurrentImgFN ( filename );
- if (featureExtractor == NULL)
- std::cerr << "feature Extractor is NULL" << std::endl;
- else
- featureExtractor->extractFeatures ( img, features, positions );
-
-
- for ( NICE::VVector::iterator i = features.begin();
- i != features.end();
- i++)
- {
-
- i->normalizeL1();
- examplesTraining.push_back(*i);
- }
-
-
- features.clear();
- positions.clear();
- numberOfTrainImage++;
- }
- }
- void FeatureLearningPrototypes::train ( const OBJREC::MultiDataset *_md )
- {
- bool loadSuccess = this->loadInitialCodebook();
-
- if ( !loadSuccess )
- {
-
-
-
-
-
-
- std::cerr << " EXTRACT FEATURES FROM TRAINING IMAGES" << std::endl;
-
- NICE::VVector examplesTraining;
- this->extractFeaturesFromTrainingImages( _md, examplesTraining );
-
-
-
-
-
-
-
-
- std::cerr << " CLUSTER FEATURES FROM TRAINING IMAGES" << std::endl;
-
- prototypes.clear();
- std::vector< double > weights;
- std::vector< int > assignment;
- clusterAlgo->cluster ( examplesTraining, prototypes, weights, assignment);
- weights.clear();
- assignment.clear();
-
-
- for (NICE::VVector::iterator protoIt = prototypes.begin(); protoIt != prototypes.end(); protoIt++)
- {
- protoIt->normalizeL1();
- }
- }
-
- this->writeInitialCodebook();
- }
- bool FeatureLearningPrototypes::loadInitialCodebook ( )
- {
- if ( b_loadInitialCodebook )
- {
- std::cerr << " INITIAL CODEBOOK ALREADY COMPUTED - RE-USE IT" << std::endl;
- std::cerr << " // WARNING - WE DO NOT VERIFY WHETHER THIS IS THE CORRECT CODEBOOK FOR THIS TRAINING SET!!!!" << std::endl;
-
- prototypes.clear();
-
- try
- {
- prototypes.read(cacheInitialCodebook);
- }
- catch (...)
- {
- std::cerr << "Error while loading initial codebook from " << cacheInitialCodebook << std::endl;
- return false;
- }
- return true;
- }
- else
- return false;
- }
- bool FeatureLearningPrototypes::writeInitialCodebook ( )
- {
- if ( b_saveInitialCodebook )
- {
- std::cerr << " SAVE INITIAL CODEBOOK " << std::endl;
-
- try
- {
- prototypes.write( cacheInitialCodebook );
- }
- catch (...)
- {
- std::cerr << "Error while saving initial codebook" << std::endl;
- return false;
- }
- return true;
- }
- else
- return false;
- }
- void FeatureLearningPrototypes::evaluateCurrentCodebookForGivenFeatures( const NICE::VVector & _features,
- const NICE::VVector & _positions,
- NICE::FloatImage & _noveltyImageGaussFiltered,
- NICE::FloatImage * _noveltyImage )
- {
- bool wasNoveltyImageGiven ( true );
- if ( _noveltyImage == NULL )
- {
- _noveltyImage = new FloatImage ( _noveltyImageGaussFiltered.width(), _noveltyImageGaussFiltered.height() );
- wasNoveltyImageGiven = false;
- }
-
- _noveltyImageGaussFiltered.set( 0.0 );
- _noveltyImage->set( 0.0 );
-
-
- NICE::VVector::const_iterator posIt = _positions.begin();
- for ( NICE::VVector::const_iterator i = _features.begin();
- i != _features.end();
- i++, posIt++)
- {
-
-
- double minDist ( std::numeric_limits<double>::max() );
- for (NICE::VVector::const_iterator it = prototypes.begin(); it != prototypes.end(); it++)
- {
-
- double tmpDist ( this->distFunction->calculate(*i,*it) );
- if (tmpDist < minDist)
- minDist = tmpDist;
- }
-
- (*_noveltyImage) ( (*posIt)[0], (*posIt)[1] ) = minDist;
- }
-
-
- float sigma ( 3.0 );
- FilterT<float, float, float> filter;
- filter.filterGaussSigmaApproximate ( *_noveltyImage, sigma, & _noveltyImageGaussFiltered );
-
- if ( ! wasNoveltyImageGiven )
- delete _noveltyImage;
- }
-
-
-
-
-
- FeatureLearningPrototypes::FeatureLearningPrototypes ( const Config *_conf,
- const MultiDataset *_md, const std::string & _section )
- : FeatureLearningGeneric ( _conf, _section )
- {
-
-
- initialNumberOfClusters = conf->gI(section, "initialNumberOfClusters", 10);
-
-
- std::string clusterAlgoString = conf->gS(section, "clusterAlgo", "kmeans");
-
-
- std::string distFunctionString = conf->gS(section, "distFunction", "euclidian");
-
-
-
-
-
-
-
-
-
-
-
-
- std::cerr << " SET UP VARIABLES AND METHODS " << std::endl;
-
- this->featureExtractor = NULL;
- this->setFeatureExtractor( true );
-
-
- this->clusterAlgo = NULL;
- this->setClusterAlgo( clusterAlgoString );
-
- if (distFunctionString.compare("euclidian") == 0)
- {
- distFunction = new NICE::EuclidianDistance<double>();
- }
- else
- {
- std::cerr << "Unknown vector distance selected, use euclidian instead" << std::endl;
- distFunction = new NICE::EuclidianDistance<double>();
- }
-
-
- this->train( _md );
-
-
- this->newImageCounter = 0;
-
-
- this->maxValForVisualization = conf->gD( section, "stupidMaxValForVisualization", 0.005 ) ;
-
-
-
- this->setFeatureExtractor( false );
- }
- FeatureLearningPrototypes::~FeatureLearningPrototypes()
- {
-
- if ( clusterAlgo != NULL )
- delete clusterAlgo;
- if ( distFunction != NULL )
- delete distFunction;
- if ( featureExtractor != NULL )
- delete featureExtractor;
- }
- NICE::FloatImage FeatureLearningPrototypes::evaluateCurrentCodebookByDistance ( const std::string & _filename , const bool & beforeComputingNewFeatures )
- {
- std::cerr << " VISUALIZATION ----- maxValForVisualization: " << maxValForVisualization << std::endl;
-
- NICE::ColorImage img( _filename );
- if ( b_showTrainingImages )
- {
- showImage( img, "Input" );
- }
-
- int xsize ( img.width() );
- int ysize ( img.height() );
-
-
- NICE::VVector features;
- NICE::VVector cfeatures;
- NICE::VVector positions;
-
- Globals::setCurrentImgFN ( _filename );
- featureExtractor->extractFeatures ( img, features, positions );
-
-
- for ( NICE::VVector::iterator i = features.begin();
- i != features.end();
- i++)
- {
-
- i->normalizeL1();
- }
-
- FloatImage noveltyImage ( xsize, ysize );
- FloatImage noveltyImageGaussFiltered ( xsize, ysize );
-
- this->evaluateCurrentCodebookForGivenFeatures( features, positions, noveltyImageGaussFiltered, &noveltyImage );
-
- double maxDist ( noveltyImage.max() );
- double maxFiltered ( noveltyImageGaussFiltered.max() );
- std::cerr << "maximum distance of Training images: " << maxDist << std::endl;
- std::cerr << "maximum distance of Training images after filtering: " << maxFiltered << std::endl;
- if ( beforeComputingNewFeatures )
- this->oldMaxDist = maxFiltered;
-
-
- NICE::ColorImage noveltyImageRGB ( xsize, ysize );
- if ( beforeComputingNewFeatures )
- {
- imageToPseudoColorWithRangeSpecification( noveltyImageGaussFiltered, noveltyImageRGB, 0 , maxValForVisualization );
- std::cerr << "set max value to: " << noveltyImageGaussFiltered.max() << std::endl;
- }
- else
- {
- imageToPseudoColorWithRangeSpecification( noveltyImageGaussFiltered, noveltyImageRGB, 0 , maxValForVisualization );
- std::cerr << "set max value to: " << this->oldMaxDist << std::endl;
- }
-
-
-
- if ( b_showResults )
- showImage(noveltyImageRGB, "Novelty Image");
- else
- {
- std::vector< std::string > list2;
- StringTools::split ( _filename, '/', list2 );
- std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter -1 ) + "_" + list2.back() + "_3_updatedNoveltyMap.ppm");
- if ( beforeComputingNewFeatures )
- destination = s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_0_initialNoveltyMap.ppm";
- noveltyImageRGB.writePPM( destination );
- }
-
-
- int tmpProtCnt ( 0 );
- for (NICE::VVector::const_iterator protIt = prototypes.begin(); protIt != prototypes.end(); protIt++, tmpProtCnt++)
- {
- double distToCurrentCluster ( std::numeric_limits<double>::max() );
- int indexOfMostSimFeat( 0 );
- double tmpDist;
- int featureCnt ( 0 );
-
- for ( NICE::VVector::iterator i = features.begin();
- i != features.end();
- i++, featureCnt++)
- {
- tmpDist = this->distFunction->calculate( *i, *protIt );
- if ( tmpDist < distToCurrentCluster )
- {
- distToCurrentCluster = tmpDist;
- indexOfMostSimFeat = featureCnt;
- }
- }
-
- int posX ( ( positions[indexOfMostSimFeat] ) [0] );
- int posY ( ( positions[indexOfMostSimFeat] ) [1] );
-
-
-
-
-
-
-
-
-
-
- size_t seg = ( size_t ) ( tmpProtCnt/(float)prototypes.size() * 6.0 );
- double y = ( 6 * tmpProtCnt/(float)prototypes.size() - seg );
- Color circleColor;
- switch ( seg ) {
- case 0:
- circleColor = Color( 0,0,(int)round(y*255) );
- break;
- case 1:
- circleColor = Color( 0,(int)round(y*255),255 );
- break;
- case 2:
- circleColor = Color( 0,255,(int)round((1-y)*255) );
- break;
- case 3:
- circleColor = Color( (int)round(y*255),255,0 );
- break;
- case 4:
- circleColor = Color( 255,(int)round((1-y)*255),0 );
- break;
- case 5:
- circleColor = Color( 255,(int)round(y*255),(int)round(y*255) );
- break;
- default:
- circleColor = Color( 255,255,255 );
- break;
- }
-
- NICE::Circle circ ( Coord( posX, posY), (int) round(2*3*sqrt(2)*( positions[indexOfMostSimFeat] )[2]) , circleColor );
- img.draw(circ);
- }
-
- if ( b_showResults )
- showImage(img, "Current image and most similar features for current prototypes");
- else
- {
- std::vector< std::string > list2;
- StringTools::split ( _filename, '/', list2 );
- std::string destination ( s_resultdir + NICE::intToString(this->newImageCounter-1) + "_" + list2.back() + "_3_updatedCurrentCluster.ppm");
- if ( beforeComputingNewFeatures )
- destination = s_resultdir + NICE::intToString(this->newImageCounter) + "_" + list2.back() + "_0_initialCurrentCluster.ppm";
-
- img.writePPM( destination );
- }
-
- return noveltyImageGaussFiltered;
- }
- NICE::ImageT< int > FeatureLearningPrototypes::evaluateCurrentCodebookByAssignments(const std::string& _filename, const bool& beforeComputingNewFeatures, const bool & _binaryShowLatestPrototype)
- {
- std::cerr << "evaluateCurrentCodebookByAssignments" << std::endl;
- NICE::ColorImage img( _filename );
-
- int xsize ( img.width() );
- int ysize ( img.height() );
-
-
- NICE::VVector features;
- NICE::VVector cfeatures;
- NICE::VVector positions;
-
- Globals::setCurrentImgFN ( _filename );
- featureExtractor->extractFeatures ( img, features, positions );
-
-
- for ( NICE::VVector::iterator i = features.begin();
- i != features.end();
- i++)
- {
-
- i->normalizeL1();
- }
-
-
- NICE::ImageT< int > clusterImage ( xsize, ysize );
- clusterImage.set ( 0 );
-
-
-
- NICE::Vector minDistances ( this->prototypes.size() );
-
- NICE::VVector::const_iterator posIt = positions.begin();
- for ( NICE::VVector::const_iterator i = features.begin();
- i != features.end();
- i++, posIt++ )
- {
-
-
- double minDist ( std::numeric_limits<double>::max() );
- int indexOfNearestPrototype ( 0 );
- int prototypeCounter ( 0 );
- for (NICE::VVector::const_iterator it = this->prototypes.begin(); it != this->prototypes.end(); it++, prototypeCounter++)
- {
-
- double tmpDist ( this->distFunction->calculate(*i,*it) );
-
- if (tmpDist < minDist)
- {
- minDist = tmpDist;
- indexOfNearestPrototype = prototypeCounter;
- }
-
-
- if ( minDistances[ prototypeCounter ] > tmpDist )
- minDistances[ prototypeCounter ] = tmpDist;
- }
-
-
-
-
-
-
- int noProtoTypes ( this->prototypes.size() -1 );
-
- for ( int tmpY = (*posIt)[1] - 1; tmpY < (*posIt)[1] + 1; tmpY++)
- {
- for ( int tmpX = (*posIt)[0] - 1; tmpX < (*posIt)[0] + 1; tmpX++)
- {
- if ( _binaryShowLatestPrototype )
- {
-
- if ( indexOfNearestPrototype == noProtoTypes)
- clusterImage ( tmpX, tmpY ) = 1;
- else
- clusterImage ( tmpX, tmpY ) = 0;
- }
- else
-
- clusterImage ( tmpX, tmpY ) = indexOfNearestPrototype;
- }
- }
- }
-
- std::cerr << "Codebook evaluation by assignments... min distances in image for every prototype: " << std::endl << " " << minDistances << std::endl;
-
-
- if ( !_binaryShowLatestPrototype )
- {
- int tmpCnt ( 0 );
- for (NICE::VVector::const_iterator protoIt = prototypes.begin(); protoIt != prototypes.end(); protoIt++, tmpCnt++)
- {
- for ( int tmpY = 1 + 2 - 2; tmpY < (2 + 2); tmpY++)
- {
- for ( int tmpX = 1 + 4*tmpCnt ; tmpX < (1 + 4*tmpCnt + 3); tmpX++)
- {
-
- clusterImage ( tmpX, tmpY ) = (Ipp8u) tmpCnt;
- }
- }
- }
- }
-
- return clusterImage;
- }
- void FeatureLearningPrototypes::evaluateCurrentCodebookByConfusionMatrix( NICE::Matrix & _confusionMat )
- {
- _confusionMat.resize ( this->prototypes.size(), this->prototypes.size() );
- _confusionMat.set( 0.0 );
-
- double tmpDist ( 0.0 );
- NICE::VVector::const_iterator protoItJ = prototypes.begin();
- for ( int j = 0; j < prototypes.size(); j++, protoItJ++)
- {
- NICE::VVector::const_iterator protoItI = protoItJ;
- for ( int i = j; i < prototypes.size(); i++, protoItI++)
- {
- tmpDist = this->distFunction->calculate( *protoItJ, *protoItI );
-
-
- _confusionMat ( i, j ) = tmpDist;
- }
- }
- }
- NICE::VVector * FeatureLearningPrototypes::getCurrentCodebook()
- {
- return &(this->prototypes);
- }
|