/** * @file LocalFeatureSift.cpp * @brief local feature with sift * @author Erik Rodner, Alexander Freytag * @date 03/10/2012 */ #include #include "vislearning/features/localfeatures/sift.h" #include "vislearning/features/localfeatures/LocalFeatureSift.h" using namespace OBJREC; using namespace std; using namespace NICE; ///////////////////// ///////////////////// ///////////////////// // CONSTRUCTORS / DESTRUCTORS ///////////////////// ///////////////////// ///////////////////// LocalFeatureSift::LocalFeatureSift() : LocalFeature () { this->octaves = 6; this->levels = 3; this->first_octave = -1; this->normalizeFeature = true; this->magnif = 3; this->deletemode = true; this->integerValues = true; this->usegpu = false; } LocalFeatureSift::LocalFeatureSift( const NICE::Config * _conf ) { this->initFromConfig( _conf ); } LocalFeatureSift::~LocalFeatureSift() { } void OBJREC::LocalFeatureSift::initFromConfig( const NICE::Config* _conf, const std::string& _confSection ) { //NOTE previous confSection defaultet to LFSiftPP! this->octaves = _conf->gI(_confSection, "octaves", 6); this->levels = _conf->gI(_confSection, "levels", 3); this->first_octave = _conf->gI(_confSection, "first_octave", -1); this->normalizeFeature = _conf->gB(_confSection, "normalize_feature", true ); this->magnif = _conf->gD(_confSection, "magnif", 3 ); this->deletemode = _conf->gB(_confSection, "deletemode", true ); this->integerValues = _conf->gB(_confSection, "integer_values", true ); #ifdef NICE_USELIB_CUDASIFT this->usegpu = _conf->gB(_confSection, "use_siftgpu", false ); #else this->usegpu = false; #endif } ///////////////////// ///////////////////// ///////////////////// // FEATURE STUFF ///////////////////// ///////////////////// ////////////////// void LocalFeatureSift::sortPositions(VVector & positions) const { // < Key , Val > map scales; for ( vector< NICE::Vector >::iterator i = positions.begin(); i != positions.end();i++) { const NICE::Vector & pos = *i; scales[pos[2]] = true; } VVector newpositions; map::iterator iter; for ( iter = scales.begin(); iter != scales.end(); ++iter ) { for ( vector< NICE::Vector >::iterator i = positions.begin(); i != positions.end();i++) { const NICE::Vector & pos = *i; if (pos[2] == iter->first) { newpositions.push_back(pos); } } } positions = newpositions; } void LocalFeatureSift::computeDesc( const NICE::Image & img, VVector & positions, VVector & descriptors ) const { if ( usegpu ) { try { #ifdef NICE_USELIB_CUDASIFT withGPU( img, positions, descriptors ); #endif } catch ( runtime_error& rte ) { cerr << "[err] LocalFeatureSift: " << rte.what() << endl; clog << "[log] use SIFTPP implementation:" << endl; withPP( img, positions, descriptors ); } } else { withPP( img, positions, descriptors ); } } int LocalFeatureSift::getDescriptors ( const NICE::Image & img, VVector & positions, VVector & descriptors ) const { sortPositions(positions); computeDesc(img, positions, descriptors); return 0; } void LocalFeatureSift::visualizeFeatures ( NICE::Image & mark, const VVector & positions, size_t color ) const { /* TODO: switch to NICE instead of ICE ice::Image mark_ice = ice::NewImg ( mark.width(), mark.height(), 255 ); for ( size_t k = 0 ; k < positions.size() ; k++ ) { const NICE::Vector & pos = positions[k]; ice::Matrix points ( 0, 2 ); const int size = 6; points.Append ( ice::Vector(-size, -size) ); points.Append ( ice::Vector(-size, size) ); points.Append ( ice::Vector(size, size) ); points.Append ( ice::Vector(size, -size) ); ice::Trafo tr; tr.Scale ( 0, 0, pos[2] ); tr.Rotate ( 0, 0, pos[3] ); tr.Shift ( pos[0], pos[1] ); ice::TransformList(tr, points); for ( int j = 0 ; j < points.rows(); j++ ) { if (points[j][0] < 0 ) points[j][0] = 0; if (points[j][0] >= mark_ice->xsize) points[j][0] = mark_ice->xsize - 1; if (points[j][1] < 0 ) points[j][1] = 0; if (points[j][1] >= mark_ice->ysize) points[j][1] = mark_ice->ysize - 1; } ice::DrawPolygon ( points, color, mark_ice ); } for ( unsigned int y = 0 ; y < mark.height(); y++ ) for ( unsigned int x = 0 ; x < mark.width(); x++ ) mark.setPixel(x,y, GetVal(mark_ice,x,y));*/ } void LocalFeatureSift::withPP( const NICE::Image & img, VVector & positions, VVector & descriptors ) const { int O = octaves ; int const S = levels ; int const omin = first_octave; float const sigman = .5 ; //.5 float const sigma0 = 1.6 * powf(2.0f, 1.0f / S) ; if (O < 1) { O = std::max(int(std::floor(log2 (std::min(img.width(),img.height()))) - omin - 3), 1) ; } const unsigned char *blockimg = (unsigned char*) img.getPixelPointer(); float *blockimgfl = new float[img.width() * img.height()]; for ( int k = 0 ; k < img.width() * img.height() ; k++ ) blockimgfl[k] = blockimg[k]; VL::Sift sift( blockimgfl, img.width(), img.height(), sigman, sigma0, O, S, omin, -1, S+1) ; //calling sift.process(.) already done in VL::Sift constructor, so omit the following process function //sift.process ( blockimgfl, img.width(), img.height() ); sift.setMagnification ( magnif ); sift.setNormalizeDescriptor ( normalizeFeature ); const int descr_size = 128; VL::float_t *descr_pt = new VL::float_t [descr_size]; VL::float_t angles[4] ; NICE::Vector feature (descr_size); NICE::Vector pos ( 4 ); for ( vector< NICE::Vector >::iterator i = positions.begin(); i != positions.end();) { const NICE::Vector & pos = *i; double x = pos[0]; double y = pos[1]; assert(pos[0] < img.width()); assert(pos[1] < img.height()); double s = pos[2]; bool deleteFeature = false; VL::Sift::Keypoint kp = sift.getKeypoint (x,y,s); double angle = 0.0; if ( pos.size() < 4 ) { int nangles = sift.computeKeypointOrientations(angles, kp); if ( nangles > 0 ) { angle = angles[0]; } else { if (deletemode) deleteFeature = true; else angle = 0; } } else { angle = pos[3]; } if ( ! deleteFeature ) { sift.computeKeypointDescriptor ( descr_pt, kp, angle ); for ( int j = 0 ; j < descr_size ; j++ ) // Umwandlung in Integer moegl. feature[j] = (integerValues ? (int)(512*descr_pt[j]) : descr_pt[j]); descriptors.push_back ( feature ); i++; } else { i = positions.erase(i); } } delete [] blockimgfl; delete [] descr_pt; } #ifdef NICE_USELIB_CUDASIFT void LocalFeatureSift::withGPU(const NICE::Image& img, VVector& positions, VVector& descriptors) const { // fill the parameter for char* argv[] = { // First octave to detect DOG keypoints "-fo" , const_cast (StringTools::convertToString (first_octave).c_str()), // Maximum number of Octaves "-no" , const_cast (StringTools::convertToString (octaves).c_str()), // Number of DOG levels in an octave. "-d" , const_cast (StringTools::convertToString (levels).c_str()), // Write unnormalized descriptor if specified. const_cast (normalizeFeature ? "" : "-unn"), // Descriptor grid size factor (magnif ??) "-dw" , const_cast (StringTools::convertToString (magnif).c_str()), // verbose levels "-v", "0" }; int argc = sizeof (argv) / sizeof (char*); // sift Instanz SiftGPU sift; // give parameter to sift sift.ParseParam (argc, argv); // check, whether siftgpu is full supported int support = sift.CreateContextGL(); if( support != SiftGPU::SIFTGPU_FULL_SUPPORTED ) throw runtime_error( "SiftGPU-support is not given by your device."); // set keypoints const int numberOfKeypoints = positions.size(); SiftGPU::SiftKeypoint keys[numberOfKeypoints]; // copy the NICEKeypoints into SiftKeypoints for (int i = 0; i < numberOfKeypoints; i++) { keys[i].x = positions[i][0]; keys[i].y = positions[i][1]; keys[i].s = positions[i][2]; keys[i].o = positions[i][3]; } sift.SetKeypointList (numberOfKeypoints, keys); // run SIFT const int imageWidth = img.width(); const int imageHeight = img.height(); const unsigned char* rawImageData = img.getPixelPointer(); sift.RunSIFT (imageWidth, imageHeight, rawImageData, GL_LUMINANCE, GL_UNSIGNED_BYTE); // get descriptors const int descr_size = 128; const int numberOfDescriptors = sift.GetFeatureNum(); float desc[descr_size * numberOfDescriptors]; sift.GetFeatureVector (NULL, desc); Vector localDesc (descr_size); // copy the SiftDescriptors into NICEDescriptors for (int i = 0; i < numberOfDescriptors; i++) { for (int j = 0; j < descr_size; j++) { localDesc[j] = (integerValues ? (int) (512 * desc[i*descr_size+j]) : desc[i*descr_size+j]); } descriptors.push_back (localDesc); } } #endif ///////////////////// INTERFACE PERSISTENT ///////////////////// // interface specific methods for store and restore ///////////////////// INTERFACE PERSISTENT ///////////////////// void LocalFeatureSift::restore ( std::istream & is, int format ) { //delete everything we knew so far... this->clear(); if ( is.good() ) { std::string tmp; is >> tmp; //class name if ( ! this->isStartTag( tmp, "LocalFeatureSift" ) ) { std::cerr << " WARNING - attempt to restore LocalFeatureSift, but start flag " << tmp << " does not match! Aborting... " << std::endl; throw; } bool b_endOfBlock ( false ) ; while ( !b_endOfBlock ) { is >> tmp; // start of block if ( this->isEndTag( tmp, "LocalFeatureSift" ) ) { b_endOfBlock = true; continue; } tmp = this->removeStartTag ( tmp ); if ( tmp.compare("octaves") == 0 ) { is >> this->octaves; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("levels") == 0 ) { is >> this->levels; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("first_octave") == 0 ) { is >> this->first_octave; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("normalizeFeature") == 0 ) { is >> this->normalizeFeature; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("magnif") == 0 ) { is >> this->magnif; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("deletemode") == 0 ) { is >> this->deletemode; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("integerValues") == 0 ) { is >> this->integerValues; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("usegpu") == 0 ) { is >> this->usegpu; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else { std::cerr << "WARNING -- unexpected LocalFeatureSift object -- " << tmp << " -- for restoration... aborting" << std::endl; throw; } } } else { std::cerr << "LocalFeatureSift::restore -- InStream not initialized - restoring not possible!" << std::endl; throw; } } void LocalFeatureSift::store ( std::ostream & os, int format ) const { if (os.good()) { // show starting point os << this->createStartTag( "LocalFeatureSift" ) << std::endl; os << this->createStartTag( "octaves" ) << std::endl; os << this->octaves << std::endl; os << this->createEndTag( "octaves" ) << std::endl; os << this->createStartTag( "levels" ) << std::endl; os << this->levels << std::endl; os << this->createEndTag( "levels" ) << std::endl; os << this->createStartTag( "first_octave" ) << std::endl; os << this->first_octave << std::endl; os << this->createEndTag( "first_octave" ) << std::endl; os << this->createStartTag( "normalizeFeature" ) << std::endl; os << this->normalizeFeature << std::endl; os << this->createEndTag( "normalizeFeature" ) << std::endl; os << this->createStartTag( "magnif" ) << std::endl; os << this->magnif << std::endl; os << this->createEndTag( "magnif" ) << std::endl; os << this->createStartTag( "deletemode" ) << std::endl; os << this->deletemode << std::endl; os << this->createEndTag( "deletemode" ) << std::endl; os << this->createStartTag( "integerValues" ) << std::endl; os << this->integerValues << std::endl; os << this->createEndTag( "integerValues" ) << std::endl; os << this->createStartTag( "usegpu" ) << std::endl; os << this->usegpu << std::endl; os << this->createEndTag( "usegpu" ) << std::endl; // done os << this->createEndTag( "LocalFeatureSift" ) << std::endl; } else { std::cerr << "OutStream not initialized - storing not possible!" << std::endl; } } void LocalFeatureSift::clear () { }