/** * @file LocalizationResult.cpp * @brief Localization result, what else? * @author Erik Rodner * @date 02/13/2008 */ #include "core/image/ImageT.h" #include "core/vector/VectorT.h" #include "core/vector/MatrixT.h" #include #include #include "vislearning/cbaselib/LocalizationResult.h" #include "ImageInfo.h" #include "core/basics/StringTools.h" // use this macro to show labeled images #undef DEBUG_LOCALIZATIONREAD #ifdef DEBUG_LOCALIZATIONREAD #include #endif using namespace OBJREC; using namespace std; using namespace NICE; /******** SingleLocalizationResult ********/ SingleLocalizationResult::SingleLocalizationResult ( ClassificationResult *_r, const NICE::Region & _reg, int _controlPoints ) : controlPoints(_controlPoints), hasRegionInformation_bool(true), reg(_reg), r(_r) { objectid = -1; reg.getRect(xi,yi,xa,ya); } SingleLocalizationResult::SingleLocalizationResult ( ClassificationResult *_r, int _xi, int _yi, int _xa, int _ya ) : controlPoints(4), xi(_xi), yi(_yi), xa(_xa), ya(_ya), hasRegionInformation_bool(true), r(_r) { objectid = -1; // reg.add (xi,yi,xa,ya); // this might lead to problems...in general the current Region representation is awful ! } double SingleLocalizationResult::getBBOverlapMeasureMin ( const SingleLocalizationResult & y ) const { double measure = 0.0; int xxi, xyi, xxa, xya; int yxi, yyi, yxa, yya; getBoundingBox ( xxi, xyi, xxa, xya ); y.getBoundingBox ( yxi, yyi, yxa, yya ); int mxi = ( xxi > yxi ) ? xxi : yxi; int myi = ( xyi > yyi ) ? xyi : yyi; int mxa = ( xxa < yxa ) ? xxa : yxa; int mya = ( xya < yya ) ? xya : yya; int iw = mxa - mxi + 1; int ih = mya - myi + 1; if ( (iw > 0) && (ih > 0) ) { // if iw>0 & ih>0 double A = (xxa - xxi + 1)*(xya - xyi + 1); double B = (yxa - yxi + 1)*(yya - yyi + 1); double overlap = A < B ? A : B; measure = iw*ih / overlap; } return measure; } double SingleLocalizationResult::getBBOverlapMeasure ( const SingleLocalizationResult & y ) const { double measure = 0.0; int xxi, xyi, xxa, xya; int yxi, yyi, yxa, yya; getBoundingBox ( xxi, xyi, xxa, xya ); y.getBoundingBox ( yxi, yyi, yxa, yya ); int mxi = ( xxi > yxi ) ? xxi : yxi; int myi = ( xyi > yyi ) ? xyi : yyi; int mxa = ( xxa < yxa ) ? xxa : yxa; int mya = ( xya < yya ) ? xya : yya; int iw = mxa - mxi + 1; int ih = mya - myi + 1; if ( (iw > 0) && (ih > 0) ) { // if iw>0 & ih>0 double overlap = (xxa - xxi + 1)*(xya - xyi + 1) + (yxa - yxi + 1)*(yya - yyi + 1) - iw*ih; measure = iw*ih / overlap; } return measure; } void SingleLocalizationResult::getBoundingBox ( int & _xi, int & _yi, int & _xa, int & _ya ) const { _xi = xi; _yi = yi; _xa = xa; _ya = ya; } void SingleLocalizationResult::getBoundingBox ( RectT & rectangle ) const { rectangle = RectT ( CoordT ( xi, yi ), CoordT ( xa, ya ) ); } void SingleLocalizationResult::getCentroid ( double & x, double & y ) const { reg.getCentroid ( x, y ); } SingleLocalizationResult::~SingleLocalizationResult () { if ( r != NULL ) delete r; } /******** LocalizationResult *********/ LocalizationResult::LocalizationResult ( int xsize, int ysize ) : cn(NULL) { hasLabeledImage = false; this->xsize = xsize; this->ysize = ysize; } LocalizationResult::LocalizationResult ( const ClassNames *_cn, int xsize, int ysize ) : cn(_cn) { hasLabeledImage = false; this->xsize = xsize; this->ysize = ysize; } LocalizationResult::~LocalizationResult () { for ( iterator k = begin(); k != end() ; k++ ) { SingleLocalizationResult *slr = *k; delete slr; } } LocalizationResult::LocalizationResult ( const ClassNames *_cn, const NICE::Image & img, int classno ) : cn(_cn) { // FIXME: just a bad predefined threshold ! const int t = 200; NICE::Region reg; #ifdef DEBUG_LOCALIZATIONREAD NICE::Image imgo (img); imgo.set(0); #endif this->xsize = img.width(); this->ysize = img.height(); for ( int y = 0 ; y < img.height(); y++ ) for ( int x = 0 ; x < img.width(); x++ ) { if ( img.getPixel(x,y) < t ) { #ifdef DEBUG_LOCALIZATIONREAD imgo.setPixel(x,y,1); #endif reg.add ( x, y ); } } #ifdef DEBUG_LOCALIZATIONREAD NICE::showImageOverlay ( imgo, imgo ); #endif ClassificationResult *r = new ClassificationResult (classno, 1.0, _cn->getMaxClassno()); push_back ( new SingleLocalizationResult ( r, reg ) ); hasLabeledImage = false; } LocalizationResult::LocalizationResult ( const ClassNames *_cn, const NICE::ColorImage & img) : cn(_cn) { map regions; xsize = img.width(); ysize = img.height(); #ifdef DEBUG_LOCALIZATIONREAD NICE::showImage ( img ); NICE::Image imgo (xsize,ysize); imgo.set(0); #endif for ( int y = 0 ; y < ysize ; y++ ) { int xstart = 0; // RGB values of the current pixel int r = img.getPixel(0,y,0); int g = img.getPixel(0,y,1); int b = img.getPixel(0,y,2); for ( int x = 0 ; x < xsize ; x++ ) { int r_next, g_next, b_next; if ( x != xsize - 1 ) { r_next = img.getPixel(x,y,0); g_next = img.getPixel(x,y,1); b_next = img.getPixel(x,y,2); } else { // at the border of the image, we should // always have a color change to add the last // line segment r_next = -1; g_next = -1; b_next = -1; } // now the RGB color changes and we have an object boundary // therefore we have to add a line segment if ( r != r_next || g != g_next || b != b_next ) { int classno; // look up class number for the label color _cn->getClassnoFromColor ( classno, r, g, b ); if ( classno >= 0 ) { // add line segment as an rectangular region regions[classno].add( xstart, y, x, y ); #ifdef DEBUG_LOCALIZATIONREAD for ( int z = xstart ; z <= x ; z++ ) imgo.setPixel(z,y,classno); #endif xstart = x+1; } } r = r_next; g = g_next; b = b_next; } } #ifdef DEBUG_LOCALIZATIONREAD showImageOverlay(imgo, imgo); #endif for ( map::const_iterator j = regions.begin(); j != regions.end(); j++ ) { int classno = j->first; ClassificationResult *r = new ClassificationResult (classno, 1.0, _cn->getMaxClassno()); push_back ( new SingleLocalizationResult ( r, j->second ) ); } hasLabeledImage = false; } void LocalizationResult::restore (istream & is, int format) { if ( format == FILEFORMAT_PASCAL2006_RESULT ) { while ( ! is.eof() ) { double score; int xi, yi, xa, ya; // refactor-nice.pl: check this substitution // old: string classname; std::string classname; if ( ! (is >> classname) ) break; if ( ! (is >> score) ) break; if ( ! (is >> xi) ) break; if ( ! (is >> yi) ) break; if ( ! (is >> xa) ) break; if ( ! (is >> ya) ) break; ClassificationResult *r = new ClassificationResult ( cn->classno(classname), score, cn->getMaxClassno() ); SingleLocalizationResult *sr = new SingleLocalizationResult ( r, xi, yi, xa, ya ); push_back ( sr ); } } else if ( format == FILEFORMAT_PASCAL2006_GROUNDTRUTH ) { #if 0 /* # Details for object 1 ("PAScat") Original label for object 1 "PAScat" : "PAScat" Bounding box for object 1 "PAScat" (Xmin, Ymin) - (Xmax, Ymax) : (11, 135) - (333, 410) */ // refactor-nice.pl: check this substitution // old: string word; std::string word; while ( ! is.eof() ) { if ( ! (is >> word) ) break; if ( word != "Bounding" ) continue; char line[1024]; is.getline (line, 1024); vector submatches; bool result = StringTools::regexMatch ( line, "box for object ([:digit]+) \"([:alpha:]+)\" (Xmin, Ymin) - (Xmax, Ymax) : (([:digit:]+) *, *([:digit:]+)) *: *(([:digit:]+) *, *([:digit:]+))", submatches ); cerr << "string: " << line << endl; for ( vector::const_iterator i = submatches.begin(); i != submatches.end(); i++ ) cerr << "submatch " << *i << endl; exit(-1); } #endif } else if ( format == FILEFORMAT_POLYGON ) { // This is limited to bounding boxes ...sorry while (! is.eof()) { #define USE_CALTECH101_POLYGON_FORMAT #ifdef USE_CALTECH101_POLYGON_FORMAT std::string filename; if ( !(is >> filename) ) break; #endif std::string classname; if ( !(is >> classname) ) break; const double score = 1.0; int classno = cn->classnoFromText(classname); uint polygon_points; if ( !(is >> polygon_points) ) break; int xi = std::numeric_limits::max(); int xa = - std::numeric_limits::max(); int yi = std::numeric_limits::max(); int ya = - std::numeric_limits::max(); for ( uint i = 0 ; i < polygon_points ; i++ ) { double x,y; if ( !(is >> x) ) break; if ( !(is >> y) ) break; if ( x < xi ) xi = x; if ( x > xa ) xa = x; if ( y < yi ) yi = y; if ( y > ya ) ya = y; } if ( classno >= 0 ) { ClassificationResult *r = new ClassificationResult ( classno, score, cn->getMaxClassno() ); SingleLocalizationResult *sr = new SingleLocalizationResult ( r, xi, yi, xa, ya ); push_back ( sr ); } } //sortEmpricalDepth(); } else if ( format == FILEFORMAT_POLYGON_SIFTFLOW ) { // parser for xml annotations of SIFTFlow dataset if ( is.good() ) { std::string tmp; is >> tmp; // annotation tag bool b_endOfBlock = false; while ( !b_endOfBlock ) { is >> tmp; // get current line // reached end of file properly if ( this->isEndTag ( tmp, "annotation") ) { b_endOfBlock = true; continue; } StringTools::normalize_string( tmp ); tmp = this->removeStartTag ( tmp ); // found new single localization result if ( tmp.compare("object") == 0 ) { std::string classname; is >> classname; classname = classname.substr( 6, classname.length()-13 ); //remove tags int classno = cn->classnoFromText(classname); bool foundPolygonBlock = false; while ( !foundPolygonBlock ) { is >> tmp; StringTools::normalize_string( tmp ); tmp = this->removeStartTag ( tmp ); if ( tmp.compare("polygon") == 0 ) foundPolygonBlock = true; } is >> tmp; // 'username' line NICE::Region newPolygon; bool endOfPolyBlock = false; while ( !endOfPolyBlock ) { is >> tmp; // or ? if ( this->isEndTag ( tmp, "polygon" ) ) { endOfPolyBlock = true; continue; } int x, y; is >> tmp; // ... StringTools::normalize_string( tmp ); tmp = tmp.substr( 3, tmp.length()-7 ); //remove tags x = atoi ( tmp.c_str() ); is >> tmp; // ... StringTools::normalize_string( tmp ); tmp = tmp.substr( 3, tmp.length()-7 ); //remove tags y = atoi ( tmp.c_str() ); newPolygon.add( x, y ); is >> tmp; // } if ( classno >= 0 ) { ClassificationResult *r = new ClassificationResult ( classno, 1.0, cn->getMaxClassno() ); SingleLocalizationResult *sr = new SingleLocalizationResult ( r, newPolygon ); push_back ( sr ); } } } } else { fthrow(IOException, "LocalizationResult::restore: InStream not initialized !"); } } else { fthrow(IOException, "LocalizationResult::restore: file format not yet supported !"); } } void LocalizationResult::loadImageInfo(std::string sFilename, int selectObjectWithUniqueId) { ImageInfo info; info.loadImageInfo(sFilename); this->loadImageInfo(info, selectObjectWithUniqueId); } void LocalizationResult::loadImageInfo(ImageInfo &p_ImageInfo, int selectObjectWithUniqueId) { try { const std::list< OBJREC::BoundingBox > *listBBoxes = p_ImageInfo.bboxes(); const double score = 1.0; OBJREC::BoundingBox box; std::list< OBJREC::BoundingBox >::const_iterator itBBoxes = listBBoxes->begin(); for(;itBBoxes != listBBoxes->end(); itBBoxes++) { box = *itBBoxes; int id = box.id(); if( selectObjectWithUniqueId != -1 && selectObjectWithUniqueId != box.unique_id_ ) //only extract bounding boxes with a specific unique id. for why, see @ continue; std::stringstream ss; ss << id; std::string classname = ss.str(); int classno = cn->classno(classname); if(classno == -1) { fprintf (stderr, "LocalizationResult::loadImageInfo: no classno found for classname %s (using classno=-1)\n", classname.c_str()); } ClassificationResult *r = new ClassificationResult ( classno, score, cn->getMaxClassno() ); r->classname = cn->text( classno ); SingleLocalizationResult *sr = new SingleLocalizationResult ( r, box.topLeft().x, box.topLeft().y, box.width(), box.height() ); sr->objectid = box.unique_id_; this->push_back ( sr ); } } catch(Exception e) { fthrow( Exception, "LocalizationResult::loadImageInfo: error loading image info (ImageLabeler xml format)"); } } void LocalizationResult::store (ostream & os, int format) const { if ( format == FILEFORMAT_PASCAL2006_RESULT ) { for ( const_iterator i = begin(); i != end(); i++ ) { const SingleLocalizationResult *sr = *i; const ClassificationResult *r = sr->r; int classno = r->classno; double score = r->scores.get(classno); int xi, yi, xa, ya; sr->getBoundingBox ( xi, yi, xa, ya ); os << cn->text(r->classno) << " " << score << " " << xi << " " << yi << " " << xa << " " << ya << " " << endl; } } else { fprintf (stderr, "LocalizationResult::store: file format not yet supported !\n"); exit(-1); } } void LocalizationResult::clear () { for ( iterator k = begin(); k != end() ; k++ ) { SingleLocalizationResult *slr = *k; delete slr; } vector::clear(); hasLabeledImage = false; } /** returns whether the depth of x is smaller than that of y !!! no transitivity !!! */ bool depthCompare ( const SingleLocalizationResult *x, const SingleLocalizationResult *y ) { /** According to the LabelMe paper of Torralba et al., Murphy */ const NICE::Region & rx = x->getRegion(); const NICE::Region & ry = y->getRegion(); NICE::Region intersect; intersect.setIntersection ( rx, ry ); int ax = rx.size(); int ay = ry.size(); int is = intersect.size(); if ( is == 0 ) { int nx = x->getControlPoints(); int ny = y->getControlPoints(); return ( nx > ny ); } else { double ratx = (double)is / ax; double raty = (double)is / ay; return ( ratx > raty ); } } bool confidenceCompare ( const SingleLocalizationResult *x, const SingleLocalizationResult *y ) { return ( x->r->confidence() > y->r->confidence() ); } void LocalizationResult::sortDescendingConfidence() { sort ( begin(), end(), confidenceCompare ); } void LocalizationResult::sortEmpricalDepth() { sort ( begin(), end(), depthCompare ); } void LocalizationResult::calcLabeledImage ( NICE::ImageT & mark, int backgroundClassNo ) const { mark.set(backgroundClassNo); fprintf (stderr, "LocalizationResult: calcLabeledImage %zd\n", size() ); for ( int y = 0 ; y < mark.height(); y++ ) for ( int x = 0 ; x < mark.width(); x++ ) for ( LocalizationResult::const_iterator k = begin(); k != end() ; k++ ) { SingleLocalizationResult *slr = *k; const NICE::Region & r = slr->getRegion(); if ( r.inside(x,y) ) { mark.setPixel(x,y,slr->r->classno); break; } } } void LocalizationResult::calcLabeledImage ( NICE::Image & mark, int backgroundClassNo ) const { NICE::ImageT markInt; markInt.resize ( mark.width(), mark.height() ); calcLabeledImage( markInt, backgroundClassNo ); for ( int y = 0; y < markInt.height(); y++ ) for ( int x = 0; x < markInt.width(); x++ ) { int cLabel = markInt.getPixelQuick( x, y ); if ( cLabel > std::numeric_limits::max() ) std::cerr << "LocalizationResult::calcLabeledImage: To many classes! Labeled image with UCHAR is not sufficient!" << std::endl; mark.setPixelQuick( x, y, (unsigned char) cLabel ); } } void LocalizationResult::getLabeledImageCache ( NICE::Image & mark ) const { assert ( hasLabeledImage ); labeledImage->copyFrom ( mark ); } void LocalizationResult::setMap ( const NICE::Image & _labeledImage ) { labeledImage = new Image( _labeledImage ); hasLabeledImage = true; } void drawOrthoLine ( NICE::ColorImage & img, int x1, int y1, int x2, int y2, int width, int sign, int r, int g, int b ) { int xi = x1; int yi = y1; int xa = x2; int ya = y2; for ( int i = 0 ; i < width; i++ ) { if ( yi == ya ) { yi = yi + sign; ya = ya + sign; } else if ( xi == xa ) { xi = xi + sign; xa = xa + sign; } else { assert ( 0 == 1 ); } if ( (xi>=0) && (yi>=0) && (xa<=img.width()) && (ya<=img.height()) ) { NICE::Line l ( Coord(xi, yi), Coord(xa, ya) ); img.draw( l, NICE::Color(r, g, b) ); } } } void LocalizationResult::displayBoxes ( NICE::ColorImage & img, const ClassNames *cn, bool display_confidence, bool invert, int width ) const { for ( LocalizationResult::const_iterator k = begin(); k != end() ; k++ ) { SingleLocalizationResult *slr = *k; int xi, yi, xa, ya; slr->getBoundingBox ( xi, yi, xa, ya ); int classno = (slr->r == NULL ) ? 0 : slr->r->classno; int r,g,b; if ( cn != NULL ) { cn->getRGBColor ( classno, r, g, b ); } else { r = 255; g = 0; b = 0; } if ( invert ) { r = 255 - r; g = 255 - g; b = 255 - b; } if ( display_confidence && (cn != NULL)) { std::string name = cn->text(classno); char caption[1024]; sprintf ( caption, "%3.2lf %s", slr->r->confidence(), name.c_str() ); // refactor-nice.pl: check this substitution // old: Text(caption, xi, yi, r, 0, img.RedImage()); // REFACTOR-FIXME Unable to map this statement // refactor-nice.pl: check this substitution // old: Text(caption, xi, yi, g, 0, img.GreenImage()); // REFACTOR-FIXME Unable to map this statement // refactor-nice.pl: check this substitution // old: Text(caption, xi, yi, b, 0, img.BlueImage()); // REFACTOR-FIXME Unable to map this statement } drawOrthoLine ( img, xi-width, yi, xa+width, yi, width, -1, r, g, b ); drawOrthoLine ( img, xa, yi, xa, ya, width, +1, r, g, b ); drawOrthoLine ( img, xa+width, ya, xi-width, ya, width, +1, r, g, b ); drawOrthoLine ( img, xi, ya, xi, yi, width, -1, r, g, b ); } }