NICE_VisLearning/cbaselib/LocalizationResult.cpp

/** 
* @file LocalizationResult.cpp
* @brief Localization result, what else?
* @author Erik Rodner
* @date 02/13/2008

*/
#include "core/image/ImageT.h"
#include "core/vector/VectorT.h"
#include "core/vector/MatrixT.h"

#include <iostream>

#include <core/image/LineT.h>

#include "vislearning/cbaselib/LocalizationResult.h"
#include "ImageInfo.h"
#include "core/basics/StringTools.h"

// use this macro to show labeled images
#undef DEBUG_LOCALIZATIONREAD

#ifdef DEBUG_LOCALIZATIONREAD
#include <core/imagedisplay/ImageDisplay.h>
#endif

using namespace OBJREC;

using namespace std;
using namespace NICE;

/******** SingleLocalizationResult ********/

SingleLocalizationResult::SingleLocalizationResult ( ClassificationResult *_r, const NICE::Region & _reg, int _controlPoints ) 
    : controlPoints(_controlPoints), hasRegionInformation_bool(true), reg(_reg), r(_r)
{
    objectid = -1;
    reg.getRect(xi,yi,xa,ya);
}

SingleLocalizationResult::SingleLocalizationResult ( ClassificationResult *_r, int _xi, int _yi, int _xa, int _ya ) 
    : controlPoints(4), xi(_xi), yi(_yi), xa(_xa), ya(_ya), hasRegionInformation_bool(true), r(_r)
{
    objectid = -1;
//    reg.add (xi,yi,xa,ya);
//    this might lead to problems...in general the current Region representation is awful !
}

double SingleLocalizationResult::getBBOverlapMeasureMin ( const SingleLocalizationResult & y ) const
{
    double measure = 0.0;
    int xxi, xyi, xxa, xya;
    int yxi, yyi, yxa, yya;
    getBoundingBox ( xxi, xyi, xxa, xya );
    y.getBoundingBox ( yxi, yyi, yxa, yya );

    int mxi = ( xxi > yxi ) ? xxi : yxi;
    int myi = ( xyi > yyi ) ? xyi : yyi;
    int mxa = ( xxa < yxa ) ? xxa : yxa;
    int mya = ( xya < yya ) ? xya : yya;

    int iw = mxa - mxi + 1;
    int ih = mya - myi + 1;

    if ( (iw > 0) && (ih > 0) )
    {
        // if iw>0 & ih>0
	double A = (xxa - xxi + 1)*(xya - xyi + 1);
	double B = (yxa - yxi + 1)*(yya - yyi + 1);

	double overlap = A < B ? A : B;
	measure = iw*ih / overlap;
    }

    return measure;
}

double SingleLocalizationResult::getBBOverlapMeasure ( const SingleLocalizationResult & y ) const
{
    double measure = 0.0;
    int xxi, xyi, xxa, xya;
    int yxi, yyi, yxa, yya;
    getBoundingBox ( xxi, xyi, xxa, xya );
    y.getBoundingBox ( yxi, yyi, yxa, yya );

    int mxi = ( xxi > yxi ) ? xxi : yxi;
    int myi = ( xyi > yyi ) ? xyi : yyi;
    int mxa = ( xxa < yxa ) ? xxa : yxa;
    int mya = ( xya < yya ) ? xya : yya;

    int iw = mxa - mxi + 1;
    int ih = mya - myi + 1;

    if ( (iw > 0) && (ih > 0) )
    {
        // if iw>0 & ih>0
	double overlap = (xxa - xxi + 1)*(xya - xyi + 1) +
			 (yxa - yxi + 1)*(yya - yyi + 1) -
			 iw*ih;
	measure = iw*ih / overlap;
    }

    return measure;
}

void SingleLocalizationResult::getBoundingBox ( int & _xi, int & _yi, int & _xa, int & _ya ) const
{
    _xi = xi;
    _yi = yi;
    _xa = xa;
    _ya = ya;
}

void SingleLocalizationResult::getBoundingBox ( RectT<int> & rectangle ) const
{
	rectangle = RectT<int> ( CoordT<int> ( xi, yi ), CoordT<int> ( xa, ya ) );
}

void SingleLocalizationResult::getCentroid ( double & x, double & y ) const
{
    reg.getCentroid ( x, y );
}

SingleLocalizationResult::~SingleLocalizationResult ()
{
    if ( r != NULL )
        delete r;  
}

/******** LocalizationResult *********/

LocalizationResult::LocalizationResult ( int xsize, int ysize ) : cn(NULL)
{
    hasLabeledImage = false;
    this->xsize = xsize;
    this->ysize = ysize;
}

LocalizationResult::LocalizationResult ( const ClassNames *_cn, int xsize, int ysize ) : cn(_cn)
{
    hasLabeledImage = false;
    this->xsize = xsize;
    this->ysize = ysize;
}

LocalizationResult::~LocalizationResult ()
{
    for ( iterator k = begin(); k != end() ; k++ )
    {
	SingleLocalizationResult *slr = *k;
	delete slr;
    }
}
LocalizationResult::LocalizationResult ( const ClassNames *_cn, const NICE::Image & img, int classno ) : cn(_cn)
{
     // FIXME: just a bad predefined threshold !
     const int t = 200; 
     NICE::Region reg;

#ifdef DEBUG_LOCALIZATIONREAD
     NICE::Image imgo (img);
     imgo.set(0);
#endif

     this->xsize = img.width();
     this->ysize = img.height();

     for ( int y = 0 ; y < img.height(); y++ )
	 for ( int x = 0 ; x < img.width(); x++ )
	 {
	     if ( img.getPixel(x,y) < t )
	     {
#ifdef DEBUG_LOCALIZATIONREAD
		 imgo.setPixel(x,y,1);
#endif
		 reg.add ( x, y );
	     }
	 }

#ifdef DEBUG_LOCALIZATIONREAD
     NICE::showImageOverlay ( imgo, imgo );
#endif

     ClassificationResult *r = new ClassificationResult (classno, 1.0, _cn->getMaxClassno());
     push_back ( new SingleLocalizationResult ( r, reg ) );

     hasLabeledImage = false;
}

LocalizationResult::LocalizationResult ( const ClassNames *_cn, const NICE::ColorImage & img) : cn(_cn)
{
     map<int, NICE::Region> regions;

     xsize = img.width();
     ysize = img.height();

#ifdef DEBUG_LOCALIZATIONREAD
     NICE::showImage ( img );
     NICE::Image imgo (xsize,ysize);
     imgo.set(0);
#endif

    
     for ( int y = 0 ; y < ysize ; y++ )
     {
       int xstart = 0;   
       // RGB values of the current pixel
       int r = img.getPixel(0,y,0);
       int g = img.getPixel(0,y,1);
       int b = img.getPixel(0,y,2);

       for ( int x = 0 ; x < xsize ; x++ )
       {
         int r_next, g_next, b_next;
         if ( x != xsize - 1 ) {
           r_next = img.getPixel(x,y,0);
           g_next = img.getPixel(x,y,1);
           b_next = img.getPixel(x,y,2);
         } else {
           // at the border of the image, we should
           // always have a color change to add the last
           // line segment
           r_next = -1;
           g_next = -1;
           b_next = -1;
         }

         // now the RGB color changes and we have an object boundary
         // therefore we have to add a line segment
         if ( r != r_next || g != g_next || b != b_next )
         {
           int classno;
           // look up class number for the label color
           _cn->getClassnoFromColor ( classno, r, g, b );
           
           if ( classno >= 0 ) {
             // add line segment as an rectangular region
             regions[classno].add( xstart, y, x, y );
             
        #ifdef DEBUG_LOCALIZATIONREAD
             for ( int z = xstart ; z <= x ; z++ )
               imgo.setPixel(z,y,classno);
        #endif
             xstart = x+1;
           }
         }

        r = r_next;
        g = g_next;
        b = b_next;
       }
     }

     #ifdef DEBUG_LOCALIZATIONREAD
     showImageOverlay(imgo, imgo);
     #endif 

     for ( map<int, NICE::Region>::const_iterator j  = regions.begin();
					    j != regions.end();
					    j++ )
     {
    	 int classno = j->first;
    	 ClassificationResult *r = new ClassificationResult (classno, 1.0, _cn->getMaxClassno());
    	 push_back ( new SingleLocalizationResult ( r, j->second ) );
     }

     hasLabeledImage = false;
}

void LocalizationResult::restore (istream & is, int format)
{
    if ( format == FILEFORMAT_PASCAL2006_RESULT ) 
    {
	while ( ! is.eof() )
	{
	    double score;
	    int xi, yi, xa, ya;
	    // refactor-nice.pl: check this substitution
	    // old: string classname;
	    std::string classname;
	    if ( ! (is >> classname) ) break;
	    if ( ! (is >> score) ) break;
	    if ( ! (is >> xi) ) break;
	    if ( ! (is >> yi) ) break;
	    if ( ! (is >> xa) ) break;
	    if ( ! (is >> ya) ) break;
	    
	    ClassificationResult *r = new ClassificationResult ( cn->classno(classname), score, cn->getMaxClassno() ); 
	    SingleLocalizationResult *sr = new SingleLocalizationResult ( r, xi, yi, xa, ya );
	    push_back ( sr );
	}
    } else if ( format == FILEFORMAT_PASCAL2006_GROUNDTRUTH ) {
#if 0
	/* # Details for object 1 ("PAScat")
	   Original label for object 1 "PAScat" : "PAScat"
	   Bounding box for object 1 "PAScat" (Xmin, Ymin) - (Xmax, Ymax) : (11, 135) - (333, 410) */
	// refactor-nice.pl: check this substitution
	// old: string word;
	std::string word;

	while ( ! is.eof() )
	{
	    if ( ! (is >> word) ) break;
	    if ( word != "Bounding" ) continue;

	    char line[1024];
	    is.getline (line, 1024);

	    vector<string> submatches;
	    bool result = StringTools::regexMatch ( line, "box for object ([:digit]+) \"([:alpha:]+)\" (Xmin, Ymin) - (Xmax, Ymax) : (([:digit:]+) *, *([:digit:]+)) *: *(([:digit:]+) *, *([:digit:]+))", submatches ); 

	    cerr << "string: " << line << endl;
	    for ( vector<string>::const_iterator i = submatches.begin(); i != submatches.end(); i++ )
		cerr << "submatch " << *i << endl;
	    exit(-1);

	}
#endif
    } else if ( format == FILEFORMAT_POLYGON ) {
		// This is limited to bounding boxes ...sorry
		while (! is.eof()) {
#define USE_CALTECH101_POLYGON_FORMAT
#ifdef USE_CALTECH101_POLYGON_FORMAT
			std::string filename;
			if ( !(is >> filename) ) break;
#endif

			std::string classname;
			if ( !(is >> classname) ) break;

			const double score = 1.0;
			int classno = cn->classnoFromText(classname);

			uint polygon_points;
			if ( !(is >> polygon_points) ) break;

			int xi = std::numeric_limits<int>::max();
			int xa = - std::numeric_limits<int>::max();
			int yi = std::numeric_limits<int>::max();
			int ya = - std::numeric_limits<int>::max();
			for ( uint i = 0 ; i < polygon_points ; i++ )
			{
				double x,y;
				if ( !(is >> x) ) break;
				if ( !(is >> y) ) break;
				if ( x < xi ) xi = x;
				if ( x > xa ) xa = x;
				if ( y < yi ) yi = y;
				if ( y > ya ) ya = y;

			}
			if ( classno >= 0 ) {
				ClassificationResult *r = new ClassificationResult ( classno, score, cn->getMaxClassno() );
				SingleLocalizationResult *sr = new SingleLocalizationResult ( r, xi, yi, xa, ya );
				push_back ( sr );
			}
		}

		//sortEmpricalDepth();
    } else if ( format == FILEFORMAT_POLYGON_SIFTFLOW ) {
        // parser for xml annotations of SIFTFlow dataset
        if ( is.good() )
        {
            std::string tmp;
            is >> tmp;      // annotation tag

            bool b_endOfBlock = false;

            while ( !b_endOfBlock )
            {
                is >> tmp;  // get current line

                // reached end of file properly
                if ( this->isEndTag ( tmp, "annotation") )
                {
                    b_endOfBlock = true;
                    continue;
                }

                StringTools::normalize_string( tmp );
                tmp = this->removeStartTag ( tmp );

                // found new single localization result
                if ( tmp.compare("object") == 0 )
                {
                    std::string classname;
                    is >> classname;
                    classname = classname.substr( 6, classname.length()-13 );  //remove tags
                    int classno = cn->classnoFromText(classname);

                    bool foundPolygonBlock = false;

                    while ( !foundPolygonBlock )
                    {
                        is >> tmp;
                        StringTools::normalize_string( tmp );
                        tmp = this->removeStartTag ( tmp );

                        if ( tmp.compare("polygon") == 0 )
                            foundPolygonBlock = true;
                    }
                    is >> tmp;  // 'username' line

                    NICE::Region newPolygon;
                    bool endOfPolyBlock = false;

                    while ( !endOfPolyBlock )
                    {
                        is >> tmp;      // <pt> or </polygon> ?
                        if ( this->isEndTag ( tmp, "polygon" ) )
                        {
                            endOfPolyBlock = true;
                            continue;
                        }

                        int x, y;

                        is >> tmp;      // <x> ... </x>
                        StringTools::normalize_string( tmp );
                        tmp = tmp.substr( 3, tmp.length()-7 );  //remove tags
                        x = atoi ( tmp.c_str() );

                        is >> tmp;      // <y> ... </y>
                        StringTools::normalize_string( tmp );
                        tmp = tmp.substr( 3, tmp.length()-7 );  //remove tags
                        y = atoi ( tmp.c_str() );

                        newPolygon.add( x, y );

                        is >> tmp;      // </pt>
                    }
                    if ( classno >= 0 ) {
                        ClassificationResult *r = new ClassificationResult ( classno, 1.0, cn->getMaxClassno() );
                        SingleLocalizationResult *sr = new SingleLocalizationResult ( r, newPolygon );
                        push_back ( sr );
                    }
                }

            }
        }
        else
        {
            fthrow(IOException, "LocalizationResult::restore: InStream not initialized !");
        }
    }
    else {
		fthrow(IOException, "LocalizationResult::restore: file format not yet supported !");
    }
}

void LocalizationResult::loadImageInfo(std::string sFilename, int selectObjectWithUniqueId)
{
    ImageInfo info;
    info.loadImageInfo(sFilename);

    this->loadImageInfo(info, selectObjectWithUniqueId);
}

void LocalizationResult::loadImageInfo(ImageInfo &p_ImageInfo, int selectObjectWithUniqueId)
{

    try
    {

        const std::list< OBJREC::BoundingBox > *listBBoxes = p_ImageInfo.bboxes();

        const double score = 1.0;
        OBJREC::BoundingBox box;
        std::list< OBJREC::BoundingBox >::const_iterator itBBoxes = listBBoxes->begin();
        for(;itBBoxes != listBBoxes->end(); itBBoxes++)
        {
            box = *itBBoxes;
            int id = box.id();

            if( selectObjectWithUniqueId != -1 && selectObjectWithUniqueId != box.unique_id_ )
                //only extract bounding boxes with a specific unique id. for why, see @
                continue;

            std::stringstream ss;
            ss << id;

            std::string classname = ss.str();
            int classno = cn->classno(classname);
            if(classno == -1)
            {
                fprintf (stderr, "LocalizationResult::loadImageInfo: no classno found for classname %s (using classno=-1)\n", classname.c_str());
            }
            ClassificationResult *r = new ClassificationResult ( classno, score, cn->getMaxClassno() );
            r->classname = cn->text( classno );
            SingleLocalizationResult *sr = new SingleLocalizationResult ( r, box.topLeft().x,
                                                                             box.topLeft().y,
                                                                             box.width(),
                                                                             box.height() );
            sr->objectid = box.unique_id_;
            this->push_back ( sr );
        }

    }
    catch(Exception e)
    {
        fthrow( Exception, "LocalizationResult::loadImageInfo: error loading image info (ImageLabeler xml format)");
    }
}

void LocalizationResult::store (ostream & os, int format) const
{
    if ( format == FILEFORMAT_PASCAL2006_RESULT ) 
    {
	for ( const_iterator i = begin(); i != end(); i++ )
	{
	    const SingleLocalizationResult *sr = *i;
	    const ClassificationResult *r = sr->r;
	    int classno = r->classno;
	    double score = r->scores.get(classno);

	    int xi, yi, xa, ya;
	    sr->getBoundingBox ( xi, yi, xa, ya );

	    os << cn->text(r->classno) << " " << score << " "
	       << xi << " "
	       << yi << " "
	       << xa << " "
	       << ya << " " << endl;
	}
    } else {
	fprintf (stderr, "LocalizationResult::store: file format not yet supported !\n");
	exit(-1);
    }
    

}

void LocalizationResult::clear ()
{
    for ( iterator k = begin(); k != end() ; k++ )
    {
	SingleLocalizationResult *slr = *k;
	delete slr;
    }
    
    vector<SingleLocalizationResult *>::clear();
    hasLabeledImage = false;   
}


/** returns whether the depth of x is smaller than that of y 
    !!! no transitivity !!! */
bool depthCompare ( const SingleLocalizationResult *x, const SingleLocalizationResult *y )
{
    /** According to the LabelMe paper of Torralba et al., Murphy */
    const NICE::Region & rx = x->getRegion();
    const NICE::Region & ry = y->getRegion();

    NICE::Region intersect;
    intersect.setIntersection ( rx, ry );
    int ax = rx.size();
    int ay = ry.size();
    int is = intersect.size();

    if ( is == 0 )
    {
	int nx = x->getControlPoints();
	int ny = y->getControlPoints();

	return ( nx > ny );
    } else {
	double ratx = (double)is / ax;
	double raty = (double)is / ay;

	return ( ratx >  raty );
    }
}

bool confidenceCompare ( const SingleLocalizationResult *x, const SingleLocalizationResult *y )
{
    return ( x->r->confidence() > y->r->confidence() );
}

void LocalizationResult::sortDescendingConfidence()
{
    sort ( begin(), end(), confidenceCompare );
}

void LocalizationResult::sortEmpricalDepth()
{
    sort ( begin(), end(), depthCompare );
}

void LocalizationResult::calcLabeledImage (
        NICE::ImageT<int> & mark,
        int backgroundClassNo ) const
{
    mark.set(backgroundClassNo);

    fprintf (stderr, "LocalizationResult: calcLabeledImage %zd\n", size() );
    for ( int y = 0 ; y < mark.height(); y++ )
        for ( int x = 0 ; x < mark.width(); x++ )
            for ( LocalizationResult::const_iterator k = begin(); k != end() ; k++ )
            {
                SingleLocalizationResult *slr = *k;
                const NICE::Region & r = slr->getRegion();

                if ( r.inside(x,y) ) {
                    mark.setPixel(x,y,slr->r->classno);
                    break;
                }
            }
}

void LocalizationResult::calcLabeledImage (
        NICE::Image & mark,
        int backgroundClassNo ) const
{
    NICE::ImageT<int> markInt;
    markInt.resize ( mark.width(), mark.height() );
    calcLabeledImage( markInt, backgroundClassNo );

    for ( int y = 0; y < markInt.height(); y++ )
        for ( int x = 0; x < markInt.width(); x++ )
        {
            int cLabel = markInt.getPixelQuick( x, y );

            if ( cLabel > std::numeric_limits<unsigned char>::max() )
                std::cerr << "LocalizationResult::calcLabeledImage: To many classes! Labeled image with UCHAR is not sufficient!"
                          << std::endl;

            mark.setPixelQuick( x, y, (unsigned char) cLabel );
        }

}

void LocalizationResult::getLabeledImageCache ( NICE::Image & mark ) const
{
    assert ( hasLabeledImage );
    labeledImage->copyFrom ( mark );
}
    
void LocalizationResult::setMap ( const NICE::Image & _labeledImage )
{
    labeledImage = new Image( _labeledImage );
    hasLabeledImage = true;
}

void drawOrthoLine ( NICE::ColorImage & img, 
		     int x1, int y1, int x2, int y2, 
		     int width, int sign,
		     int r,
		     int g,
		     int b )
{
    int xi = x1; int yi = y1;
    int xa = x2; int ya = y2;
    for ( int i = 0 ; i < width; i++ )
    {
		if ( yi == ya ) {
			yi = yi + sign;
			ya = ya + sign; 
		} else if ( xi == xa ) {
			xi = xi + sign;
			xa = xa + sign;
		} else {
			assert ( 0 == 1 );
		}
		if ( (xi>=0) && (yi>=0) && (xa<=img.width())
			 && (ya<=img.height()) )
		{
			NICE::Line l ( Coord(xi, yi), Coord(xa, ya) );
			img.draw( l, NICE::Color(r, g, b) );
		}
    }
}

void LocalizationResult::displayBoxes ( NICE::ColorImage & img, const ClassNames *cn, 
					bool display_confidence, bool invert, int width ) const
{
    for ( LocalizationResult::const_iterator k = begin(); k != end() ; k++ )
    {
		SingleLocalizationResult *slr = *k;
		int xi, yi, xa, ya;
		slr->getBoundingBox ( xi, yi, xa, ya );
		
		int classno = (slr->r == NULL ) ? 0 : slr->r->classno;
		int r,g,b;
		if ( cn != NULL ) {
			cn->getRGBColor ( classno, r, g, b );	
		} else {
			r = 255;
			g = 0;
			b = 0;
		}

		if ( invert ) {
			r = 255 - r;
			g = 255 - g;
			b = 255 - b;
		}

		if ( display_confidence && (cn != NULL)) {
			std::string name = cn->text(classno);
			char caption[1024];
			sprintf ( caption, "%3.2lf %s", slr->r->confidence(), name.c_str() );
			// refactor-nice.pl: check this substitution
			// old: Text(caption, xi, yi, r, 0, img.RedImage());
			// REFACTOR-FIXME Unable to map this statement
			// refactor-nice.pl: check this substitution
			// old: Text(caption, xi, yi, g, 0, img.GreenImage());
			// REFACTOR-FIXME Unable to map this statement
			// refactor-nice.pl: check this substitution
			// old: Text(caption, xi, yi, b, 0, img.BlueImage());
			// REFACTOR-FIXME Unable to map this statement
		}

		drawOrthoLine ( img, xi-width, yi, xa+width, yi, width, -1, r, g, b );
		drawOrthoLine ( img, xa, yi, xa, ya, width, +1, r, g, b );
		drawOrthoLine ( img, xa+width, ya, xi-width, ya, width, +1, r, g, b );
		drawOrthoLine ( img, xi, ya, xi, yi, width, -1, r, g, b );
    }
}