NICE_SemSeg/semseg/SemSegTools.cpp

/**
* @file SemSegTools.cpp
* @brief tools for semantic segmentation
* @author Erik Rodner, Sven Sickert
* @date 03/19/2009

*/
#include <iostream>
#include <iomanip>

#include "SemSegTools.h"

using namespace OBJREC;

using namespace std;
using namespace NICE;

#undef DEBUG_LOCALIZATION
#undef DEBUG

void SemSegTools::segmentToOverlay (
        const NICE::Image *orig,
        const NICE::ColorImage & segment,
        NICE::ColorImage & result )
{
    int xsize = orig->width();
    int ysize = orig->height();

    result.resize( xsize, ysize );
    std::vector< NICE::MatrixT<double> > channelMat;

    double alpha = .3;

    for (int c = 0; c < 3; c++)
    {
        NICE::MatrixT<double> chan ( xsize, ysize );
        channelMat.push_back( chan );
    }

    for (int y = 0; y < ysize; y++)
        for (int x = 0; x < xsize; x++)
        {
            uchar val = orig->getPixelQuick(x,y);
            for (int c = 0; c < 3; c++)
                channelMat[c](x,y) = alpha*(double)val
                        + (1.0-alpha)*(double)segment.getPixel( x, y, c );
        }

    for (int y = 0; y < ysize; y++)
        for (int x = 0; x < xsize; x++)
            for (int c = 0; c < 3; c++)
            {
                int val = channelMat[c](x,y);
                result.setPixel( x, y, c, (uchar)val);
            }

}

void SemSegTools::updateConfusionMatrix(
        const ImageT<int> &img,
        const ImageT<int> &gt,
        Matrix &M,
        const std::set<int> &forbiddenClasses,
        map<int,int> & classMapping )
{
    double subsamplex = gt.width() / ( double ) img.width();
    double subsampley = gt.height() / ( double ) img.height();

    for ( int y = 0 ; y < gt.height() ; y++ )
        for ( int x = 0 ; x < gt.width() ; x++ )
        {
            int xx = ( int ) ( x / subsamplex );
            int yy = ( int ) ( y / subsampley );

            if ( xx < 0 ) xx = 0;

            if ( yy < 0 ) yy = 0;

            if ( xx > img.width() - 1 ) xx = img.width() - 1;

            if ( yy > img.height() - 1 ) yy = img.height() - 1;

            int cimg = img.getPixel ( xx, yy );
            int gimg = gt.getPixel ( x, y );

            if ( forbiddenClasses.find ( gimg ) == forbiddenClasses.end() )
            {
                M ( classMapping[gimg], classMapping[cimg] ) ++;
            }
        }
}

void SemSegTools::computeClassificationStatistics(
        Matrix &confMat,
        const ClassNames &classNames,
        const std::set<int> &forbiddenClasses,
        map<int,int> & classMappingInv )
{
    std::cout << "\nPERFORMANCE" << std::endl;
    std::cout << "###########\n" << std::endl;

    double overallTrue = confMat.trace();
    double sumAll  = 0.0;

    // overall recognition rate
    for ( int r = 0; r < (int) confMat.rows(); r++ )
        for ( int c = 0; c < (int) confMat.cols(); c++ )
            sumAll += confMat( r, c );

    overallTrue /= sumAll;

    double precision = 0.0, recall = 0.0, f1score = 0.0, iuScore = 0.0;

    // binary classification
    for ( int c = 0; c < classMappingInv.size(); c++ )
    {
        double precBase = 0.0, recBase = 0.0;
        for ( int r = 0; r < classMappingInv.size(); r++ )
            precBase += confMat(r,c);

        for ( int cc = 0; cc < classMappingInv.size(); cc++ )
            recBase += confMat(c,cc);

        precision += confMat(c,c) / precBase;
        recall += confMat(c,c) / recBase;
        iuScore += confMat(c,c) / (precBase+recBase-confMat(c,c));
    }

    precision /= classMappingInv.size();
    recall /= classMappingInv.size();
    iuScore /= classMappingInv.size();
    f1score = 2.0*(precision*recall)/(precision+recall);

    // normalizing confMat using rows
    for ( int r = 0 ; r < (int) confMat.rows() ; r++ )
    {
        double sum = 0.0;

        for ( int c = 0 ; c < (int) confMat.cols() ; c++ )
            sum += confMat ( r, c );

        if ( std::fabs ( sum ) > 1e-4 )
            for ( int c = 0 ; c < (int) confMat.cols() ; c++ )
                confMat ( r, c ) /= sum;
    }

    // printing confusion matrix
    short int printWidth = 16;
    std::cout.precision(6);
    std::cout << std::setw(printWidth) << "";
    for (int r = 0; r < (int) confMat.rows(); r++)
    {
        int cl = classMappingInv[r];
        if ( classNames.existsClassno ( cl )
             && ( forbiddenClasses.find ( cl ) == forbiddenClasses.end() ) )
        {
            std::string cname = classNames.text ( cl );
            std::cout << std::setw(printWidth) << cname.c_str();
        }
    }
    std::cout << std::endl;
    for (int r = 0; r < (int) confMat.rows(); r++)
    {
        int cl = classMappingInv[r];
        if ( classNames.existsClassno ( cl )
             && ( forbiddenClasses.find ( cl ) == forbiddenClasses.end() ) )
        {
            std::string cname = classNames.text ( cl );
            std::cout << std::setw(printWidth) << cname.c_str();

            for (int c = 0; c < (int) confMat.cols(); c++)
                std::cout << std::setw(printWidth) << std::fixed << confMat (r, c);

            std::cout << std::endl;
        }
    }

    // print classification statistics
    std::cout << "\nAccuracy: " << overallTrue;
    std::cout << "\nPrecision: " << precision;
    std::cout << "\nRecall: " << recall;
    std::cout << "\nF1Score: " << f1score;
    std::cout << "\nIU: " << iuScore;
    std::cout << "\n\nAverage Recognition Rate: " << confMat.trace() / (double)classMappingInv.size();
    //std::cout << "\nLower Bound: " << 1.0 /(double)classMappingInv.size();
    std::cout << std::endl;
}

void SemSegTools::computeResourceStatistics (
        NICE::ResourceStatistics &rs )
{
    std::cout << "\nSTATISTICS" << std::endl;
    std::cout << "##########\n" << std::endl;

    long maxMemory;
    double userCPUTime, sysCPUTime;
    rs.getStatistics ( maxMemory, userCPUTime, sysCPUTime );
    std::cout << "Memory (max):    " << maxMemory << " KB" << std::endl;
    std::cout << "CPU Time (user): " << userCPUTime << " seconds" << std::endl;
    std::cout << "CPU Time (sys):  " << sysCPUTime << " seconds" << std::endl;
}

void SemSegTools::saveResultsToImageFile(
        const Config *conf,
        const string &section,
        const ColorImage &orig,
        const ColorImage &gtruth,
        const ColorImage &segment,
        const string &file,
        string & outStr )
{
    std::string resultDir = conf->gS ( section, "resultdir", "." );
    std::string outputType = conf->gS ( section, "output_type", "ppm" );
    std::string outputPostfix = conf->gS ( section, "output_postfix", "" );

    NICE::ColorImage overlaySegment, overlayGTruth;

    NICE::Image* origGrey = orig.getChannel(1);
    segmentToOverlay( origGrey, segment, overlaySegment );
    segmentToOverlay( origGrey, gtruth, overlayGTruth );

    std::stringstream out;
    out << resultDir << "/" << file << outputPostfix;

#ifdef DEBUG
    std::cout << "Writing to file " << out.str() << "_*." << outputType << std::endl;
#endif

    orig.write ( out.str() + "_orig." + outputType );
    segment.write ( out.str() + "_result." + outputType );
    gtruth.write ( out.str() + "_groundtruth." + outputType );
    overlaySegment.write ( out.str() + "_overlay_res." + outputType );
    overlayGTruth.write ( out.str() + "_overlay_gt." + outputType );

    outStr = out.str();
}

void SemSegTools::collectTrainingExamples ( 
        const Config * conf,
        const std::string & section,
        const LabeledSet & train,
        const ClassNames & cn,
        Examples & examples,
        vector<CachedExample *> & imgexamples )
{
    assert ( train.count() > 0 );
    examples.clear();
    imgexamples.clear();

    int grid_size_x = conf->gI(section, "grid_size_x", 5 );
    int grid_size_y = conf->gI(section, "grid_size_y", 5 );
    int grid_border_x = conf->gI(section, "grid_border_x", 20 );
    int grid_border_y = conf->gI(section, "grid_border_y", 20 );

    std::string selection = conf->gS(section, "train_selection" );

    set<int> classnoSelection;
    cn.getSelection ( selection, classnoSelection );
    
    bool useExcludedAsBG = conf->gB(section, "use_excluded_as_background", false );

    int backgroundClassNo = 0;
    
    if ( useExcludedAsBG )
    {
        backgroundClassNo = cn.classno("various");
        assert ( backgroundClassNo >= 0 );
    }

    LOOP_ALL_S (train)
    {
        EACH_INFO(image_classno,imgInfo);
        std::string imgfn = imgInfo.img();

        if ( ! imgInfo.hasLocalizationInfo() ) {
            std::cerr << "WARNING: NO localization info found for "
                      << imgfn << " !" << std::endl;
            continue;
        }

        int xsize, ysize;
        CachedExample *ce = new CachedExample ( imgfn );
        ce->getImageSize ( xsize, ysize );
        imgexamples.push_back ( ce );

        const LocalizationResult *locResult = imgInfo.localization();
        if ( locResult->size() <= 0 ) {
            std::cerr << "WARNING: NO ground truth polygons found for "
                      << imgfn << " !" << std::endl;
            continue;
        }

        std::cerr << "SemSegTools: Collecting pixel examples from localization info: "
                  << imgfn << std::endl;

        NICE::ImageT<int> pixelLabels (xsize, ysize);
        pixelLabels.set(0);
        locResult->calcLabeledImage ( pixelLabels, cn.getBackgroundClass() );

#ifdef DEBUG_LOCALIZATION
        NICE::Image img (imgfn);
        showImage(img);
        showImage(pixelLabels);
#endif

        Example pce ( ce, 0, 0 );
        for ( int x = 0 ; x < xsize ; x += grid_size_x )
            for ( int y = 0 ; y < ysize ; y += grid_size_y )
            {
                if ( (x >= grid_border_x) &&
                     ( y >= grid_border_y ) && ( x < xsize - grid_border_x ) &&
                     ( y < ysize - grid_border_x ) )
                {
                    pce.x = x; pce.y = y;
                    int classno = pixelLabels.getPixel(x,y);

                    if ( classnoSelection.find(classno) != classnoSelection.end() ) {
                        examples.push_back ( pair<int, Example> (
                                                 classno,
                                                 pce // FIXME: offset handling
                                                 ) );
                    } else if ( useExcludedAsBG ) {
                        examples.push_back ( pair<int, Example> (
                                                 backgroundClassNo,
                                                 pce // FIXME: offset handling
                                                 ) );
                    }
                }
            }
    }

    std::cerr << "total number of examples: " << (int)examples.size() << std::endl;
}