NICE_SemSeg/semseg/SemanticSegmentation.cpp

/**
* @file SemanticSegmentation.cpp
* @brief abstract interface for semantic segmentation algorithms
* @author Erik Rodner, Alexander Freytag, Sven Sickert
* @date 03/19/2009

*/
#include <iostream>

#include "core/image/MultiChannelImage3DT.h"
#include "core/basics/StringTools.h"

#include "SemanticSegmentation.h"
#include "vislearning/baselib/Preprocess.h"
#include "vislearning/baselib/Globals.h"

using namespace OBJREC;
using namespace std;
using namespace NICE;

///////////////////// ///////////////////// /////////////////////
//                   CONSTRUCTORS / DESTRUCTORS
///////////////////// ///////////////////// /////////////////////     

SemanticSegmentation::SemanticSegmentation ( )
    : iterationCountSuffix(1)
{
  this->imagetype = IMAGETYPE_RGB;
  
  this->classNames = new ClassNames();

}

SemanticSegmentation::SemanticSegmentation ( const Config *conf,
                                             const ClassNames *classNames )
    : iterationCountSuffix(1)
{
  ///////////
  // same code as in empty constructor - duplication can be avoided with C++11 allowing for constructor delegation
  /////////// 
  
  ///////////
  // here comes the new code part different from the empty constructor
  /////////// 

  this->classNames = classNames; 
  
  this->initFromConfig( conf ); 
}

SemanticSegmentation::~SemanticSegmentation()
{
}

void SemanticSegmentation::initFromConfig(const Config* conf, const string& s_confSection)
{
  std::string imagetype_s = conf->gS ( "main", "imagetype", "rgb" );

  if ( imagetype_s == "rgb" )
    imagetype = IMAGETYPE_RGB;
  else if ( imagetype_s == "gray" )
    imagetype = IMAGETYPE_GRAY;
  else {
    fprintf ( stderr, "SemanticSegmentation:: unknown image type option\n" );
    exit ( -1 );
  }

  // dangerous!!!
  Preprocess::Init ( conf );
}

///////////////////// ///////////////////// /////////////////////
//                      SEGMENTATION STUFF
///////////////////// ///////////////////// /////////////////////

void SemanticSegmentation::semanticseg ( const std::string & filename,
    NICE::Image & segresult,
    NICE::MultiChannelImageT<double> & probabilities )
{
  Globals::setCurrentImgFN ( filename );
  CachedExample *ce;
  if ( imagetype == IMAGETYPE_RGB )
  {
    NICE::ColorImage img = Preprocess::ReadImgAdvRGB ( filename );
    ce = new CachedExample ( img );
  } else {

    NICE::Image img = Preprocess::ReadImgAdv ( filename );
    ce = new CachedExample ( img );
  }
  fprintf ( stderr, "Starting Semantic Segmentation !\n" );
  semanticseg ( ce, segresult, probabilities );
  delete ce;
}

///////////////////// ///////////////////// /////////////////////
//                      DATA CONVERSION
///////////////////// ///////////////////// ///////////////////// 
void SemanticSegmentation::convertLSetToSparseExamples ( Examples &examples, LabeledSetVector &lvec )
{
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertLSetToExamples starts" << endl;
#endif
  for ( map< int, vector<NICE::Vector *> >::iterator iter = lvec.begin(); iter != lvec.end(); ++iter )
  {
    for ( int j = 0; j < ( int ) iter->second.size(); j++ )
    {
      Vector &tmp = * ( iter->second[j] );
      int dim = tmp.size();
      SparseVector *vec = new SparseVector ( dim );
      for ( int j = 0; j < dim; j++ )
      {
        if ( tmp[j] != 0.0 )
        {
          ( *vec ) [j] = tmp[j];
        }
      }
      Example ex;
      ex.svec = vec;
      examples.push_back ( pair<int, Example> ( iter->first, ex ) );
    }
  }
  lvec.clear();
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertLSetToExamples finished" << endl;
#endif
}

void SemanticSegmentation::convertLSetToExamples ( Examples &examples, LabeledSetVector &lvec, const bool & removeOldDataPointer )
{
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertLSetToExamples starts" << endl;
#endif
  for ( map< int, vector<NICE::Vector *> >::iterator iter = lvec.begin(); iter != lvec.end(); ++iter )
  {
    for ( int j = 0; j < (int)iter->second.size(); j++ )
    {
      NICE::Vector *vec = new NICE::Vector ( * ( iter->second[j] ) );
      Example ex ( vec );
      examples.push_back ( pair<int, Example> ( iter->first, ex ) );
    }
  }

  if (!removeOldDataPointer)
  {
    //NOTE this is only useful, if our classifier does NOT need the data explicitely
    lvec.clear();
  }
  else
  {
    lvec.removePointersToDataWithoutDeletion();
    //after setting all the pointers to NULL, we can savely clear the LSet without deleting the previously
    //stored features, which might be needed somewhere else, e.g., in the VCNearestNeighbour
    lvec.clear();
  }

#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertLSetToExamples finished" << endl;
#endif
}

void SemanticSegmentation::convertExamplesToLSet ( Examples &examples, LabeledSetVector &lvec )
{
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertExamplesToLSet starts" << endl;
#endif
  lvec.clear();
  for ( int i = 0; i < ( int ) examples.size(); i++ )
  {
    if ( examples[i].second.vec != NULL )
    {
      lvec.add ( examples[i].first, *examples[i].second.vec );
      delete examples[i].second.vec;
      examples[i].second.vec = NULL;
    }
    else
    {
      if ( examples[i].second.svec != NULL )
      {
        NICE::Vector v;
        examples[i].second.svec->convertToVectorT(v);
        lvec.add ( examples[i].first, v );
        delete examples[i].second.svec;
        examples[i].second.svec = NULL;        
      }
      else
      {
        throw ( "no features for LabeledSet" );
      }
    }

  }
  examples.clear();  
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertExamplesToLSet finished" << endl;
#endif
}

void SemanticSegmentation::convertExamplesToVVector ( VVector &feats, Examples &examples, vector<int> &label )
{
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertExamplesToVVector starts" << endl;
#endif
  feats.clear();
  label.clear();
  for ( int i = 0; i < ( int ) examples.size(); i++ )
  {
    label.push_back ( examples[i].first );
    feats.push_back ( *examples[i].second.vec );
    delete examples[i].second.vec;
    examples[i].second.vec = NULL;
  }
  examples.clear();
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertExamplesToVVector finished" << endl;
#endif
}

void SemanticSegmentation::convertVVectorToExamples ( VVector &feats, Examples &examples, vector<int> &label )
{
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertVVectorToExamples starts" << endl;
#endif
  for ( int i = 0; i < ( int ) feats.size(); i++ )
  {
    NICE::Vector *v = new NICE::Vector ( feats[i] );
    Example ex ( v );
    ex.position = 0; //TODO: hier mal was besseres überlegen, damit Klassifikator wieder Bildspezifisch lernt
    examples.push_back ( pair<int, Example> ( label[i], ex ) );
    feats[i].clear();
  }
  feats.clear();
  label.clear();
#ifdef DEBUG_PRINTS
  cout << "SemSegRegionBased::convertVVectorToExamples finished" << endl;
#endif
}

void SemanticSegmentation::setIterationCountSuffix( const int & _iterationCountSuffix)
{
  this->iterationCountSuffix = _iterationCountSuffix;
}

void SemanticSegmentation::setClassNames ( const OBJREC::ClassNames * _classNames )
{
  this->classNames = _classNames;
}

void SemanticSegmentation::getDepthVector ( const LabeledSet *Files,
                                            vector<int> & depthVec,
                                            const bool run3Dseg )
{
  std::string oldName;
  int zsize = 0;
  bool isInit = false;

  for (LabeledSet::const_iterator it = Files->begin(); it != Files->end(); it++)
  {
    for (std::vector<ImageInfo *>::const_iterator jt = it->second.begin();
         jt != it->second.end(); jt++)
    {
      ImageInfo & info = *(*jt);
      std::string file = info.img();

      std::vector< std::string > list;
      StringTools::split ( file, '/', list );
      std::string filename = list.back();
      uint found = filename.find_last_of ( "_" );
      if (run3Dseg && found < filename.size() && found-3 > 0 )
      {
        std::string curName = filename.substr ( found-3,3 );
        if ( !isInit )
        {
          oldName = curName;
          isInit = true;
        }
        if ( curName.compare ( oldName ) == 0 ) // if strings match up
        {
          zsize++;
        }
        else
        {
          depthVec.push_back ( zsize );
          zsize = 1;
          oldName = curName;
        }
      }
      else
      {
        zsize = 1;
        depthVec.push_back ( zsize );
      }

    }
  }
  depthVec.push_back ( zsize );
}

void SemanticSegmentation::make3DImage ( const std::vector<std::string> & filelist,
    NICE::MultiChannelImage3DT<double> & imgData )
{
  bool isInit = false;
  for ( int it = 0; it < ( int ) filelist.size(); it++ )
  {
    if ( imagetype == IMAGETYPE_RGB )
    {
      NICE::ColorImage img;
      try
      {
        img.read ( filelist[it] );
      }
      catch ( ImageException iE )
      {
        fprintf ( stderr, "Failed to open color image file: %s\n", filelist[it].c_str() );
        fprintf ( stderr, "%s\n", iE.what() );
        exit ( -1 );
      }
      if ( !isInit )
      {
        imgData.reInit ( img.width(),img.height(),filelist.size(),3 );
        isInit = true;
      }
      for ( int y = 0; y < img.height(); y++ )
      {
        for ( int x = 0; x < img.width(); x++ )
        {
          for ( int r = 0; r < 3; r++ )
          {
            imgData.set ( x, y, it, img.getPixel ( x,y,r ), r );
          }
        }
      }
    }
    else
    {
      NICE::Image img;
      try
      {
        img.read ( filelist[it] );
      }
      catch ( ImageException iE )
      {
        fprintf ( stderr, "Failed to open image file: %s\n", filelist[it].c_str() );
        fprintf ( stderr, "%s\n", iE.what() );
        exit ( -1 );
      }
      if ( !isInit )
      {
        imgData.reInit ( img.width(),img.height(),filelist.size(),1 );
        isInit = true;
      }
      for ( int y = 0; y < img.height(); y++ )
      {
        for ( int x = 0; x < img.width(); x++ )
        {
          imgData.set ( x, y, it, img.getPixel ( x,y ), 0 );
        }
      }
    }
  }

  if ( imagetype == IMAGETYPE_GRAY )
  {
    imgData.equalizeHistogram( 0 );
#ifdef DEBUG_PRINTS
    for (int z = 0; z < imgData.depth(); z++)
    {
      NICE::Image im = imgData.getChannel( z, 0);
      im.write( filelist[z]+"_eq.pgm");
    }
#endif
  }
}

void SemanticSegmentation::getProbabilityMap ( const NICE::MultiChannelImage3DT<double> & prob )
{
  std::string s;

  for ( int cl = 0; cl < prob.channels(); cl++ )
    for ( int z = 0; z < prob.depth(); z++ )
    {
      NICE::ColorImage img( prob.width(),prob.height() );
      NICE::ImageT<double> m = prob.getChannelT(z, cl);
      imageToPseudoColor(m, img);

      std::stringstream out;
      out << "probmap_s" << z << "_c" << cl << ".ppm";
      s = out.str();
      img.write( s );

      //showImage(img, "Probability map");
      //getchar();
    }
}

///////////////////// INTERFACE PERSISTENT /////////////////////
// interface specific methods for store and restore
///////////////////// INTERFACE PERSISTENT /////////////////////

void SemanticSegmentation::restore ( std::istream & is, int format )
{
  //delete everything we knew so far...
  this->clear();

  bool b_restoreVerbose ( false );
#ifdef B_RESTOREVERBOSE
  b_restoreVerbose = true;
#endif

  if ( is.good() )
  {
    if ( b_restoreVerbose )
      std::cerr << " restore SemanticSegmentation" << std::endl;

    std::string tmp;
    is >> tmp; //class name

    if ( ! this->isStartTag( tmp, "SemanticSegmentation" ) )
    {
      std::cerr << " WARNING - attempt to restore SemanticSegmentation, but start flag " << tmp << " does not match! Aborting... " << std::endl;
      throw;
    }

    is.precision (numeric_limits<double>::digits10 + 1);

    bool b_endOfBlock ( false ) ;

    while ( !b_endOfBlock )
    {
      is >> tmp; // start of block

      if ( this->isEndTag( tmp, "SemanticSegmentation" ) )
      {
        b_endOfBlock = true;
        continue;
      }

      tmp = this->removeStartTag ( tmp );

      if ( b_restoreVerbose )
        std::cerr << " currently restore section " << tmp << " in SemanticSegmentation" << std::endl;

      if ( tmp.compare("classNames") == 0 )
      {
  //dirty solution to circumvent the const-flag
        const_cast<ClassNames*>(this->classNames)->restore ( is, format );

        is >> tmp; // end of block
        tmp = this->removeEndTag ( tmp );
      }
      else if ( tmp.compare("imagetype") == 0 )
      {
        unsigned int ui_imagetyp;
        is >> ui_imagetyp;
        this->imagetype = static_cast<IMAGETYP> ( ui_imagetyp );

        is >> tmp; // end of block
        tmp = this->removeEndTag ( tmp );
      }
      else if ( tmp.compare("iterationCountSuffix") == 0 )
      {
        is >> this->iterationCountSuffix;

        is >> tmp; // end of block
        tmp = this->removeEndTag ( tmp );
      }
      else
      {
      std::cerr << "WARNING -- unexpected SemanticSegmentation object -- " << tmp << " -- for restoration... aborting" << std::endl;
      throw;
      }
    }
  }
  else
  {
    std::cerr << "SemanticSegmentation::restore -- InStream not initialized - restoring not possible!" << std::endl;
    throw;
  }


 //TODO check whether we also have to do something linke Preprocess::Init ( conf );
}

void SemanticSegmentation::store ( std::ostream & os, int format ) const
{
  if (os.good())
  {
    // show starting point
    os << this->createStartTag( "SemanticSegmentation" ) << std::endl;

    os.precision (numeric_limits<double>::digits10 + 1);

    os << this->createStartTag( "classNames" ) << std::endl;
    this->classNames->store ( os, format );
    os << this->createEndTag( "classNames" ) << std::endl;

    //

    os << this->createStartTag( "imagetype" ) << std::endl;
    os << imagetype << std::endl;
    os << this->createEndTag( "imagetype" ) << std::endl;

    //

    os << this->createStartTag( "iterationCountSuffix" ) << std::endl;
    os << iterationCountSuffix << std::endl;
    os << this->createEndTag( "iterationCountSuffix" ) << std::endl;

    // done
    os << this->createEndTag( "SemanticSegmentation" ) << std::endl;
  }
  else
  {
    std::cerr << "OutStream not initialized - storing not possible!" << std::endl;
  }
}

void SemanticSegmentation::clear ()
{
 //TODO
}