/** * @file SemanticSegmentation.cpp * @brief abstract interface for semantic segmentation algorithms * @author Erik Rodner, Alexander Freytag, Sven Sickert * @date 03/19/2009 */ #include "SemanticSegmentation.h" #include #include #include #include using namespace OBJREC; using namespace std; using namespace NICE; ///////////////////// ///////////////////// ///////////////////// // CONSTRUCTORS / DESTRUCTORS ///////////////////// ///////////////////// ///////////////////// SemanticSegmentation::SemanticSegmentation ( ) : iterationCountSuffix(1) { this->imagetype = IMAGETYPE_RGB; this->classNames = new ClassNames(); } SemanticSegmentation::SemanticSegmentation ( const Config *conf, const ClassNames *classNames ) : iterationCountSuffix(1) { /////////// // same code as in empty constructor - duplication can be avoided with C++11 allowing for constructor delegation /////////// /////////// // here comes the new code part different from the empty constructor /////////// this->classNames = classNames; this->initFromConfig( conf ); } SemanticSegmentation::~SemanticSegmentation() { } void SemanticSegmentation::initFromConfig(const Config* conf, const string& s_confSection) { std::string imagetype_s = conf->gS ( "main", "imagetype", "rgb" ); coarseMode = conf->gB( "main", "coarse_mode", false ); if ( imagetype_s == "rgb" ) imagetype = IMAGETYPE_RGB; else if ( imagetype_s == "gray" ) imagetype = IMAGETYPE_GRAY; else { fprintf ( stderr, "SemanticSegmentation:: unknown image type option\n" ); exit ( -1 ); } // dangerous!!! Preprocess::Init ( conf ); } ///////////////////// ///////////////////// ///////////////////// // SEGMENTATION STUFF ///////////////////// ///////////////////// ///////////////////// void SemanticSegmentation::semanticseg ( const std::string & filename, NICE::ImageT & segresult, NICE::MultiChannelImageT & probabilities ) { Globals::setCurrentImgFN ( filename ); CachedExample *ce; if ( imagetype == IMAGETYPE_RGB ) { NICE::ColorImage img = Preprocess::ReadImgAdvRGB ( filename ); ce = new CachedExample ( img ); } else { NICE::Image img = Preprocess::ReadImgAdv ( filename ); ce = new CachedExample ( img ); } #ifdef DEBUG_PRINTS fprintf ( stderr, "Starting Semantic Segmentation !\n" ); #endif this->semanticseg ( ce, segresult, probabilities ); delete ce; } void SemanticSegmentation::classify ( const std::vector & filelist, NICE::MultiChannelImageT & segresult, NICE::MultiChannelImage3DT & probabilities ) { for ( int it = 0; it < ( int ) filelist.size(); it++ ) { NICE::MultiChannelImageT probs; NICE::ImageT res ( segresult.width(), segresult.height() ); this->semanticseg( filelist[it], res, probs ); probabilities.addChannel( probs ); segresult.addChannel( res ); } } ///////////////////// ///////////////////// ///////////////////// // DATA CONVERSION ///////////////////// ///////////////////// ///////////////////// void SemanticSegmentation::convertLSetToSparseExamples ( Examples &examples, LabeledSetVector &lvec ) { #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertLSetToExamples starts" << endl; #endif for ( map< int, vector >::iterator iter = lvec.begin(); iter != lvec.end(); ++iter ) { for ( int j = 0; j < ( int ) iter->second.size(); j++ ) { Vector &tmp = * ( iter->second[j] ); int dim = tmp.size(); SparseVector *vec = new SparseVector ( dim ); for ( int j = 0; j < dim; j++ ) { if ( tmp[j] != 0.0 ) { ( *vec ) [j] = tmp[j]; } } Example ex; ex.svec = vec; examples.push_back ( pair ( iter->first, ex ) ); } } lvec.clear(); #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertLSetToExamples finished" << endl; #endif } void SemanticSegmentation::convertLSetToExamples ( Examples &examples, LabeledSetVector &lvec, const bool & removeOldDataPointer ) { #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertLSetToExamples starts" << endl; #endif for ( map< int, vector >::iterator iter = lvec.begin(); iter != lvec.end(); ++iter ) { for ( int j = 0; j < (int)iter->second.size(); j++ ) { NICE::Vector *vec = new NICE::Vector ( * ( iter->second[j] ) ); Example ex ( vec ); examples.push_back ( pair ( iter->first, ex ) ); } } if (!removeOldDataPointer) { //NOTE this is only useful, if our classifier does NOT need the data explicitely lvec.clear(); } else { lvec.removePointersToDataWithoutDeletion(); //after setting all the pointers to NULL, we can savely clear the LSet without deleting the previously //stored features, which might be needed somewhere else, e.g., in the VCNearestNeighbour lvec.clear(); } #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertLSetToExamples finished" << endl; #endif } void SemanticSegmentation::convertExamplesToLSet ( Examples &examples, LabeledSetVector &lvec ) { #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertExamplesToLSet starts" << endl; #endif lvec.clear(); for ( int i = 0; i < ( int ) examples.size(); i++ ) { if ( examples[i].second.vec != NULL ) { lvec.add ( examples[i].first, *examples[i].second.vec ); delete examples[i].second.vec; examples[i].second.vec = NULL; } else { if ( examples[i].second.svec != NULL ) { NICE::Vector v; examples[i].second.svec->convertToVectorT(v); lvec.add ( examples[i].first, v ); delete examples[i].second.svec; examples[i].second.svec = NULL; } else { throw ( "no features for LabeledSet" ); } } } examples.clear(); #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertExamplesToLSet finished" << endl; #endif } void SemanticSegmentation::convertExamplesToVVector ( VVector &feats, Examples &examples, vector &label ) { #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertExamplesToVVector starts" << endl; #endif feats.clear(); label.clear(); for ( int i = 0; i < ( int ) examples.size(); i++ ) { label.push_back ( examples[i].first ); feats.push_back ( *examples[i].second.vec ); delete examples[i].second.vec; examples[i].second.vec = NULL; } examples.clear(); #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertExamplesToVVector finished" << endl; #endif } void SemanticSegmentation::convertVVectorToExamples ( VVector &feats, Examples &examples, vector &label ) { #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertVVectorToExamples starts" << endl; #endif for ( int i = 0; i < ( int ) feats.size(); i++ ) { NICE::Vector *v = new NICE::Vector ( feats[i] ); Example ex ( v ); ex.position = 0; //TODO: hier mal was besseres überlegen, damit Klassifikator wieder Bildspezifisch lernt examples.push_back ( pair ( label[i], ex ) ); feats[i].clear(); } feats.clear(); label.clear(); #ifdef DEBUG_PRINTS cout << "SemSegRegionBased::convertVVectorToExamples finished" << endl; #endif } void SemanticSegmentation::setIterationCountSuffix( const int & _iterationCountSuffix) { this->iterationCountSuffix = _iterationCountSuffix; } void SemanticSegmentation::setClassNames ( const OBJREC::ClassNames * _classNames ) { this->classNames = _classNames; } void SemanticSegmentation::getDepthVector ( const LabeledSet *Files, vector & depthVec, const bool run3Dseg ) { std::string oldName; int zsize = 0; bool isInit = false; for (LabeledSet::const_iterator it = Files->begin(); it != Files->end(); it++) { for (std::vector::const_iterator jt = it->second.begin(); jt != it->second.end(); jt++) { ImageInfo & info = *(*jt); std::string file = info.img(); std::vector< std::string > list; StringTools::split ( file, '/', list ); std::string filename = list.back(); uint found = filename.find_last_of ( "_" ); if (run3Dseg && found < filename.size() && found-3 > 0 ) { std::string curName = filename.substr ( found-3,3 ); if ( !isInit ) { oldName = curName; isInit = true; } if ( curName.compare ( oldName ) == 0 ) // if strings match up { zsize++; } else { depthVec.push_back ( zsize ); zsize = 1; oldName = curName; } } else { zsize = 1; depthVec.push_back ( zsize ); } } } depthVec.push_back ( zsize ); } void SemanticSegmentation::make3DImage ( const std::vector & filelist, NICE::MultiChannelImage3DT & imgData ) { bool isInit = false; for ( int it = 0; it < ( int ) filelist.size(); it++ ) { if ( imagetype == IMAGETYPE_RGB ) { NICE::ColorImage img; try { img.read ( filelist[it] ); } catch ( ImageException iE ) { fprintf ( stderr, "Failed to open color image file: %s\n", filelist[it].c_str() ); fprintf ( stderr, "%s\n", iE.what() ); exit ( -1 ); } if ( !isInit ) { imgData.reInit ( img.width(),img.height(),filelist.size(),3 ); isInit = true; } for ( int y = 0; y < img.height(); y++ ) { for ( int x = 0; x < img.width(); x++ ) { for ( int r = 0; r < 3; r++ ) { imgData.set ( x, y, it, img.getPixel ( x,y,r ), r ); } } } } else { NICE::Image img; try { img.read ( filelist[it] ); } catch ( ImageException iE ) { fprintf ( stderr, "Failed to open image file: %s\n", filelist[it].c_str() ); fprintf ( stderr, "%s\n", iE.what() ); exit ( -1 ); } if ( !isInit ) { imgData.reInit ( img.width(),img.height(),filelist.size(),1 ); isInit = true; } for ( int y = 0; y < img.height(); y++ ) { for ( int x = 0; x < img.width(); x++ ) { imgData.set ( x, y, it, img.getPixel ( x,y ), 0 ); } } } } if ( imagetype == IMAGETYPE_GRAY ) { imgData.equalizeHistogram( 0 ); #ifdef DEBUG_PRINTS for (int z = 0; z < imgData.depth(); z++) { NICE::Image im = imgData.getChannel( z, 0); im.write( filelist[z]+"_eq.pgm"); } #endif } } void SemanticSegmentation::getProbabilityMap ( const NICE::MultiChannelImage3DT & prob ) { std::string s; for ( int cl = 0; cl < prob.channels(); cl++ ) for ( int z = 0; z < prob.depth(); z++ ) { NICE::ColorImage img( prob.width(),prob.height() ); NICE::ImageT m = prob.getChannelT(z, cl); imageToPseudoColor(m, img); std::stringstream out; out << "probmap_s" << z << "_c" << cl << ".ppm"; s = out.str(); img.write( s ); //showImage(img, "Probability map"); //getchar(); } } ///////////////////// INTERFACE PERSISTENT ///////////////////// // interface specific methods for store and restore ///////////////////// INTERFACE PERSISTENT ///////////////////// void SemanticSegmentation::restore ( std::istream & is, int format ) { //delete everything we knew so far... this->clear(); bool b_restoreVerbose ( false ); #ifdef B_RESTOREVERBOSE b_restoreVerbose = true; #endif if ( is.good() ) { if ( b_restoreVerbose ) std::cerr << " restore SemanticSegmentation" << std::endl; std::string tmp; is >> tmp; //class name if ( ! this->isStartTag( tmp, "SemanticSegmentation" ) ) { std::cerr << " WARNING - attempt to restore SemanticSegmentation, but start flag " << tmp << " does not match! Aborting... " << std::endl; throw; } is.precision (numeric_limits::digits10 + 1); bool b_endOfBlock ( false ) ; while ( !b_endOfBlock ) { is >> tmp; // start of block if ( this->isEndTag( tmp, "SemanticSegmentation" ) ) { b_endOfBlock = true; continue; } tmp = this->removeStartTag ( tmp ); if ( b_restoreVerbose ) std::cerr << " currently restore section " << tmp << " in SemanticSegmentation" << std::endl; if ( tmp.compare("classNames") == 0 ) { //dirty solution to circumvent the const-flag const_cast(this->classNames)->restore ( is, format ); is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("imagetype") == 0 ) { unsigned int ui_imagetyp; is >> ui_imagetyp; this->imagetype = static_cast ( ui_imagetyp ); is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else if ( tmp.compare("iterationCountSuffix") == 0 ) { is >> this->iterationCountSuffix; is >> tmp; // end of block tmp = this->removeEndTag ( tmp ); } else { std::cerr << "WARNING -- unexpected SemanticSegmentation object -- " << tmp << " -- for restoration... aborting" << std::endl; throw; } } } else { std::cerr << "SemanticSegmentation::restore -- InStream not initialized - restoring not possible!" << std::endl; throw; } //TODO check whether we also have to do something linke Preprocess::Init ( conf ); } void SemanticSegmentation::store ( std::ostream & os, int format ) const { if (os.good()) { // show starting point os << this->createStartTag( "SemanticSegmentation" ) << std::endl; os.precision (numeric_limits::digits10 + 1); os << this->createStartTag( "classNames" ) << std::endl; this->classNames->store ( os, format ); os << this->createEndTag( "classNames" ) << std::endl; // os << this->createStartTag( "imagetype" ) << std::endl; os << imagetype << std::endl; os << this->createEndTag( "imagetype" ) << std::endl; // os << this->createStartTag( "iterationCountSuffix" ) << std::endl; os << iterationCountSuffix << std::endl; os << this->createEndTag( "iterationCountSuffix" ) << std::endl; // done os << this->createEndTag( "SemanticSegmentation" ) << std::endl; } else { std::cerr << "OutStream not initialized - storing not possible!" << std::endl; } } void SemanticSegmentation::clear () { //TODO }