testSemanticSegmentation3D.cpp 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444
  1. // Beispielhafter Aufruf: BUILD_x86_64/progs/testSemanticSegmentation -config <CONFIGFILE>
  2. /**
  3. * @file testSemanticSegmentation.cpp
  4. * @brief test semantic segmentation routines for 3d images and 2d images
  5. * @author Erik Rodner, Björn Fröhlich, Sven Sickert
  6. * @date 03/20/2008
  7. */
  8. #ifdef NICE_USELIB_OPENMP
  9. #include <omp.h>
  10. #endif
  11. #include "core/basics/Config.h"
  12. #include "core/basics/StringTools.h"
  13. #include "vislearning/baselib/ICETools.h"
  14. #include "core/image/MultiChannelImage3DT.h"
  15. #include "semseg/semseg/SemSegContextTree3D.h"
  16. #include "semseg/semseg/SemSegTools.h"
  17. #include "core/basics/ResourceStatistics.h"
  18. #include "core/image/Morph.h"
  19. #include <fstream>
  20. #include <vector>
  21. #undef DEBUG
  22. using namespace OBJREC;
  23. using namespace NICE;
  24. using namespace std;
  25. void updateMatrix ( const NICE::ImageT<int> & img,
  26. const NICE::ImageT<int> & gt,
  27. NICE::Matrix & M,
  28. const set<int> & forbidden_classes,
  29. map<int,int> & classMapping )
  30. {
  31. double subsamplex = gt.width() / ( double ) img.width();
  32. double subsampley = gt.height() / ( double ) img.height();
  33. for ( int y = 0 ; y < gt.height() ; y++ )
  34. for ( int x = 0 ; x < gt.width() ; x++ )
  35. {
  36. int xx = ( int ) ( x / subsamplex );
  37. int yy = ( int ) ( y / subsampley );
  38. if ( xx < 0 ) xx = 0;
  39. if ( yy < 0 ) yy = 0;
  40. if ( xx > img.width() - 1 ) xx = img.width() - 1;
  41. if ( yy > img.height() - 1 ) yy = img.height() - 1;
  42. int cimg = img.getPixel ( xx, yy );
  43. int gimg = gt.getPixel ( x, y );
  44. if ( forbidden_classes.find ( gimg ) == forbidden_classes.end() )
  45. {
  46. M ( classMapping[gimg], classMapping[cimg] ) ++;
  47. }
  48. }
  49. }
  50. void startClassification (SemanticSegmentation *semseg,
  51. std::vector< NICE::Matrix > & M_vec,
  52. const Config & conf,
  53. const LabeledSet* testFiles,
  54. const ClassNames & classNames,
  55. const set<int> & forbidden_classes,
  56. map<int,int> & classMapping,
  57. const string & resultdir,
  58. const bool doCrossVal)
  59. {
  60. bool show_results = conf.gB ( "debug", "show_results", false );
  61. bool write_results = conf.gB ( "debug", "write_results", false );
  62. bool writeProbMaps = conf.gB ( "debug", "write_prob_maps", false );
  63. if (doCrossVal)
  64. write_results = false;
  65. bool run_3Dseg = conf.gB( "SSContextTree", "run_3dseg", false);
  66. bool postProcessing = conf.gB( "main", "post_process", false);
  67. string output_type = conf.gS ( "debug", "output_type", "ppm" );
  68. string output_postfix = conf.gS ( "debug", "output_postfix", "" );
  69. vector< int > zsizeVec;
  70. semseg->getDepthVector ( testFiles, zsizeVec, run_3Dseg );
  71. int depthCount = 0, idx = 0;
  72. vector< string > filelist;
  73. NICE::MultiChannelImageT<int> segresult;
  74. NICE::MultiChannelImageT<int> gt;
  75. for (LabeledSet::const_iterator it = testFiles->begin(); it != testFiles->end(); it++)
  76. {
  77. for (std::vector<ImageInfo *>::const_iterator jt = it->second.begin();
  78. jt != it->second.end(); jt++)
  79. {
  80. ImageInfo & info = *(*jt);
  81. std::string file = info.img();
  82. filelist.push_back ( file );
  83. depthCount++;
  84. NICE::ImageT<int> lm;
  85. NICE::ImageT<int> lm_gt;
  86. if ( info.hasLocalizationInfo() )
  87. {
  88. const LocalizationResult *l_gt = info.localization();
  89. lm.resize ( l_gt->xsize, l_gt->ysize );
  90. lm.set ( 0 );
  91. lm_gt.resize ( l_gt->xsize, l_gt->ysize );
  92. lm_gt.set ( 0 );
  93. l_gt->calcLabeledImage ( lm, classNames.getBackgroundClass() );
  94. #ifdef DEBUG
  95. cout << "testSemanticSegmentation3D: Generating Labeled NICE::Image (Ground-Truth)" << endl;
  96. #endif
  97. l_gt->calcLabeledImage ( lm_gt, classNames.getBackgroundClass() );
  98. }
  99. segresult.addChannel ( lm );
  100. gt.addChannel ( lm_gt );
  101. int depthBoundary = 0;
  102. if ( run_3Dseg )
  103. {
  104. depthBoundary = zsizeVec[idx];
  105. }
  106. if ( depthCount < depthBoundary ) continue;
  107. NICE::MultiChannelImage3DT<double> probabilities;
  108. semseg->classify ( filelist, segresult, probabilities );
  109. // save to file
  110. for ( int z = 0; z < segresult.channels(); z++ )
  111. {
  112. std::string fname = StringTools::baseName ( filelist[z], false );
  113. if ( show_results || write_results )
  114. {
  115. NICE::ColorImage orig ( filelist[z] );
  116. NICE::ColorImage rgb;
  117. NICE::ColorImage rgb_gt;
  118. NICE::ColorImage ov_rgb;
  119. NICE::ColorImage ov_rgb_gt;
  120. for ( int y = 0 ; y < orig.height(); y++ )
  121. {
  122. for ( int x = 0 ; x < orig.width(); x++ )
  123. {
  124. lm.setPixel ( x, y, segresult.get ( x, y, ( uint ) z ) );
  125. if ( run_3Dseg )
  126. lm_gt.setPixel ( x, y, gt.get ( x, y, ( uint ) z ) );
  127. }
  128. }
  129. // confusion matrix
  130. NICE::Matrix M ( classMapping.size(), classMapping.size() );
  131. M.set ( 0 );
  132. updateMatrix ( lm, lm_gt, M, forbidden_classes, classMapping );
  133. M_vec.push_back ( M );
  134. classNames.labelToRGB ( lm, rgb );
  135. classNames.labelToRGB ( lm_gt, rgb_gt );
  136. if (postProcessing)
  137. {
  138. // median filter
  139. for (int r = 0; r < 3; r++)
  140. {
  141. NICE::Image postIm(rgb.width(), rgb.height());
  142. NICE::median(*(rgb.getChannel(r)), &postIm, 1);
  143. for (int y = 0; y < rgb.height(); y++)
  144. for (int x = 0; x < rgb.width(); x++)
  145. rgb.setPixel(x,y,r, postIm.getPixelQuick(x,y));
  146. }
  147. }
  148. if ( write_results )
  149. {
  150. SemSegTools::segmentToOverlay ( orig.getChannel(1), rgb, ov_rgb );
  151. SemSegTools::segmentToOverlay ( orig.getChannel(1), rgb_gt, ov_rgb_gt );
  152. std::stringstream out;
  153. if ( output_postfix.size() > 0 )
  154. out << resultdir << "/" << fname << output_postfix;
  155. else
  156. out << resultdir << "/" << fname;
  157. #ifdef DEBUG
  158. cout << "Writing to file " << out.str() << "_*." << output_type << endl;
  159. #endif
  160. orig.write ( out.str() + "_orig." + output_type );
  161. rgb.write ( out.str() + "_result." + output_type );
  162. rgb_gt.write ( out.str() + "_groundtruth." + output_type );
  163. ov_rgb.write ( out.str() + "_overlay_res." + output_type );
  164. ov_rgb_gt.write ( out.str() + "_overlay_gt." + output_type );
  165. // write Probability maps
  166. if (writeProbMaps)
  167. {
  168. NICE::ColorImage prob_map( probabilities.width(), probabilities.height() );
  169. prob_map.set(0,0,0);
  170. int iNumChannels = probabilities.channels();
  171. for ( int idxProbMap = 0; idxProbMap < iNumChannels; idxProbMap++)
  172. {
  173. for ( int y = 0 ; y < probabilities.height(); y++ )
  174. {
  175. for ( int x = 0 ; x < probabilities.width(); x++ )
  176. {
  177. double probVal = probabilities.get( x, y, z, idxProbMap ) * 255.0;
  178. int tmp = round(probVal);
  179. for ( int c = 0 ; c < 3 ; c++ )
  180. prob_map.setPixel( x, y, c, tmp );
  181. }
  182. }
  183. std::stringstream ssFileProbMap;
  184. //ssFileProbMap << out.str() << "_probs." << "c" << idxProbMap << "." << output_type;
  185. ssFileProbMap << out.str() << "_probs." << "c-" << classNames.code( idxProbMap ) << "." << output_type;
  186. //classNames
  187. prob_map.write ( ssFileProbMap.str() );
  188. }
  189. }
  190. }
  191. }
  192. }
  193. // prepare for new 3d image
  194. filelist.clear();
  195. segresult.reInit(0,0,0);
  196. gt.reInit(0,0,0);
  197. depthCount = 0;
  198. idx++;
  199. }
  200. }
  201. segresult.freeData();
  202. }
  203. /**
  204. test semantic segmentation routines
  205. */
  206. int main ( int argc, char **argv )
  207. {
  208. std::set_terminate ( __gnu_cxx::__verbose_terminate_handler );
  209. Config conf ( argc, argv );
  210. ResourceStatistics rs;
  211. /*---------------CONFIGURATION---------------*/
  212. bool doCrossVal = conf.gB ( "debug", "do_crossval", false );
  213. string resultdir = conf.gS ( "debug", "resultdir", "." );
  214. /*-------------------------------------------*/
  215. #ifdef DEBUG
  216. cerr << "Writing Results to " << resultdir << endl;
  217. #endif
  218. std::vector< NICE::Matrix > M_vec;
  219. MultiDataset md ( &conf );
  220. const ClassNames & classNames = md.getClassNames ( "train" );
  221. set<int> forbidden_classes;
  222. classNames.getSelection ( conf.gS ( "analysis", "forbidden_classes", "" ),
  223. forbidden_classes );
  224. vector<bool> usedClasses ( classNames.numClasses(), true );
  225. for ( set<int>::const_iterator it = forbidden_classes.begin();
  226. it != forbidden_classes.end(); ++it)
  227. {
  228. usedClasses [ *it ] = false;
  229. }
  230. map<int,int> classMapping;
  231. int j = 0;
  232. for ( int i = 0; i < usedClasses.size(); i++ )
  233. if (usedClasses[i])
  234. {
  235. classMapping[i] = j;
  236. j++;
  237. }
  238. // initialize semantic segmentation method
  239. SemanticSegmentation *semseg = NULL;
  240. // TRAINING AND TESTING
  241. if (!doCrossVal)
  242. {
  243. semseg = new SemSegContextTree3D ( &conf, &classNames );
  244. // STANDARD EVALUATION
  245. cout << "\nTRAINING" << endl;
  246. cout << "########\n" << endl;
  247. semseg->train( &md );
  248. cout << "\nCLASSIFICATION" << endl;
  249. cout << "##############\n" << endl;
  250. const LabeledSet *testFiles = md["test"];
  251. startClassification (semseg, M_vec, conf, testFiles, classNames,
  252. forbidden_classes, classMapping, resultdir, doCrossVal );
  253. delete semseg;
  254. }
  255. else
  256. {
  257. // CROSS-VALIDATION
  258. for (int cval = 1; cval <= 10; cval++)
  259. {
  260. semseg = new SemSegContextTree3D ( &conf, &classNames );
  261. stringstream ss;
  262. ss << cval;
  263. string cvaltrain = "train_cv" + ss.str();
  264. string cvaltest = "test_cv" + ss.str();
  265. cout << "\nTRAINING " << cval << endl;
  266. cout << "###########\n" << endl;
  267. const LabeledSet *trainFiles = md[cvaltrain];
  268. semseg->train( trainFiles );
  269. cout << "\nCLASSIFICATION " << cval << endl;
  270. cout << "#################\n" << endl;
  271. const LabeledSet *testFiles = md[cvaltest];
  272. startClassification (semseg, M_vec, conf, testFiles, classNames,
  273. forbidden_classes, classMapping, resultdir, doCrossVal );
  274. delete semseg;
  275. }
  276. }
  277. cout << "\nSTATISTICS" << endl;
  278. cout << "##########\n" << endl;
  279. long maxMemory;
  280. double userCPUTime, sysCPUTime;
  281. rs.getStatistics ( maxMemory, userCPUTime, sysCPUTime );
  282. cout << "Memory (max): " << maxMemory << " KB" << endl;
  283. cout << "CPU Time (user): " << userCPUTime << " seconds" << endl;
  284. cout << "CPU Time (sys): " << sysCPUTime << " seconds" << endl;
  285. cout << "\nPERFORMANCE" << endl;
  286. cout << "###########\n" << endl;
  287. double overall = 0.0;
  288. double sumall = 0.0;
  289. NICE::Matrix M ( classMapping.size(), classMapping.size() );
  290. M.set ( 0 );
  291. for ( int s = 0; s < ( int ) M_vec.size(); s++ )
  292. {
  293. NICE::Matrix M_tmp = M_vec[s];
  294. for ( int r = 0; r < ( int ) M_tmp.rows(); r++ )
  295. for ( int c = 0; c < ( int ) M_tmp.cols(); c++ )
  296. {
  297. if ( r == c )
  298. overall += M_tmp ( r, c );
  299. sumall += M_tmp ( r, c );
  300. M ( r, c ) += M_tmp ( r, c );
  301. }
  302. }
  303. overall /= sumall;
  304. cout << "Confusion Matrix:" << endl;
  305. cout.precision(4);
  306. for (int r = 0; r < (int) M.rows(); r++)
  307. {
  308. for (int c = 0; c < (int) M.cols(); c++)
  309. cout << M(r,c)/sumall << " ";
  310. cout << endl;
  311. }
  312. // metrics for binary classification
  313. double precision, recall, f1score = -1.0;
  314. if (classNames.numClasses() == 2)
  315. {
  316. precision = (double)M(1,1) / (double)(M(1,1)+M(0,1));
  317. recall = (double)M(1,1) / (double)(M(1,1)+M(1,0));
  318. f1score = 2.0*(precision*recall)/(precision+recall);
  319. }
  320. // normalizing M using rows
  321. for ( int r = 0 ; r < ( int ) M.rows() ; r++ )
  322. {
  323. double sum = 0.0;
  324. for ( int c = 0 ; c < ( int ) M.cols() ; c++ )
  325. sum += M ( r, c );
  326. if ( fabs ( sum ) > 1e-4 )
  327. for ( int c = 0 ; c < ( int ) M.cols() ; c++ )
  328. M ( r, c ) /= sum;
  329. }
  330. double avg_perf = 0.0;
  331. int classes_trained = 0;
  332. for ( int r = 0 ; r < ( int ) M.rows() ; r++ )
  333. {
  334. if ( ( classNames.existsClassno ( r ) ) && ( forbidden_classes.find ( r ) == forbidden_classes.end() ) )
  335. {
  336. avg_perf += M ( r, r );
  337. double lsum = 0.0;
  338. for ( int r2 = 0; r2 < ( int ) M.rows(); r2++ )
  339. {
  340. lsum += M ( r,r2 );
  341. }
  342. if ( lsum != 0.0 )
  343. {
  344. classes_trained++;
  345. }
  346. }
  347. }
  348. // print results of evaluation
  349. cout << "Overall Recogntion Rate: " << overall << endl;
  350. cout << "Average Recogntion Rate: " << avg_perf / ( classes_trained ) << endl;
  351. cout << "Lower Bound: " << 1.0 / classes_trained << endl;
  352. cout << "Precision: " << precision << endl;
  353. cout << "Recall: " << recall << endl;
  354. cout << "F1Score: " << f1score << endl;
  355. cout <<"\nClasses:" << endl;
  356. for ( int r = 0 ; r < ( int ) M.rows() ; r++ )
  357. {
  358. if ( ( classNames.existsClassno ( r ) ) && ( forbidden_classes.find ( r ) == forbidden_classes.end() ) )
  359. {
  360. std::string classname = classNames.text ( r );
  361. cout << classname.c_str() << ": " << M ( r, r ) << endl;
  362. }
  363. }
  364. return 0;
  365. }