SemSegContextTree.cpp 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658
  1. #include "SemSegContextTree.h"
  2. #include "vislearning/baselib/Globals.h"
  3. #include "vislearning/baselib/ProgressBar.h"
  4. #include "core/basics/StringTools.h"
  5. #include "vislearning/cbaselib/CachedExample.h"
  6. #include "vislearning/cbaselib/PascalResults.h"
  7. #include "vislearning/baselib/ColorSpace.h"
  8. #include "objrec/segmentation/RSMeanShift.h"
  9. #include "objrec/segmentation/RSGraphBased.h"
  10. #include "core/basics/numerictools.h"
  11. #include "vislearning/baselib/ICETools.h"
  12. #include "core/basics/Timer.h"
  13. #include "core/basics/vectorio.h"
  14. #include "core/image/FilterT.h"
  15. #include <omp.h>
  16. #include <iostream>
  17. #undef LOCALFEATS
  18. #undef WRITEGLOB
  19. #undef TEXTONMAP
  20. //#define LOCALFEATS
  21. using namespace OBJREC;
  22. using namespace std;
  23. using namespace NICE;
  24. SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *md )
  25. : SemanticSegmentation ( conf, & ( md->getClassNames ( "train" ) ) )
  26. {
  27. this->conf = conf;
  28. string section = "SSContextTree";
  29. lfcw = new LFColorWeijer ( conf );
  30. grid = conf->gI ( section, "grid", 10 );
  31. maxSamples = conf->gI ( section, "max_samples", 2000 );
  32. minFeats = conf->gI ( section, "min_feats", 50 );
  33. maxDepth = conf->gI ( section, "max_depth", 10 );
  34. windowSize = conf->gI ( section, "window_size", 16 );
  35. featsPerSplit = conf->gI ( section, "feats_per_split", 200 );
  36. useShannonEntropy = conf->gB ( section, "use_shannon_entropy", true );
  37. nbTrees = conf->gI ( section, "amount_trees", 1 );
  38. string segmentationtype = conf->gS ( section, "segmentation_type", "meanshift" );
  39. useGaussian = conf->gB ( section, "use_gaussian", true );
  40. randomTests = conf->gI ( section, "random_tests", 10 );
  41. bool saveLoadData = conf->gB ( "debug", "save_load_data", false );
  42. string fileLocation = conf->gS ( "debug", "datafile", "tmp.txt" );
  43. if ( useGaussian )
  44. throw ( "there something wrong with using gaussian! first fix it!" );
  45. pixelWiseLabeling = false;
  46. if ( segmentationtype == "meanshift" )
  47. segmentation = new RSMeanShift ( conf );
  48. else if ( segmentationtype == "none" )
  49. {
  50. segmentation = NULL;
  51. pixelWiseLabeling = true;
  52. }
  53. else if ( segmentationtype == "felzenszwalb" )
  54. segmentation = new RSGraphBased ( conf );
  55. else
  56. throw ( "no valid segmenation_type\n please choose between none, meanshift and felzenszwalb\n" );
  57. ftypes = conf->gI ( section, "features", 2 );;
  58. string featsec = "Features";
  59. if ( conf->gB ( featsec, "minus", true ) )
  60. ops.push_back ( new Minus() );
  61. if ( conf->gB ( featsec, "minus_abs", true ) )
  62. ops.push_back ( new MinusAbs() );
  63. if ( conf->gB ( featsec, "addition", true ) )
  64. ops.push_back ( new Addition() );
  65. if ( conf->gB ( featsec, "only1", true ) )
  66. ops.push_back ( new Only1() );
  67. if ( conf->gB ( featsec, "rel_x", true ) )
  68. ops.push_back ( new RelativeXPosition() );
  69. if ( conf->gB ( featsec, "rel_y", true ) )
  70. ops.push_back ( new RelativeYPosition() );
  71. if ( conf->gB ( featsec, "bi_int_cent", true ) )
  72. cops.push_back ( new BiIntegralCenteredOps() );
  73. if ( conf->gB ( featsec, "int_cent", true ) )
  74. cops.push_back ( new IntegralCenteredOps() );
  75. if ( conf->gB ( featsec, "int", true ) )
  76. cops.push_back ( new IntegralOps() );
  77. if ( conf->gB ( featsec, "haar_horz", true ) )
  78. cops.push_back ( new HaarHorizontal() );
  79. if ( conf->gB ( featsec, "haar_vert", true ) )
  80. cops.push_back ( new HaarVertical() );
  81. if ( conf->gB ( featsec, "haar_diag", true ) )
  82. cops.push_back ( new HaarDiag() );
  83. if ( conf->gB ( featsec, "haar3_horz", true ) )
  84. cops.push_back ( new Haar3Horiz() );
  85. if ( conf->gB ( featsec, "haar3_vert", true ) )
  86. cops.push_back ( new Haar3Vert() );
  87. if ( conf->gB ( featsec, "glob", true ) )
  88. cops.push_back ( new GlobalFeats() );
  89. useGradient = conf->gB ( featsec, "use_gradient", true );
  90. opOverview = vector<int> ( NBOPERATIONS, 0 );
  91. contextOverview = vector<vector<double> > ( maxDepth, vector<double> ( 2, 0.0 ) );
  92. calcVal.push_back ( new MCImageAccess() );
  93. calcVal.push_back ( new ClassificationResultAccess() );
  94. classnames = md->getClassNames ( "train" );
  95. ///////////////////////////////////
  96. // Train Segmentation Context Trees
  97. ///////////////////////////////////
  98. if ( saveLoadData )
  99. {
  100. if ( FileMgt::fileExists ( fileLocation ) )
  101. read ( fileLocation );
  102. else
  103. {
  104. train ( md );
  105. write ( fileLocation );
  106. }
  107. }
  108. else
  109. {
  110. train ( md );
  111. }
  112. }
  113. SemSegContextTree::~SemSegContextTree()
  114. {
  115. }
  116. double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<NICE::MultiChannelImageT<unsigned short int> > &currentfeats, std::vector<NICE::MultiChannelImageT<double> > &integralImgs, const std::vector<NICE::MatrixT<int> > &labels, int node, Operation *&splitop, double &splitval, const int &tree )
  117. {
  118. Timer t;
  119. t.start();
  120. int imgCount = 0, featdim = 0;
  121. try
  122. {
  123. imgCount = ( int ) feats.size();
  124. featdim = feats[0].channels();
  125. }
  126. catch ( Exception )
  127. {
  128. cerr << "no features computed?" << endl;
  129. }
  130. double bestig = -numeric_limits< double >::max();
  131. splitop = NULL;
  132. splitval = -1.0;
  133. set<vector<int> >selFeats;
  134. map<int, int> e;
  135. int featcounter = forest[tree][node].featcounter;
  136. if ( featcounter < minFeats )
  137. {
  138. //cout << "only " << featcounter << " feats in current node -> it's a leaf" << endl;
  139. return 0.0;
  140. }
  141. vector<double> fraction ( a.size(), 0.0 );
  142. for ( uint i = 0; i < fraction.size(); i++ )
  143. {
  144. if ( forbidden_classes.find ( labelmapback[i] ) != forbidden_classes.end() )
  145. fraction[i] = 0;
  146. else
  147. fraction[i] = ( ( double ) maxSamples ) / ( ( double ) featcounter * a[i] * a.size() );
  148. //cout << "fraction["<<i<<"]: "<< fraction[i] << " a[" << i << "]: " << a[i] << endl;
  149. }
  150. featcounter = 0;
  151. for ( int iCounter = 0; iCounter < imgCount; iCounter++ )
  152. {
  153. int xsize = ( int ) currentfeats[iCounter].width();
  154. int ysize = ( int ) currentfeats[iCounter].height();
  155. for ( int x = 0; x < xsize; x++ )
  156. {
  157. for ( int y = 0; y < ysize; y++ )
  158. {
  159. if ( currentfeats[iCounter].get ( x, y, tree ) == node )
  160. {
  161. int cn = labels[iCounter] ( x, y );
  162. double randD = ( double ) rand() / ( double ) RAND_MAX;
  163. if ( labelmap.find ( cn ) == labelmap.end() )
  164. continue;
  165. if ( randD < fraction[labelmap[cn]] )
  166. {
  167. vector<int> tmp ( 3, 0 );
  168. tmp[0] = iCounter;
  169. tmp[1] = x;
  170. tmp[2] = y;
  171. featcounter++;
  172. selFeats.insert ( tmp );
  173. e[cn]++;
  174. }
  175. }
  176. }
  177. }
  178. }
  179. //cout << "size: " << selFeats.size() << endl;
  180. //getchar();
  181. map<int, int>::iterator mapit;
  182. double globent = 0.0;
  183. for ( mapit = e.begin() ; mapit != e.end(); mapit++ )
  184. {
  185. //cout << "class: " << mapit->first << ": " << mapit->second << endl;
  186. double p = ( double ) ( *mapit ).second / ( double ) featcounter;
  187. globent += p * log2 ( p );
  188. }
  189. globent = -globent;
  190. if ( globent < 0.5 )
  191. {
  192. //cout << "globent to small: " << globent << endl;
  193. return 0.0;
  194. }
  195. int classes = ( int ) forest[tree][0].dist.size();
  196. featsel.clear();
  197. for ( int i = 0; i < featsPerSplit; i++ )
  198. {
  199. int x1, x2, y1, y2;
  200. int ft = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ftypes );
  201. int tmpws = windowSize;
  202. if ( integralImgs[0].width() == 0 )
  203. ft = 0;
  204. if ( ft > 0 )
  205. {
  206. tmpws *= 4;
  207. }
  208. if ( useGaussian )
  209. {
  210. double sigma = ( double ) tmpws * 2.0;
  211. x1 = randGaussDouble ( sigma ) * ( double ) tmpws;
  212. x2 = randGaussDouble ( sigma ) * ( double ) tmpws;
  213. y1 = randGaussDouble ( sigma ) * ( double ) tmpws;
  214. y2 = randGaussDouble ( sigma ) * ( double ) tmpws;
  215. }
  216. else
  217. {
  218. x1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  219. x2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  220. y1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  221. y2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  222. }
  223. if ( ft == 0 )
  224. {
  225. int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) featdim );
  226. int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) featdim );
  227. int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ops.size() );
  228. Operation *op = ops[o]->clone();
  229. op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
  230. op->setContext ( false );
  231. featsel.push_back ( op );
  232. }
  233. else if ( ft == 1 )
  234. {
  235. int opssize = ( int ) ops.size();
  236. //opssize = 0;
  237. int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( ( ( double ) cops.size() ) + ( double ) opssize ) );
  238. Operation *op;
  239. if ( o < opssize )
  240. {
  241. int chans = ( int ) forest[0][0].dist.size();
  242. int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  243. int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  244. op = ops[o]->clone();
  245. op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
  246. op->setContext ( true );
  247. }
  248. else
  249. {
  250. int chans = integralImgs[0].channels();
  251. int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  252. int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  253. o -= opssize;
  254. op = cops[o]->clone();
  255. op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
  256. if ( f1 < forest[0][0].dist.size() )
  257. op->setContext ( true );
  258. else
  259. op->setContext ( false );
  260. }
  261. featsel.push_back ( op );
  262. }
  263. }
  264. #pragma omp parallel for private(mapit)
  265. for ( int f = 0; f < featsPerSplit; f++ )
  266. {
  267. double l_bestig = -numeric_limits< double >::max();
  268. double l_splitval = -1.0;
  269. set<vector<int> >::iterator it;
  270. vector<double> vals;
  271. double maxval = -numeric_limits<double>::max();
  272. double minval = numeric_limits<double>::max();
  273. for ( it = selFeats.begin() ; it != selFeats.end(); it++ )
  274. {
  275. Features feat;
  276. feat.feats = &feats[ ( *it ) [0]];
  277. feat.cfeats = &currentfeats[ ( *it ) [0]];
  278. feat.cTree = tree;
  279. feat.tree = &forest[tree];
  280. feat.integralImg = &integralImgs[ ( *it ) [0]];
  281. double val = featsel[f]->getVal ( feat, ( *it ) [1], ( *it ) [2] );
  282. vals.push_back ( val );
  283. maxval = std::max ( val, maxval );
  284. minval = std::min ( val, minval );
  285. }
  286. if ( minval == maxval )
  287. continue;
  288. double scale = maxval - minval;
  289. vector<double> splits;
  290. for ( int r = 0; r < randomTests; r++ )
  291. {
  292. splits.push_back ( ( ( double ) rand() / ( double ) RAND_MAX*scale ) + minval );
  293. }
  294. for ( int run = 0 ; run < randomTests; run++ )
  295. {
  296. set<vector<int> >::iterator it2;
  297. double val = splits[run];
  298. map<int, int> eL, eR;
  299. int counterL = 0, counterR = 0;
  300. int counter2 = 0;
  301. for ( it2 = selFeats.begin() ; it2 != selFeats.end(); it2++, counter2++ )
  302. {
  303. int cn = labels[ ( *it2 ) [0]] ( ( *it2 ) [1], ( *it2 ) [2] );
  304. //cout << "vals[counter2] " << vals[counter2] << " val: " << val << endl;
  305. if ( vals[counter2] < val )
  306. {
  307. //left entropie:
  308. eL[cn] = eL[cn] + 1;
  309. counterL++;
  310. }
  311. else
  312. {
  313. //right entropie:
  314. eR[cn] = eR[cn] + 1;
  315. counterR++;
  316. }
  317. }
  318. double leftent = 0.0;
  319. for ( mapit = eL.begin() ; mapit != eL.end(); mapit++ )
  320. {
  321. double p = ( double ) ( *mapit ).second / ( double ) counterL;
  322. leftent -= p * log2 ( p );
  323. }
  324. double rightent = 0.0;
  325. for ( mapit = eR.begin() ; mapit != eR.end(); mapit++ )
  326. {
  327. double p = ( double ) ( *mapit ).second / ( double ) counterR;
  328. rightent -= p * log2 ( p );
  329. }
  330. //cout << "rightent: " << rightent << " leftent: " << leftent << endl;
  331. double pl = ( double ) counterL / ( double ) ( counterL + counterR );
  332. double ig = globent - ( 1.0 - pl ) * rightent - pl * leftent;
  333. //double ig = globent - rightent - leftent;
  334. if ( useShannonEntropy )
  335. {
  336. double esplit = - ( pl * log ( pl ) + ( 1 - pl ) * log ( 1 - pl ) );
  337. ig = 2 * ig / ( globent + esplit );
  338. }
  339. if ( ig > l_bestig )
  340. {
  341. l_bestig = ig;
  342. l_splitval = val;
  343. }
  344. }
  345. #pragma omp critical
  346. {
  347. //cout << "globent: " << globent << " bestig " << bestig << " splitfeat: " << splitfeat << " splitval: " << splitval << endl;
  348. //cout << "globent: " << globent << " l_bestig " << l_bestig << " f: " << p << " l_splitval: " << l_splitval << endl;
  349. //cout << "p: " << featsubset[f] << endl;
  350. if ( l_bestig > bestig )
  351. {
  352. bestig = l_bestig;
  353. splitop = featsel[f];
  354. splitval = l_splitval;
  355. }
  356. }
  357. }
  358. //getchar();
  359. //splitop->writeInfos();
  360. //cout<< "ig: " << bestig << endl;
  361. //FIXME: delete all features!
  362. /*for(int i = 0; i < featsPerSplit; i++)
  363. {
  364. if(featsel[i] != splitop)
  365. delete featsel[i];
  366. }*/
  367. #ifdef debug
  368. cout << "globent: " << globent << " bestig " << bestig << " splitval: " << splitval << endl;
  369. #endif
  370. return bestig;
  371. }
  372. inline double SemSegContextTree::getMeanProb ( const int &x, const int &y, const int &channel, const MultiChannelImageT<unsigned short int> &currentfeats )
  373. {
  374. double val = 0.0;
  375. for ( int tree = 0; tree < nbTrees; tree++ )
  376. {
  377. val += forest[tree][currentfeats.get ( x,y,tree ) ].dist[channel];
  378. }
  379. return val / ( double ) nbTrees;
  380. }
  381. void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<SparseVectorInt> &infeats, NICE::MultiChannelImageT<SparseVectorInt> &integralImage )
  382. {
  383. int xsize = infeats.width();
  384. int ysize = infeats.height();
  385. integralImage ( 0, 0 ).add ( infeats.get ( 0, 0 ) );
  386. //first column
  387. for ( int y = 1; y < ysize; y++ )
  388. {
  389. integralImage ( 0, y ).add ( infeats.get ( 0, y ) );
  390. integralImage ( 0, y ).add ( integralImage ( 0, y - 1 ) );
  391. }
  392. //first row
  393. for ( int x = 1; x < xsize; x++ )
  394. {
  395. integralImage ( x, 0 ).add ( infeats.get ( x, 0 ) );
  396. integralImage ( x, 0 ).add ( integralImage ( x - 1, 0 ) );
  397. }
  398. //rest
  399. for ( int y = 1; y < ysize; y++ )
  400. {
  401. for ( int x = 1; x < xsize; x++ )
  402. {
  403. integralImage ( x, y ).add ( infeats.get ( x, y ) );
  404. integralImage ( x, y ).add ( integralImage ( x, y - 1 ) );
  405. integralImage ( x, y ).add ( integralImage ( x - 1, y ) );
  406. integralImage ( x, y ).sub ( integralImage ( x - 1, y - 1 ) );
  407. }
  408. }
  409. }
  410. void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<unsigned short int> &currentfeats, const NICE::MultiChannelImageT<double> &lfeats, NICE::MultiChannelImageT<double> &integralImage )
  411. {
  412. int xsize = currentfeats.width();
  413. int ysize = currentfeats.height();
  414. int channels = ( int ) forest[0][0].dist.size();
  415. #pragma omp parallel for
  416. for ( int c = 0; c < channels; c++ )
  417. {
  418. integralImage.set ( 0, 0, getMeanProb ( 0, 0, c, currentfeats ), c );
  419. //first column
  420. for ( int y = 1; y < ysize; y++ )
  421. {
  422. integralImage.set ( 0, y, getMeanProb ( 0, y, c, currentfeats ) + integralImage.get ( 0, y - 1, c ), c );
  423. }
  424. //first row
  425. for ( int x = 1; x < xsize; x++ )
  426. {
  427. integralImage.set ( x, 0, getMeanProb ( x, 0, c, currentfeats ) + integralImage.get ( x - 1, 0, c ), c );
  428. }
  429. //rest
  430. for ( int y = 1; y < ysize; y++ )
  431. {
  432. for ( int x = 1; x < xsize; x++ )
  433. {
  434. double val = getMeanProb ( x, y, c, currentfeats ) + integralImage.get ( x, y - 1, c ) + integralImage.get ( x - 1, y, c ) - integralImage.get ( x - 1, y - 1, c );
  435. integralImage.set ( x, y, val, c );
  436. }
  437. }
  438. }
  439. int channels2 = ( int ) lfeats.channels();
  440. xsize = lfeats.width();
  441. ysize = lfeats.height();
  442. if ( integralImage.get ( xsize - 1, ysize - 1, channels ) == 0.0 )
  443. {
  444. #pragma omp parallel for
  445. for ( int c1 = 0; c1 < channels2; c1++ )
  446. {
  447. int c = channels + c1;
  448. integralImage.set ( 0, 0, lfeats.get ( 0, 0, c1 ), c );
  449. //first column
  450. for ( int y = 1; y < ysize; y++ )
  451. {
  452. integralImage.set ( 0, y, lfeats.get ( 0, y, c1 ) + integralImage.get ( 0, y, c ), c );
  453. }
  454. //first row
  455. for ( int x = 1; x < xsize; x++ )
  456. {
  457. integralImage.set ( x, 0, lfeats.get ( x, 0, c1 ) + integralImage.get ( x, 0, c ), c );
  458. }
  459. //rest
  460. for ( int y = 1; y < ysize; y++ )
  461. {
  462. for ( int x = 1; x < xsize; x++ )
  463. {
  464. double val = lfeats.get ( x, y, c1 ) + integralImage.get ( x, y - 1, c ) + integralImage.get ( x - 1, y, c ) - integralImage.get ( x - 1, y - 1, c );
  465. integralImage.set ( x, y, val, c );
  466. }
  467. }
  468. }
  469. }
  470. }
  471. inline double computeWeight ( const double &d, const double &dim )
  472. {
  473. return 1.0 / ( pow ( 2, ( double ) ( dim - d + 1 ) ) );
  474. }
  475. void SemSegContextTree::train ( const MultiDataset *md )
  476. {
  477. const LabeledSet train = * ( *md ) ["train"];
  478. const LabeledSet *trainp = &train;
  479. ProgressBar pb ( "compute feats" );
  480. pb.show();
  481. //TODO: Speichefresser!, lohnt sich sparse?
  482. vector<MultiChannelImageT<double> > allfeats;
  483. vector<MultiChannelImageT<unsigned short int> > currentfeats;
  484. vector<MatrixT<int> > labels;
  485. #ifdef TEXTONMAP
  486. vector<MultiChannelImageT<SparseVectorInt> > textonMap;
  487. #endif
  488. vector<MultiChannelImageT<SparseVectorInt> > integralTexton;
  489. std::string forbidden_classes_s = conf->gS ( "analysis", "donttrain", "" );
  490. if ( forbidden_classes_s == "" )
  491. {
  492. forbidden_classes_s = conf->gS ( "analysis", "forbidden_classes", "" );
  493. }
  494. classnames.getSelection ( forbidden_classes_s, forbidden_classes );
  495. int imgcounter = 0;
  496. int amountPixels = 0;
  497. LOOP_ALL_S ( *trainp )
  498. {
  499. EACH_INFO ( classno, info );
  500. NICE::ColorImage img;
  501. std::string currentFile = info.img();
  502. CachedExample *ce = new CachedExample ( currentFile );
  503. const LocalizationResult *locResult = info.localization();
  504. if ( locResult->size() <= 0 )
  505. {
  506. fprintf ( stderr, "WARNING: NO ground truth polygons found for %s !\n",
  507. currentFile.c_str() );
  508. continue;
  509. }
  510. fprintf ( stderr, "SemSegCsurka: Collecting pixel examples from localization info: %s\n", currentFile.c_str() );
  511. int xsize, ysize;
  512. ce->getImageSize ( xsize, ysize );
  513. amountPixels += xsize * ysize;
  514. MatrixT<int> tmpMat ( xsize, ysize );
  515. currentfeats.push_back ( MultiChannelImageT<unsigned short int> ( xsize, ysize, nbTrees ) );
  516. currentfeats[imgcounter].setAll ( 0 );
  517. #ifdef TEXTONMAP
  518. textonMap.push_back ( MultiChannelImageT<SparseVectorInt> ( xsize / grid + 1, ysize / grid + 1, 1 ) );
  519. integralTexton.push_back ( MultiChannelImageT<SparseVectorInt> ( xsize / grid + 1, ysize / grid + 1, 1 ) );
  520. #endif
  521. labels.push_back ( tmpMat );
  522. try {
  523. img = ColorImage ( currentFile );
  524. } catch ( Exception ) {
  525. cerr << "SemSeg: error opening image file <" << currentFile << ">" << endl;
  526. continue;
  527. }
  528. Globals::setCurrentImgFN ( currentFile );
  529. //TODO: resize image?!
  530. MultiChannelImageT<double> feats;
  531. allfeats.push_back ( feats );
  532. // read image and do some simple transformations
  533. extractBasicFeatures (allfeats[imgcounter], img, currentFile);
  534. // getting groundtruth
  535. NICE::Image pixelLabels ( xsize, ysize );
  536. pixelLabels.set ( 0 );
  537. locResult->calcLabeledImage ( pixelLabels, ( *classNames ).getBackgroundClass() );
  538. for ( int x = 0; x < xsize; x++ )
  539. {
  540. for ( int y = 0; y < ysize; y++ )
  541. {
  542. classno = pixelLabels.getPixel ( x, y );
  543. labels[imgcounter] ( x, y ) = classno;
  544. if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
  545. continue;
  546. labelcounter[classno]++;
  547. }
  548. }
  549. imgcounter++;
  550. pb.update ( trainp->count() );
  551. delete ce;
  552. }
  553. pb.hide();
  554. map<int, int>::iterator mapit;
  555. int classes = 0;
  556. for ( mapit = labelcounter.begin(); mapit != labelcounter.end(); mapit++ )
  557. {
  558. labelmap[mapit->first] = classes;
  559. labelmapback[classes] = mapit->first;
  560. classes++;
  561. }
  562. //balancing
  563. int featcounter = 0;
  564. a = vector<double> ( classes, 0.0 );
  565. for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
  566. {
  567. int xsize = ( int ) currentfeats[iCounter].width();
  568. int ysize = ( int ) currentfeats[iCounter].height();
  569. for ( int x = 0; x < xsize; x++ )
  570. {
  571. for ( int y = 0; y < ysize; y++ )
  572. {
  573. featcounter++;
  574. int cn = labels[iCounter] ( x, y );
  575. if ( labelmap.find ( cn ) == labelmap.end() )
  576. continue;
  577. a[labelmap[cn]] ++;
  578. }
  579. }
  580. }
  581. for ( int i = 0; i < ( int ) a.size(); i++ )
  582. {
  583. a[i] /= ( double ) featcounter;
  584. }
  585. #ifdef DEBUG
  586. for ( int i = 0; i < ( int ) a.size(); i++ )
  587. {
  588. cout << "a[" << i << "]: " << a[i] << endl;
  589. }
  590. cout << "a.size: " << a.size() << endl;
  591. #endif
  592. depth = 0;
  593. int uniquenumber = 0;
  594. for ( int t = 0; t < nbTrees; t++ )
  595. {
  596. vector<TreeNode> tree;
  597. tree.push_back ( TreeNode() );
  598. tree[0].dist = vector<double> ( classes, 0.0 );
  599. tree[0].depth = depth;
  600. tree[0].featcounter = amountPixels;
  601. tree[0].nodeNumber = uniquenumber;
  602. uniquenumber++;
  603. forest.push_back ( tree );
  604. }
  605. vector<int> startnode ( nbTrees, 0 );
  606. bool allleaf = false;
  607. //int baseFeatSize = allfeats[0].size();
  608. vector<MultiChannelImageT<double> > integralImgs ( imgcounter, MultiChannelImageT<double>() );
  609. while ( !allleaf && depth < maxDepth )
  610. {
  611. depth++;
  612. #ifdef DEBUG
  613. cout << "depth: " << depth << endl;
  614. #endif
  615. allleaf = true;
  616. vector<MultiChannelImageT<unsigned short int> > lastfeats = currentfeats;
  617. #if 1
  618. Timer timer;
  619. timer.start();
  620. #endif
  621. double weight = computeWeight ( depth, maxDepth ) - computeWeight ( depth - 1, maxDepth );
  622. if ( depth == 1 )
  623. {
  624. weight = computeWeight ( 1, maxDepth );
  625. }
  626. for ( int tree = 0; tree < nbTrees; tree++ )
  627. {
  628. int t = ( int ) forest[tree].size();
  629. int s = startnode[tree];
  630. startnode[tree] = t;
  631. //TODO vielleicht parallel wenn nächste schleife trotzdem noch parallelsiert würde, die hat mehr gewicht
  632. //#pragma omp parallel for
  633. for ( int i = s; i < t; i++ )
  634. {
  635. if ( !forest[tree][i].isleaf && forest[tree][i].left < 0 )
  636. {
  637. #if 0
  638. timer.stop();
  639. cout << "time 1: " << timer.getLast() << endl;
  640. timer.start();
  641. #endif
  642. Operation *splitfeat = NULL;
  643. double splitval;
  644. double bestig = getBestSplit ( allfeats, lastfeats, integralImgs, labels, i, splitfeat, splitval, tree );
  645. for ( int ii = 0; ii < lastfeats.size(); ii++ )
  646. {
  647. for ( int c = 0; c < lastfeats[ii].channels(); c++ )
  648. {
  649. short unsigned int minv, maxv;
  650. lastfeats[ii].statistics ( minv, maxv, c );
  651. //cout << "min: " << minv << " max: " << maxv << endl;
  652. }
  653. }
  654. #if 0
  655. timer.stop();
  656. double tl = timer.getLast();
  657. if ( tl > 10.0 )
  658. {
  659. cout << "time 2: " << tl << endl;
  660. cout << "slow split: " << splitfeat->writeInfos() << endl;
  661. getchar();
  662. }
  663. timer.start();
  664. #endif
  665. forest[tree][i].feat = splitfeat;
  666. forest[tree][i].decision = splitval;
  667. if ( splitfeat != NULL )
  668. {
  669. allleaf = false;
  670. int left = forest[tree].size();
  671. forest[tree].push_back ( TreeNode() );
  672. forest[tree].push_back ( TreeNode() );
  673. int right = left + 1;
  674. forest[tree][i].left = left;
  675. forest[tree][i].right = right;
  676. forest[tree][left].dist = vector<double> ( classes, 0.0 );
  677. forest[tree][right].dist = vector<double> ( classes, 0.0 );
  678. forest[tree][left].depth = depth;
  679. forest[tree][right].depth = depth;
  680. forest[tree][left].featcounter = 0;
  681. forest[tree][right].featcounter = 0;
  682. forest[tree][left].nodeNumber = uniquenumber;
  683. int leftu = uniquenumber;
  684. uniquenumber++;
  685. forest[tree][right].nodeNumber = uniquenumber;
  686. int rightu = uniquenumber;
  687. uniquenumber++;
  688. forest[tree][right].featcounter = 0;
  689. #if 0
  690. timer.stop();
  691. cout << "time 3: " << timer.getLast() << endl;
  692. timer.start();
  693. #endif
  694. #pragma omp parallel for
  695. for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
  696. {
  697. int xsize = currentfeats[iCounter].width();
  698. int ysize = currentfeats[iCounter].height();
  699. for ( int x = 0; x < xsize; x++ )
  700. {
  701. for ( int y = 0; y < ysize; y++ )
  702. {
  703. if ( currentfeats[iCounter].get ( x, y, tree ) == i )
  704. {
  705. Features feat;
  706. feat.feats = &allfeats[iCounter];
  707. feat.cfeats = &lastfeats[iCounter];
  708. feat.cTree = tree;
  709. feat.tree = &forest[tree];
  710. feat.integralImg = &integralImgs[iCounter];
  711. double val = splitfeat->getVal ( feat, x, y );
  712. int subx = x / grid;
  713. int suby = y / grid;
  714. #pragma omp critical
  715. if ( val < splitval )
  716. {
  717. currentfeats[iCounter].set ( x, y, left, tree );
  718. if ( labelmap.find ( labels[iCounter] ( x, y ) ) != labelmap.end() )
  719. forest[tree][left].dist[labelmap[labels[iCounter] ( x, y ) ]]++;
  720. forest[tree][left].featcounter++;
  721. SparseVectorInt v;
  722. v.insert ( pair<int, double> ( leftu, weight ) );
  723. #ifdef TEXTONMAP
  724. textonMap[iCounter] ( subx, suby ).add ( v );
  725. #endif
  726. }
  727. else
  728. {
  729. currentfeats[iCounter].set ( x, y, right, tree );
  730. if ( labelmap.find ( labels[iCounter] ( x, y ) ) != labelmap.end() )
  731. forest[tree][right].dist[labelmap[labels[iCounter] ( x, y ) ]]++;
  732. forest[tree][right].featcounter++;
  733. //feld im subsampled finden und in diesem rechts hochzählen
  734. SparseVectorInt v;
  735. v.insert ( pair<int, double> ( rightu, weight ) );
  736. #ifdef TEXTONMAP
  737. textonMap[iCounter] ( subx, suby ).add ( v );
  738. #endif
  739. }
  740. }
  741. }
  742. }
  743. }
  744. #if 0
  745. timer.stop();
  746. cout << "time 4: " << timer.getLast() << endl;
  747. timer.start();
  748. #endif
  749. // forest[tree][right].featcounter = forest[tree][i].featcounter - forest[tree][left].featcounter;
  750. double lcounter = 0.0, rcounter = 0.0;
  751. for ( uint d = 0; d < forest[tree][left].dist.size(); d++ )
  752. {
  753. if ( forbidden_classes.find ( labelmapback[d] ) != forbidden_classes.end() )
  754. {
  755. forest[tree][left].dist[d] = 0;
  756. forest[tree][right].dist[d] = 0;
  757. }
  758. else
  759. {
  760. forest[tree][left].dist[d] /= a[d];
  761. lcounter += forest[tree][left].dist[d];
  762. forest[tree][right].dist[d] /= a[d];
  763. rcounter += forest[tree][right].dist[d];
  764. }
  765. }
  766. #if 0
  767. timer.stop();
  768. cout << "time 5: " << timer.getLast() << endl;
  769. timer.start();
  770. #endif
  771. if ( lcounter <= 0 || rcounter <= 0 )
  772. {
  773. cout << "lcounter : " << lcounter << " rcounter: " << rcounter << endl;
  774. cout << "splitval: " << splitval << " splittype: " << splitfeat->writeInfos() << endl;
  775. cout << "bestig: " << bestig << endl;
  776. for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
  777. {
  778. int xsize = currentfeats[iCounter].width();
  779. int ysize = currentfeats[iCounter].height();
  780. int counter = 0;
  781. for ( int x = 0; x < xsize; x++ )
  782. {
  783. for ( int y = 0; y < ysize; y++ )
  784. {
  785. if ( lastfeats[iCounter].get ( x, y, tree ) == i )
  786. {
  787. if ( ++counter > 30 )
  788. break;
  789. Features feat;
  790. feat.feats = &allfeats[iCounter];
  791. feat.cfeats = &lastfeats[iCounter];
  792. feat.cTree = tree;
  793. feat.tree = &forest[tree];
  794. feat.integralImg = &integralImgs[iCounter];
  795. double val = splitfeat->getVal ( feat, x, y );
  796. cout << "splitval: " << splitval << " val: " << val << endl;
  797. }
  798. }
  799. }
  800. }
  801. assert ( lcounter > 0 && rcounter > 0 );
  802. }
  803. for ( uint d = 0; d < forest[tree][left].dist.size(); d++ )
  804. {
  805. forest[tree][left].dist[d] /= lcounter;
  806. forest[tree][right].dist[d] /= rcounter;
  807. }
  808. }
  809. else
  810. {
  811. forest[tree][i].isleaf = true;
  812. }
  813. }
  814. }
  815. #if 0
  816. timer.stop();
  817. cout << "time after tree: " << timer.getLast() << endl;
  818. timer.start();
  819. #endif
  820. }
  821. //compute integral image
  822. int channels = classes + allfeats[0].channels();
  823. #if 0
  824. timer.stop();
  825. cout << "time for part0: " << timer.getLast() << endl;
  826. timer.start();
  827. #endif
  828. if ( integralImgs[0].width() == 0 )
  829. {
  830. for ( int i = 0; i < imgcounter; i++ )
  831. {
  832. int xsize = allfeats[i].width();
  833. int ysize = allfeats[i].height();
  834. integralImgs[i].reInit ( xsize, ysize, channels );
  835. integralImgs[i].setAll ( 0.0 );
  836. }
  837. }
  838. #if 0
  839. timer.stop();
  840. cout << "time for part1: " << timer.getLast() << endl;
  841. timer.start();
  842. #endif
  843. #pragma omp parallel for
  844. for ( int i = 0; i < imgcounter; i++ )
  845. {
  846. computeIntegralImage ( currentfeats[i], allfeats[i], integralImgs[i] );
  847. #ifdef TEXTONMAP
  848. computeIntegralImage ( textonMap[i], integralTexton[i] );
  849. #endif
  850. }
  851. #if 1
  852. timer.stop();
  853. cout << "time for depth " << depth << ": " << timer.getLast() << endl;
  854. #endif
  855. }
  856. #ifdef WRITEGLOB
  857. ofstream outstream ( "globtrain.feat" );
  858. for ( int i = 0; i < textonMap.size(); i++ )
  859. {
  860. set<int> usedclasses;
  861. for ( uint x = 0; x < labels[i].rows(); x++ )
  862. {
  863. for ( uint y = 0; y < labels[i].cols(); y++ )
  864. {
  865. int classno = labels[i] ( x, y );
  866. if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
  867. continue;
  868. usedclasses.insert ( classno );
  869. }
  870. }
  871. cout << "labels.cols: " << labels[i].cols() << " labels.rows " << labels[i].rows() << endl;
  872. cout << "currentfeats : " << allfeats[i].width() << " allfeats[i].height(); " << allfeats[i].height() << endl;
  873. set<int>::iterator it;
  874. for ( it = usedclasses.begin() ; it != usedclasses.end(); it++ )
  875. outstream << *it << " ";
  876. outstream << endl;
  877. integralTexton[i] ( integralTexton[i].width() - 1, integralTexton[i].height() - 1 ).store ( outstream );
  878. }
  879. outstream.close();
  880. #endif
  881. cout << "uniquenumber " << uniquenumber << endl;
  882. //getchar();
  883. #ifdef DEBUG
  884. for ( int tree = 0; tree < nbTrees; tree++ )
  885. {
  886. int t = ( int ) forest[tree].size();
  887. for ( int i = 0; i < t; i++ )
  888. {
  889. printf ( "tree[%i]: left: %i, right: %i", i, forest[tree][i].left, forest[tree][i].right );
  890. if ( !forest[tree][i].isleaf && forest[tree][i].left != -1 )
  891. {
  892. cout << ", feat: " << forest[tree][i].feat->writeInfos() << " ";
  893. opOverview[forest[tree][i].feat->getOps() ]++;
  894. contextOverview[forest[tree][i].depth][ ( int ) forest[tree][i].feat->getContext() ]++;
  895. }
  896. for ( int d = 0; d < ( int ) forest[tree][i].dist.size(); d++ )
  897. {
  898. cout << " " << forest[tree][i].dist[d];
  899. }
  900. cout << endl;
  901. }
  902. }
  903. for ( uint c = 0; c < ops.size(); c++ )
  904. {
  905. cout << ops[c]->writeInfos() << ": " << opOverview[ops[c]->getOps() ] << endl;
  906. }
  907. for ( uint c = 0; c < cops.size(); c++ )
  908. {
  909. cout << cops[c]->writeInfos() << ": " << opOverview[cops[c]->getOps() ] << endl;
  910. }
  911. for ( int d = 0; d < maxDepth; d++ )
  912. {
  913. double sum = contextOverview[d][0] + contextOverview[d][1];
  914. contextOverview[d][0] /= sum;
  915. contextOverview[d][1] /= sum;
  916. cout << "depth: " << d << " woContext: " << contextOverview[d][0] << " wContext: " << contextOverview[d][1] << endl;
  917. }
  918. #endif
  919. }
  920. void SemSegContextTree::extractBasicFeatures ( NICE::MultiChannelImageT<double> &feats, const ColorImage &img, const string &currentFile)
  921. {
  922. int xsize = img.width();
  923. int ysize = img.height();
  924. //TODO: resize image?!
  925. #ifdef LOCALFEATS
  926. lfcw->getFeats ( img, feats );
  927. #else
  928. feats.reInit ( xsize, ysize, 3 );
  929. for ( int x = 0; x < xsize; x++ )
  930. {
  931. for ( int y = 0; y < ysize; y++ )
  932. {
  933. for ( int r = 0; r < 3; r++ )
  934. {
  935. feats.set ( x, y, img.getPixel ( x, y, r ), r );
  936. }
  937. }
  938. }
  939. feats = ColorSpace::rgbtolab ( feats );
  940. #endif
  941. if ( useGradient )
  942. {
  943. int currentsize = feats.channels();
  944. feats.addChannel ( currentsize );
  945. for ( int c = 0; c < currentsize; c++ )
  946. {
  947. ImageT<double> tmp = feats[c];
  948. ImageT<double> tmp2 = feats[c+currentsize];
  949. NICE::FilterT<double>::sobel ( tmp, tmp2 );
  950. }
  951. }
  952. }
  953. void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult, NICE::MultiChannelImageT<double> & probabilities )
  954. {
  955. int xsize;
  956. int ysize;
  957. ce->getImageSize ( xsize, ysize );
  958. int numClasses = classNames->numClasses();
  959. fprintf ( stderr, "ContextTree classification !\n" );
  960. probabilities.reInit ( xsize, ysize, numClasses );
  961. probabilities.setAll ( 0 );
  962. #ifdef TEXTONMAP
  963. MultiChannelImageT<SparseVectorInt> textonMap ( xsize / grid + 1, ysize / grid + 1, 1 );
  964. MultiChannelImageT<SparseVectorInt> integralTexton ( xsize / grid + 1, ysize / grid + 1, 1 );
  965. #endif
  966. std::string currentFile = Globals::getCurrentImgFN();
  967. MultiChannelImageT<double> feats;
  968. NICE::ColorImage img;
  969. try {
  970. img = ColorImage ( currentFile );
  971. } catch ( Exception ) {
  972. cerr << "SemSeg: error opening image file <" << currentFile << ">" << endl;
  973. return;
  974. }
  975. extractBasicFeatures(feats, img, currentFile); //read image and do some simple transformations
  976. bool allleaf = false;
  977. MultiChannelImageT<double> integralImg;
  978. MultiChannelImageT<unsigned short int> currentfeats ( xsize, ysize, nbTrees );
  979. currentfeats.setAll ( 0 );
  980. depth = 0;
  981. for ( int d = 0; d < maxDepth && !allleaf; d++ )
  982. {
  983. depth++;
  984. #ifdef TEXTONMAP
  985. double weight = computeWeight ( depth, maxDepth ) - computeWeight ( depth - 1, maxDepth );
  986. if ( depth == 1 )
  987. {
  988. weight = computeWeight ( 1, maxDepth );
  989. }
  990. #endif
  991. allleaf = true;
  992. MultiChannelImageT<unsigned short int> lastfeats = currentfeats;
  993. int tree;
  994. #pragma omp parallel for private(tree)
  995. for ( tree = 0; tree < nbTrees; tree++ )
  996. {
  997. for ( int x = 0; x < xsize; x++ )
  998. {
  999. for ( int y = 0; y < ysize; y++ )
  1000. {
  1001. int t = currentfeats.get ( x, y, tree );
  1002. if ( forest[tree][t].left > 0 )
  1003. {
  1004. allleaf = false;
  1005. Features feat;
  1006. feat.feats = &feats;
  1007. feat.cfeats = &lastfeats;
  1008. feat.cTree = tree;
  1009. feat.tree = &forest[tree];
  1010. feat.integralImg = &integralImg;
  1011. double val = forest[tree][t].feat->getVal ( feat, x, y );
  1012. int subx = x / grid;
  1013. int suby = y / grid;
  1014. if ( val < forest[tree][t].decision )
  1015. {
  1016. currentfeats.set ( x, y, forest[tree][t].left, tree );
  1017. #ifdef TEXTONMAP
  1018. #pragma omp critical
  1019. {
  1020. SparseVectorInt v;
  1021. v.insert ( pair<int, double> ( forest[tree][forest[tree][t].left].nodeNumber, weight ) );
  1022. textonMap ( subx, suby ).add ( v );
  1023. }
  1024. #endif
  1025. }
  1026. else
  1027. {
  1028. currentfeats.set ( x, y, forest[tree][t].right, tree );
  1029. #ifdef TEXTONMAP
  1030. #pragma omp critical
  1031. {
  1032. SparseVectorInt v;
  1033. v.insert ( pair<int, double> ( forest[tree][forest[tree][t].right].nodeNumber, weight ) );
  1034. textonMap ( subx, suby ).add ( v );
  1035. }
  1036. #endif
  1037. }
  1038. /*if ( x == xpos && y == ypos )
  1039. {
  1040. cout << "val: " << val << " decision: " << forest[tree][t].decision << " details: " << forest[tree][t].feat->writeInfos() << endl;
  1041. }*/
  1042. }
  1043. }
  1044. }
  1045. if ( depth < maxDepth )
  1046. {
  1047. //compute integral image
  1048. int channels = ( int ) labelmap.size() + feats.channels();
  1049. if ( integralImg.width() == 0 )
  1050. {
  1051. int xsize = feats.width();
  1052. int ysize = feats.height();
  1053. integralImg.reInit ( xsize, ysize, channels );
  1054. integralImg.setAll ( 0.0 );
  1055. }
  1056. }
  1057. }
  1058. if ( depth < maxDepth )
  1059. {
  1060. computeIntegralImage ( currentfeats, feats, integralImg );
  1061. #ifdef TEXTONMAP
  1062. computeIntegralImage ( textonMap, integralTexton );
  1063. #endif
  1064. }
  1065. }
  1066. // cout << forest[0][currentfeats.get ( xpos, ypos, 0 ) ].dist << endl;
  1067. #ifdef WRITEGLOB
  1068. ofstream outstream ( "globtest.feat", ofstream::app );
  1069. outstream << 0 << endl;
  1070. integralTexton ( integralTexton.width() - 1, integralTexton.height() - 1 ).store ( outstream );
  1071. outstream.close();
  1072. #endif
  1073. string cndir = conf->gS ( "SSContextTree", "cndir", "" );
  1074. int classes = ( int ) probabilities.channels();
  1075. vector<int> useclass ( classes, 1 );
  1076. #ifdef WRITEGLOB
  1077. std::vector< std::string > list;
  1078. StringTools::split ( currentFile, '/', list );
  1079. string orgname = list.back();
  1080. ofstream ostream ( "filelist.txt", ofstream::app );
  1081. ostream << orgname << ".dat" << endl;
  1082. ostream.close();
  1083. if ( cndir != "" )
  1084. {
  1085. useclass = vector<int> ( classes, 0 );
  1086. ifstream infile ( ( cndir + "/" + orgname + ".dat" ).c_str() );
  1087. #undef OLD
  1088. #ifdef OLD
  1089. while ( !infile.eof() && infile.good() )
  1090. {
  1091. int tmp;
  1092. infile >> tmp;
  1093. assert ( tmp >= 0 && tmp < classes );
  1094. useclass[tmp] = 1;
  1095. }
  1096. #else
  1097. int c = 0;
  1098. vector<double> probs ( classes, 0.0 );
  1099. while ( !infile.eof() && infile.good() )
  1100. {
  1101. infile >> probs[c];
  1102. c++;
  1103. }
  1104. vector<double> sorted = probs;
  1105. sort ( sorted.begin(), sorted.end() );
  1106. double thr = sorted[10];
  1107. if ( thr < 0.0 )
  1108. thr = 0.0;
  1109. for ( int c = 0; c < classes; c++ )
  1110. {
  1111. if ( probs[c] < thr )
  1112. {
  1113. useclass[c] = 1;
  1114. }
  1115. }
  1116. #endif
  1117. for ( int c = 0; c < classes; c++ )
  1118. {
  1119. if ( useclass[c] == 0 )
  1120. {
  1121. probabilities.set ( -numeric_limits< double >::max(), c );
  1122. }
  1123. }
  1124. }
  1125. #endif
  1126. if ( pixelWiseLabeling )
  1127. {
  1128. //finales labeln:
  1129. //long int offset = 0;
  1130. for ( int x = 0; x < xsize; x++ )
  1131. {
  1132. for ( int y = 0; y < ysize; y++ )
  1133. {
  1134. double maxvalue = - numeric_limits<double>::max(); //TODO: das kann auch nur pro knoten gemacht werden, nicht pro pixel
  1135. int maxindex = 0;
  1136. uint s = forest[0][0].dist.size();
  1137. for ( uint i = 0; i < s; i++ )
  1138. {
  1139. int currentclass = labelmapback[i];
  1140. if ( useclass[currentclass] )
  1141. {
  1142. probabilities ( x, y, currentclass ) = getMeanProb ( x, y, i, currentfeats );
  1143. if ( probabilities ( x, y, currentclass ) > maxvalue )
  1144. {
  1145. maxvalue = probabilities ( x, y, currentclass );
  1146. maxindex = currentclass;
  1147. }
  1148. }
  1149. }
  1150. segresult.setPixel ( x, y, maxindex );
  1151. if ( maxvalue > 1 )
  1152. cout << "maxvalue: " << maxvalue << endl;
  1153. }
  1154. }
  1155. #undef VISUALIZE
  1156. #ifdef VISUALIZE
  1157. for ( int j = 0 ; j < ( int ) probabilities.numChannels; j++ )
  1158. {
  1159. //cout << "class: " << j << endl;//" " << cn.text ( j ) << endl;
  1160. NICE::Matrix tmp ( probabilities.height(), probabilities.width() );
  1161. double maxval = -numeric_limits<double>::max();
  1162. double minval = numeric_limits<double>::max();
  1163. for ( int y = 0; y < probabilities.height(); y++ )
  1164. for ( int x = 0; x < probabilities.width(); x++ )
  1165. {
  1166. double val = probabilities ( x, y, j );
  1167. tmp ( y, x ) = val;
  1168. maxval = std::max ( val, maxval );
  1169. minval = std::min ( val, minval );
  1170. }
  1171. tmp ( 0, 0 ) = 1.0;
  1172. tmp ( 0, 1 ) = 0.0;
  1173. NICE::ColorImage imgrgb ( probabilities.width(), probabilities.height() );
  1174. ICETools::convertToRGB ( tmp, imgrgb );
  1175. cout << "maxval = " << maxval << " minval: " << minval << " for class " << j << endl; //cn.text ( j ) << endl;
  1176. std::string s;
  1177. std::stringstream out;
  1178. out << "tmpprebmap" << j << ".ppm";
  1179. s = out.str();
  1180. imgrgb.write ( s );
  1181. //showImage(imgrgb, "Ergebnis");
  1182. //getchar();
  1183. }
  1184. cout << "fertsch" << endl;
  1185. getchar();
  1186. cout << "weiter gehtsch" << endl;
  1187. #endif
  1188. }
  1189. else
  1190. {
  1191. //final labeling using segmentation
  1192. Matrix regions;
  1193. //showImage(img);
  1194. int regionNumber = segmentation->segRegions ( img, regions );
  1195. cout << "regions: " << regionNumber << endl;
  1196. int dSize = forest[0][0].dist.size();
  1197. vector<vector<double> > regionProbs ( regionNumber, vector<double> ( dSize, 0.0 ) );
  1198. vector<int> bestlabels ( regionNumber, 0 );
  1199. for ( int y = 0; y < img.height(); y++ )
  1200. {
  1201. for ( int x = 0; x < img.width(); x++ )
  1202. {
  1203. int cregion = regions ( x, y );
  1204. for ( int d = 0; d < dSize; d++ )
  1205. {
  1206. regionProbs[cregion][d] += getMeanProb ( x, y, d, currentfeats );
  1207. }
  1208. }
  1209. }
  1210. for ( int r = 0; r < regionNumber; r++ )
  1211. {
  1212. double maxval = regionProbs[r][0];
  1213. bestlabels[r] = 0;
  1214. for ( int d = 1; d < dSize; d++ )
  1215. {
  1216. if ( maxval < regionProbs[r][d] )
  1217. {
  1218. maxval = regionProbs[r][d];
  1219. bestlabels[r] = d;
  1220. }
  1221. }
  1222. bestlabels[r] = labelmapback[bestlabels[r]];
  1223. }
  1224. for ( int y = 0; y < img.height(); y++ )
  1225. {
  1226. for ( int x = 0; x < img.width(); x++ )
  1227. {
  1228. segresult.setPixel ( x, y, bestlabels[regions ( x,y ) ] );
  1229. }
  1230. }
  1231. #define WRITEREGIONS
  1232. #ifdef WRITEREGIONS
  1233. RegionGraph rg;
  1234. segmentation->getGraphRepresentation ( img, regions, rg );
  1235. for ( uint pos = 0; pos < regionProbs.size(); pos++ )
  1236. {
  1237. rg[pos]->setProbs ( regionProbs[pos] );
  1238. }
  1239. std::string s;
  1240. std::stringstream out;
  1241. std::vector< std::string > list;
  1242. StringTools::split ( Globals::getCurrentImgFN (), '/', list );
  1243. out << "rgout/" << list.back() << ".graph";
  1244. string writefile = out.str();
  1245. rg.write ( writefile );
  1246. #endif
  1247. }
  1248. cout << "segmentation finished" << endl;
  1249. }
  1250. void SemSegContextTree::store ( std::ostream & os, int format ) const
  1251. {
  1252. os << nbTrees << endl;
  1253. classnames.store ( os );
  1254. map<int, int>::const_iterator it;
  1255. os << labelmap.size() << endl;
  1256. for ( it = labelmap.begin() ; it != labelmap.end(); it++ )
  1257. os << ( *it ).first << " " << ( *it ).second << endl;
  1258. os << labelmapback.size() << endl;
  1259. for ( it = labelmapback.begin() ; it != labelmapback.end(); it++ )
  1260. os << ( *it ).first << " " << ( *it ).second << endl;
  1261. int trees = forest.size();
  1262. os << trees << endl;
  1263. for ( int t = 0; t < trees; t++ )
  1264. {
  1265. int nodes = forest[t].size();
  1266. os << nodes << endl;
  1267. for ( int n = 0; n < nodes; n++ )
  1268. {
  1269. os << forest[t][n].left << " " << forest[t][n].right << " " << forest[t][n].decision << " " << forest[t][n].isleaf << " " << forest[t][n].depth << " " << forest[t][n].featcounter << " " << forest[t][n].nodeNumber << endl;
  1270. os << forest[t][n].dist << endl;
  1271. if ( forest[t][n].feat == NULL )
  1272. os << -1 << endl;
  1273. else
  1274. {
  1275. os << forest[t][n].feat->getOps() << endl;
  1276. forest[t][n].feat->store ( os );
  1277. }
  1278. }
  1279. }
  1280. }
  1281. void SemSegContextTree::restore ( std::istream & is, int format )
  1282. {
  1283. is >> nbTrees;
  1284. classnames.restore ( is );
  1285. int lsize;
  1286. is >> lsize;
  1287. labelmap.clear();
  1288. for ( int l = 0; l < lsize; l++ )
  1289. {
  1290. int first, second;
  1291. is >> first;
  1292. is >> second;
  1293. labelmap[first] = second;
  1294. }
  1295. is >> lsize;
  1296. labelmapback.clear();
  1297. for ( int l = 0; l < lsize; l++ )
  1298. {
  1299. int first, second;
  1300. is >> first;
  1301. is >> second;
  1302. labelmapback[first] = second;
  1303. }
  1304. int trees;
  1305. is >> trees;
  1306. forest.clear();
  1307. for ( int t = 0; t < trees; t++ )
  1308. {
  1309. vector<TreeNode> tmptree;
  1310. forest.push_back ( tmptree );
  1311. int nodes;
  1312. is >> nodes;
  1313. //cout << "nodes: " << nodes << endl;
  1314. for ( int n = 0; n < nodes; n++ )
  1315. {
  1316. TreeNode tmpnode;
  1317. forest[t].push_back ( tmpnode );
  1318. is >> forest[t][n].left;
  1319. is >> forest[t][n].right;
  1320. is >> forest[t][n].decision;
  1321. is >> forest[t][n].isleaf;
  1322. is >> forest[t][n].depth;
  1323. is >> forest[t][n].featcounter;
  1324. is >> forest[t][n].nodeNumber;
  1325. is >> forest[t][n].dist;
  1326. int feattype;
  1327. is >> feattype;
  1328. assert ( feattype < NBOPERATIONS );
  1329. forest[t][n].feat = NULL;
  1330. if ( feattype >= 0 )
  1331. {
  1332. for ( uint o = 0; o < ops.size(); o++ )
  1333. {
  1334. if ( ops[o]->getOps() == feattype )
  1335. {
  1336. forest[t][n].feat = ops[o]->clone();
  1337. break;
  1338. }
  1339. }
  1340. if ( forest[t][n].feat == NULL )
  1341. {
  1342. for ( uint o = 0; o < cops.size(); o++ )
  1343. {
  1344. if ( cops[o]->getOps() == feattype )
  1345. {
  1346. forest[t][n].feat = cops[o]->clone();
  1347. break;
  1348. }
  1349. }
  1350. }
  1351. assert ( forest[t][n].feat != NULL );
  1352. forest[t][n].feat->restore ( is );
  1353. }
  1354. }
  1355. }
  1356. }