SemSegContextTree.cpp 61 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360
  1. #include "SemSegContextTree.h"
  2. #include "vislearning/baselib/Globals.h"
  3. #include "vislearning/baselib/ProgressBar.h"
  4. #include "core/basics/StringTools.h"
  5. #include "vislearning/cbaselib/CachedExample.h"
  6. #include "vislearning/cbaselib/PascalResults.h"
  7. #include "vislearning/baselib/ColorSpace.h"
  8. #include "objrec/segmentation/RSMeanShift.h"
  9. #include "objrec/segmentation/RSGraphBased.h"
  10. #include "core/basics/numerictools.h"
  11. #include "vislearning/baselib/ICETools.h"
  12. #include "core/basics/Timer.h"
  13. #include "core/basics/vectorio.h"
  14. #include <omp.h>
  15. #include <iostream>
  16. #define BOUND(x,min,max) (((x)<(min))?(min):((x)>(max)?(max):(x)))
  17. #undef LOCALFEATS
  18. #undef WRITEGLOB
  19. #undef TEXTONMAP
  20. //#define LOCALFEATS
  21. using namespace OBJREC;
  22. using namespace std;
  23. using namespace NICE;
  24. class MCImageAccess: public ValueAccess
  25. {
  26. public:
  27. virtual double getVal ( const Features &feats, const int &x, const int &y, const int &channel )
  28. {
  29. return feats.feats->get ( x, y, channel );
  30. }
  31. virtual string writeInfos()
  32. {
  33. return "raw";
  34. }
  35. virtual ValueTypes getType()
  36. {
  37. return RAWFEAT;
  38. }
  39. };
  40. class ClassificationResultAcess: public ValueAccess
  41. {
  42. public:
  43. virtual double getVal ( const Features &feats, const int &x, const int &y, const int &channel )
  44. {
  45. return ( *feats.tree ) [feats.cfeats->get ( x,y,feats.cTree ) ].dist[channel];
  46. }
  47. virtual string writeInfos()
  48. {
  49. return "context";
  50. }
  51. virtual ValueTypes getType()
  52. {
  53. return CONTEXT;
  54. }
  55. };
  56. class SparseImageAcess: public ValueAccess
  57. {
  58. private:
  59. double scale;
  60. public:
  61. virtual double getVal ( const Features &feats, const int &x, const int &y, const int &channel )
  62. {
  63. //MultiChannelImageT<SparseVectorInt> textonMap;
  64. //TODO: implement access
  65. return -1.0;
  66. }
  67. virtual string writeInfos()
  68. {
  69. return "context";
  70. }
  71. virtual ValueTypes getType()
  72. {
  73. return CONTEXT;
  74. }
  75. };
  76. void Operation::restore ( std::istream &is )
  77. {
  78. is >> x1;
  79. is >> x2;
  80. is >> y1;
  81. is >> y2;
  82. is >> channel1;
  83. is >> channel2;
  84. int tmp;
  85. is >> tmp;
  86. if ( tmp >= 0 )
  87. {
  88. if ( tmp == RAWFEAT )
  89. {
  90. values = new MCImageAccess();
  91. }
  92. else if ( tmp == CONTEXT )
  93. {
  94. values = new ClassificationResultAcess();
  95. }
  96. else
  97. {
  98. throw ( "no valid ValueAccess" );
  99. }
  100. }
  101. else
  102. {
  103. values = NULL;
  104. }
  105. }
  106. std::string Operation::writeInfos()
  107. {
  108. std::stringstream ss;
  109. ss << " x1: " << x1 << " y1: " << y1 << " x2: " << x2 << " y2: " << y2 << " c1: " << channel1 << " c2: " << channel2;
  110. return ss.str();
  111. }
  112. void Operation::set ( int ws, int c1size, int c2size, int c3size, bool useGaussian )
  113. {
  114. int types = 1;
  115. if ( c2size > 0 )
  116. {
  117. types++;
  118. }
  119. if ( c3size > 0 )
  120. {
  121. types++;
  122. }
  123. types = std::min(types, maxtypes);
  124. int ft = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) types );
  125. if ( ft > 0 )
  126. {
  127. ws *= 4;
  128. }
  129. if ( useGaussian )
  130. {
  131. double sigma = ( double ) ws * 2.0;
  132. x1 = randGaussDouble ( sigma ) * ( double ) ws;
  133. x2 = randGaussDouble ( sigma ) * ( double ) ws;
  134. y1 = randGaussDouble ( sigma ) * ( double ) ws;
  135. y2 = randGaussDouble ( sigma ) * ( double ) ws;
  136. }
  137. else
  138. {
  139. x1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ws ) - ws / 2;
  140. x2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ws ) - ws / 2;
  141. y1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ws ) - ws / 2;
  142. y2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ws ) - ws / 2;
  143. }
  144. if ( ft == RAWFEAT )
  145. {
  146. values = new MCImageAccess();
  147. }
  148. else if ( ft == CONTEXT )
  149. {
  150. values = new ClassificationResultAcess();
  151. }
  152. else
  153. {
  154. values = new SparseImageAcess();
  155. }
  156. }
  157. class Minus: public Operation
  158. {
  159. public:
  160. virtual double getVal ( const Features &feats, const int &x, const int &y )
  161. {
  162. int xsize, ysize;
  163. getXY ( feats, xsize, ysize );
  164. double v1 = values->getVal ( feats, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
  165. double v2 = values->getVal ( feats, BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel2 );
  166. return v1 -v2;
  167. }
  168. virtual Operation* clone()
  169. {
  170. return new Minus();
  171. }
  172. virtual string writeInfos()
  173. {
  174. string out = "Minus";
  175. if ( values != NULL )
  176. out += values->writeInfos();
  177. return out + Operation::writeInfos();
  178. }
  179. virtual OperationTypes getOps()
  180. {
  181. return MINUS;
  182. }
  183. };
  184. class MinusAbs: public Operation
  185. {
  186. public:
  187. virtual double getVal ( const Features &feats, const int &x, const int &y )
  188. {
  189. int xsize, ysize;
  190. getXY ( feats, xsize, ysize );
  191. double v1 = values->getVal ( feats, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
  192. double v2 = values->getVal ( feats, BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel2 );
  193. return abs ( v1 -v2 );
  194. }
  195. virtual Operation* clone()
  196. {
  197. return new MinusAbs();
  198. };
  199. virtual string writeInfos()
  200. {
  201. string out = "MinusAbs";
  202. if ( values != NULL )
  203. out += values->writeInfos();
  204. return out;
  205. }
  206. virtual OperationTypes getOps()
  207. {
  208. return MINUSABS;
  209. }
  210. };
  211. class Addition: public Operation
  212. {
  213. public:
  214. virtual double getVal ( const Features &feats, const int &x, const int &y )
  215. {
  216. int xsize, ysize;
  217. getXY ( feats, xsize, ysize );
  218. double v1 = values->getVal ( feats, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
  219. double v2 = values->getVal ( feats, BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize -
  220. 1 ), channel2 );
  221. return v1 + v2;
  222. }
  223. virtual Operation* clone()
  224. {
  225. return new Addition();
  226. }
  227. virtual string writeInfos()
  228. {
  229. string out = "Addition";
  230. if ( values != NULL )
  231. out += values->writeInfos();
  232. return out + Operation::writeInfos();
  233. }
  234. virtual OperationTypes getOps()
  235. {
  236. return ADDITION;
  237. }
  238. };
  239. class Only1: public Operation
  240. {
  241. public:
  242. virtual double getVal ( const Features &feats, const int &x, const int &y )
  243. {
  244. int xsize, ysize;
  245. getXY ( feats, xsize, ysize );
  246. double v1 = values->getVal ( feats, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
  247. return v1;
  248. }
  249. virtual Operation* clone()
  250. {
  251. return new Only1();
  252. }
  253. virtual string writeInfos()
  254. {
  255. string out = "Only1";
  256. if ( values != NULL )
  257. out += values->writeInfos();
  258. return out + Operation::writeInfos();
  259. }
  260. virtual OperationTypes getOps()
  261. {
  262. return ONLY1;
  263. }
  264. };
  265. class RelativeXPosition: public Operation
  266. {
  267. public:
  268. virtual double getVal ( const Features &feats, const int &x, const int &y )
  269. {
  270. int xsize, ysize;
  271. getXY ( feats, xsize, ysize );
  272. return ( double ) x / ( double ) xsize;
  273. }
  274. virtual Operation* clone()
  275. {
  276. return new RelativeXPosition();
  277. }
  278. virtual string writeInfos()
  279. {
  280. return "RelativeXPosition" + Operation::writeInfos();
  281. }
  282. virtual OperationTypes getOps()
  283. {
  284. return RELATIVEXPOSITION;
  285. }
  286. };
  287. class RelativeYPosition: public Operation
  288. {
  289. public:
  290. virtual double getVal ( const Features &feats, const int &x, const int &y )
  291. {
  292. int xsize, ysize;
  293. getXY ( feats, xsize, ysize );
  294. return ( double ) x / ( double ) xsize;
  295. }
  296. virtual Operation* clone()
  297. {
  298. return new RelativeYPosition();
  299. }
  300. virtual string writeInfos()
  301. {
  302. return "RelativeYPosition" + Operation::writeInfos();
  303. }
  304. virtual OperationTypes getOps()
  305. {
  306. return RELATIVEYPOSITION;
  307. }
  308. };
  309. // uses mean of classification in window given by (x1,y1) (x2,y2)
  310. class IntegralOps: public Operation
  311. {
  312. public:
  313. virtual void set ( int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values )
  314. {
  315. x1 = min ( _x1, _x2 );
  316. y1 = min ( _y1, _y2 );
  317. x2 = max ( _x1, _x2 );
  318. y2 = max ( _y1, _y2 );
  319. channel1 = _channel1;
  320. channel2 = _channel2;
  321. values = _values;
  322. }
  323. virtual double getVal ( const Features &feats, const int &x, const int &y )
  324. {
  325. int xsize, ysize;
  326. getXY ( feats, xsize, ysize );
  327. return computeMean ( *feats.integralImg, BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel1 );
  328. }
  329. inline double computeMean ( const NICE::MultiChannelImageT<double> &intImg, const int &uLx, const int &uLy, const int &lRx, const int &lRy, const int &chan )
  330. {
  331. double val1 = intImg.get ( uLx, uLy, chan );
  332. double val2 = intImg.get ( lRx, uLy, chan );
  333. double val3 = intImg.get ( uLx, lRy, chan );
  334. double val4 = intImg.get ( lRx, lRy, chan );
  335. double area = ( lRx - uLx ) * ( lRy - uLy );
  336. if ( area == 0 )
  337. return 0.0;
  338. return ( val1 + val4 - val2 - val3 ) / area;
  339. }
  340. virtual Operation* clone()
  341. {
  342. return new IntegralOps();
  343. }
  344. virtual string writeInfos()
  345. {
  346. return "IntegralOps" + Operation::writeInfos();
  347. }
  348. virtual OperationTypes getOps()
  349. {
  350. return INTEGRAL;
  351. }
  352. };
  353. //like a global bag of words to model the current appearance of classes in an image without local context
  354. class GlobalFeats: public IntegralOps
  355. {
  356. public:
  357. virtual double getVal ( const Features &feats, const int &x, const int &y )
  358. {
  359. int xsize, ysize;
  360. getXY ( feats, xsize, ysize );
  361. return computeMean ( *feats.integralImg, 0, 0, xsize - 1, ysize - 1, channel1 );
  362. }
  363. virtual Operation* clone()
  364. {
  365. return new GlobalFeats();
  366. }
  367. virtual string writeInfos()
  368. {
  369. return "GlobalFeats" + Operation::writeInfos();
  370. }
  371. virtual OperationTypes getOps()
  372. {
  373. return GLOBALFEATS;
  374. }
  375. };
  376. //uses mean of Integral image given by x1, y1 with current pixel as center
  377. class IntegralCenteredOps: public IntegralOps
  378. {
  379. public:
  380. virtual void set ( int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values )
  381. {
  382. x1 = abs ( _x1 );
  383. y1 = abs ( _y1 );
  384. x2 = abs ( _x2 );
  385. y2 = abs ( _y2 );
  386. channel1 = _channel1;
  387. channel2 = _channel2;
  388. values = _values;
  389. }
  390. virtual double getVal ( const Features &feats, const int &x, const int &y )
  391. {
  392. int xsize, ysize;
  393. getXY ( feats, xsize, ysize );
  394. return computeMean ( *feats.integralImg, BOUND ( x - x1, 0, xsize - 1 ), BOUND ( y - y1, 0, ysize - 1 ), BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 );
  395. }
  396. virtual Operation* clone()
  397. {
  398. return new IntegralCenteredOps();
  399. }
  400. virtual string writeInfos()
  401. {
  402. return "IntegralCenteredOps" + Operation::writeInfos();
  403. }
  404. virtual OperationTypes getOps()
  405. {
  406. return INTEGRALCENT;
  407. }
  408. };
  409. //uses different of mean of Integral image given by two windows, where (x1,y1) is the width and height of window1 and (x2,y2) of window 2
  410. class BiIntegralCenteredOps: public IntegralCenteredOps
  411. {
  412. public:
  413. virtual void set ( int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values )
  414. {
  415. x1 = min ( abs ( _x1 ), abs ( _x2 ) );
  416. y1 = min ( abs ( _y1 ), abs ( _y2 ) );
  417. x2 = max ( abs ( _x1 ), abs ( _x2 ) );
  418. y2 = max ( abs ( _y1 ), abs ( _y2 ) );
  419. channel1 = _channel1;
  420. channel2 = _channel2;
  421. values = _values;
  422. }
  423. virtual double getVal ( const Features &feats, const int &x, const int &y )
  424. {
  425. int xsize, ysize;
  426. getXY ( feats, xsize, ysize );
  427. return computeMean ( *feats.integralImg, BOUND ( x - x1, 0, xsize - 1 ), BOUND ( y - y1, 0, ysize - 1 ), BOUND ( x + x1, 0, xsize - 1 ), BOUND ( y + y1, 0, ysize - 1 ), channel1 ) - computeMean ( *feats.integralImg, BOUND ( x - x2, 0, xsize - 1 ), BOUND ( y - y2, 0, ysize - 1 ), BOUND ( x + x2, 0, xsize - 1 ), BOUND ( y + y2, 0, ysize - 1 ), channel1 );
  428. }
  429. virtual Operation* clone()
  430. {
  431. return new BiIntegralCenteredOps();
  432. }
  433. virtual string writeInfos()
  434. {
  435. return "BiIntegralCenteredOps" + Operation::writeInfos();
  436. }
  437. virtual OperationTypes getOps()
  438. {
  439. return BIINTEGRALCENT;
  440. }
  441. };
  442. /** horizontal Haar features
  443. * ++
  444. * --
  445. */
  446. class HaarHorizontal: public IntegralCenteredOps
  447. {
  448. virtual double getVal ( const Features &feats, const int &x, const int &y )
  449. {
  450. int xsize, ysize;
  451. getXY ( feats, xsize, ysize );
  452. int tlx = BOUND ( x - x1, 0, xsize - 1 );
  453. int tly = BOUND ( y - y1, 0, ysize - 1 );
  454. int lrx = BOUND ( x + x1, 0, xsize - 1 );
  455. int lry = BOUND ( y + y1, 0, ysize - 1 );
  456. return computeMean ( *feats.integralImg, tlx, tly, lrx, y, channel1 ) - computeMean ( *feats.integralImg, tlx, y, lrx, lry, channel1 );
  457. }
  458. virtual Operation* clone()
  459. {
  460. return new HaarHorizontal();
  461. }
  462. virtual string writeInfos()
  463. {
  464. return "HaarHorizontal" + Operation::writeInfos();
  465. }
  466. virtual OperationTypes getOps()
  467. {
  468. return HAARHORIZ;
  469. }
  470. };
  471. /** vertical Haar features
  472. * +-
  473. * +-
  474. */
  475. class HaarVertical: public IntegralCenteredOps
  476. {
  477. virtual double getVal ( const Features &feats, const int &x, const int &y )
  478. {
  479. int xsize, ysize;
  480. getXY ( feats, xsize, ysize );
  481. int tlx = BOUND ( x - x1, 0, xsize - 1 );
  482. int tly = BOUND ( y - y1, 0, ysize - 1 );
  483. int lrx = BOUND ( x + x1, 0, xsize - 1 );
  484. int lry = BOUND ( y + y1, 0, ysize - 1 );
  485. return computeMean ( *feats.integralImg, tlx, tly, x, lry, channel1 ) - computeMean ( *feats.integralImg, x, tly, lrx, lry, channel1 );
  486. }
  487. virtual Operation* clone()
  488. {
  489. return new HaarVertical();
  490. }
  491. virtual string writeInfos()
  492. {
  493. return "HaarVertical" + Operation::writeInfos();
  494. }
  495. virtual OperationTypes getOps()
  496. {
  497. return HAARVERT;
  498. }
  499. };
  500. /** vertical Haar features
  501. * +-
  502. * -+
  503. */
  504. class HaarDiag: public IntegralCenteredOps
  505. {
  506. virtual double getVal ( const Features &feats, const int &x, const int &y )
  507. {
  508. int xsize, ysize;
  509. getXY ( feats, xsize, ysize );
  510. int tlx = BOUND ( x - x1, 0, xsize - 1 );
  511. int tly = BOUND ( y - y1, 0, ysize - 1 );
  512. int lrx = BOUND ( x + x1, 0, xsize - 1 );
  513. int lry = BOUND ( y + y1, 0, ysize - 1 );
  514. return computeMean ( *feats.integralImg, tlx, tly, x, y, channel1 ) + computeMean ( *feats.integralImg, x, y, lrx, lry, channel1 ) - computeMean ( *feats.integralImg, tlx, y, x, lry, channel1 ) - computeMean ( *feats.integralImg, x, tly, lrx, y, channel1 );
  515. }
  516. virtual Operation* clone()
  517. {
  518. return new HaarDiag();
  519. }
  520. virtual string writeInfos()
  521. {
  522. return "HaarDiag" + Operation::writeInfos();
  523. }
  524. virtual OperationTypes getOps()
  525. {
  526. return HAARDIAG;
  527. }
  528. };
  529. /** horizontal Haar features
  530. * +++
  531. * ---
  532. * +++
  533. */
  534. class Haar3Horiz: public BiIntegralCenteredOps
  535. {
  536. virtual double getVal ( const Features &feats, const int &x, const int &y )
  537. {
  538. int xsize, ysize;
  539. getXY ( feats, xsize, ysize );
  540. int tlx = BOUND ( x - x2, 0, xsize - 1 );
  541. int tly = BOUND ( y - y2, 0, ysize - 1 );
  542. int mtly = BOUND ( y - y1, 0, ysize - 1 );
  543. int mlry = BOUND ( y + y1, 0, ysize - 1 );
  544. int lrx = BOUND ( x + x2, 0, xsize - 1 );
  545. int lry = BOUND ( y + y2, 0, ysize - 1 );
  546. return computeMean ( *feats.integralImg, tlx, tly, lrx, mtly, channel1 ) - computeMean ( *feats.integralImg, tlx, mtly, lrx, mlry, channel1 ) + computeMean ( *feats.integralImg, tlx, mlry, lrx, lry, channel1 );
  547. }
  548. virtual Operation* clone()
  549. {
  550. return new Haar3Horiz();
  551. }
  552. virtual string writeInfos()
  553. {
  554. return "Haar3Horiz" + Operation::writeInfos();
  555. }
  556. virtual OperationTypes getOps()
  557. {
  558. return HAAR3HORIZ;
  559. }
  560. };
  561. /** vertical Haar features
  562. * +-+
  563. * +-+
  564. * +-+
  565. */
  566. class Haar3Vert: public BiIntegralCenteredOps
  567. {
  568. virtual double getVal ( const Features &feats, const int &x, const int &y )
  569. {
  570. int xsize, ysize;
  571. getXY ( feats, xsize, ysize );
  572. int tlx = BOUND ( x - x2, 0, xsize - 1 );
  573. int tly = BOUND ( y - y2, 0, ysize - 1 );
  574. int mtlx = BOUND ( x - x1, 0, xsize - 1 );
  575. int mlrx = BOUND ( x + x1, 0, xsize - 1 );
  576. int lrx = BOUND ( x + x2, 0, xsize - 1 );
  577. int lry = BOUND ( y + y2, 0, ysize - 1 );
  578. return computeMean ( *feats.integralImg, tlx, tly, mtlx, lry, channel1 ) - computeMean ( *feats.integralImg, mtlx, tly, mlrx, lry, channel1 ) + computeMean ( *feats.integralImg, mlrx, tly, lrx, lry, channel1 );
  579. }
  580. virtual Operation* clone()
  581. {
  582. return new Haar3Vert();
  583. }
  584. virtual string writeInfos()
  585. {
  586. return "Haar3Vert" + Operation::writeInfos();
  587. }
  588. virtual OperationTypes getOps()
  589. {
  590. return HAAR3VERT;
  591. }
  592. };
  593. SemSegContextTree::SemSegContextTree ( const Config *conf, const MultiDataset *md )
  594. : SemanticSegmentation ( conf, & ( md->getClassNames ( "train" ) ) )
  595. {
  596. this->conf = conf;
  597. string section = "SSContextTree";
  598. lfcw = new LFColorWeijer ( conf );
  599. grid = conf->gI ( section, "grid", 10 );
  600. maxSamples = conf->gI ( section, "max_samples", 2000 );
  601. minFeats = conf->gI ( section, "min_feats", 50 );
  602. maxDepth = conf->gI ( section, "max_depth", 10 );
  603. windowSize = conf->gI ( section, "window_size", 16 );
  604. featsPerSplit = conf->gI ( section, "feats_per_split", 200 );
  605. useShannonEntropy = conf->gB ( section, "use_shannon_entropy", true );
  606. nbTrees = conf->gI ( section, "amount_trees", 1 );
  607. string segmentationtype = conf->gS ( section, "segmentation_type", "meanshift" );
  608. useGaussian = conf->gB ( section, "use_gaussian", true );
  609. randomTests = conf->gI ( section, "random_tests", 10 );
  610. bool saveLoadData = conf->gB ( "debug", "save_load_data", false );
  611. string fileLocation = conf->gS ( "debug", "datafile", "tmp.txt" );
  612. if ( useGaussian )
  613. throw ( "there something wrong with using gaussian! first fix it!" );
  614. pixelWiseLabeling = false;
  615. if ( segmentationtype == "meanshift" )
  616. segmentation = new RSMeanShift ( conf );
  617. else if ( segmentationtype == "none" )
  618. {
  619. segmentation = NULL;
  620. pixelWiseLabeling = true;
  621. }
  622. else if ( segmentationtype == "felzenszwalb" )
  623. segmentation = new RSGraphBased ( conf );
  624. else
  625. throw ( "no valid segmenation_type\n please choose between none, meanshift and felzenszwalb\n" );
  626. ftypes = conf->gI ( section, "features", 2 );;
  627. string featsec = "Features";
  628. if ( conf->gB ( featsec, "minus", true ) )
  629. ops.push_back ( new Minus() );
  630. if ( conf->gB ( featsec, "minus_abs", true ) )
  631. ops.push_back ( new MinusAbs() );
  632. if ( conf->gB ( featsec, "addition", true ) )
  633. ops.push_back ( new Addition() );
  634. if ( conf->gB ( featsec, "only1", true ) )
  635. ops.push_back ( new Only1() );
  636. if ( conf->gB ( featsec, "rel_x", true ) )
  637. ops.push_back ( new RelativeXPosition() );
  638. if ( conf->gB ( featsec, "rel_y", true ) )
  639. ops.push_back ( new RelativeYPosition() );
  640. if ( conf->gB ( featsec, "bi_int_cent", true ) )
  641. cops.push_back ( new BiIntegralCenteredOps() );
  642. if ( conf->gB ( featsec, "int_cent", true ) )
  643. cops.push_back ( new IntegralCenteredOps() );
  644. if ( conf->gB ( featsec, "int", true ) )
  645. cops.push_back ( new IntegralOps() );
  646. if ( conf->gB ( featsec, "haar_horz", true ) )
  647. cops.push_back ( new HaarHorizontal() );
  648. if ( conf->gB ( featsec, "haar_vert", true ) )
  649. cops.push_back ( new HaarVertical() );
  650. if ( conf->gB ( featsec, "haar_diag", true ) )
  651. cops.push_back ( new HaarDiag() );
  652. if ( conf->gB ( featsec, "haar3_horz", true ) )
  653. cops.push_back ( new Haar3Horiz() );
  654. if ( conf->gB ( featsec, "haar3_vert", true ) )
  655. cops.push_back ( new Haar3Vert() );
  656. if ( conf->gB ( featsec, "glob", true ) )
  657. cops.push_back ( new GlobalFeats() );
  658. opOverview = vector<int> ( NBOPERATIONS, 0 );
  659. contextOverview = vector<vector<double> > ( maxDepth, vector<double> ( 2, 0.0 ) );
  660. calcVal.push_back ( new MCImageAccess() );
  661. calcVal.push_back ( new ClassificationResultAcess() );
  662. classnames = md->getClassNames ( "train" );
  663. ///////////////////////////////////
  664. // Train Segmentation Context Trees
  665. ///////////////////////////////////
  666. if ( saveLoadData )
  667. {
  668. if ( FileMgt::fileExists ( fileLocation ) )
  669. read ( fileLocation );
  670. else
  671. {
  672. train ( md );
  673. write ( fileLocation );
  674. }
  675. }
  676. else
  677. {
  678. train ( md );
  679. }
  680. }
  681. SemSegContextTree::~SemSegContextTree()
  682. {
  683. }
  684. double SemSegContextTree::getBestSplit ( std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<NICE::MultiChannelImageT<unsigned short int> > &currentfeats, std::vector<NICE::MultiChannelImageT<double> > &integralImgs, const std::vector<NICE::MatrixT<int> > &labels, int node, Operation *&splitop, double &splitval, const int &tree )
  685. {
  686. Timer t;
  687. t.start();
  688. int imgCount = 0, featdim = 0;
  689. try
  690. {
  691. imgCount = ( int ) feats.size();
  692. featdim = feats[0].channels();
  693. }
  694. catch ( Exception )
  695. {
  696. cerr << "no features computed?" << endl;
  697. }
  698. double bestig = -numeric_limits< double >::max();
  699. splitop = NULL;
  700. splitval = -1.0;
  701. set<vector<int> >selFeats;
  702. map<int, int> e;
  703. int featcounter = forest[tree][node].featcounter;
  704. if ( featcounter < minFeats )
  705. {
  706. //cout << "only " << featcounter << " feats in current node -> it's a leaf" << endl;
  707. return 0.0;
  708. }
  709. vector<double> fraction ( a.size(), 0.0 );
  710. for ( uint i = 0; i < fraction.size(); i++ )
  711. {
  712. if ( forbidden_classes.find ( labelmapback[i] ) != forbidden_classes.end() )
  713. fraction[i] = 0;
  714. else
  715. fraction[i] = ( ( double ) maxSamples ) / ( ( double ) featcounter * a[i] * a.size() );
  716. //cout << "fraction["<<i<<"]: "<< fraction[i] << " a[" << i << "]: " << a[i] << endl;
  717. }
  718. featcounter = 0;
  719. for ( int iCounter = 0; iCounter < imgCount; iCounter++ )
  720. {
  721. int xsize = ( int ) currentfeats[iCounter].width();
  722. int ysize = ( int ) currentfeats[iCounter].height();
  723. for ( int x = 0; x < xsize; x++ )
  724. {
  725. for ( int y = 0; y < ysize; y++ )
  726. {
  727. if ( currentfeats[iCounter].get ( x, y, tree ) == node )
  728. {
  729. int cn = labels[iCounter] ( x, y );
  730. double randD = ( double ) rand() / ( double ) RAND_MAX;
  731. if ( labelmap.find ( cn ) == labelmap.end() )
  732. continue;
  733. if ( randD < fraction[labelmap[cn]] )
  734. {
  735. vector<int> tmp ( 3, 0 );
  736. tmp[0] = iCounter;
  737. tmp[1] = x;
  738. tmp[2] = y;
  739. featcounter++;
  740. selFeats.insert ( tmp );
  741. e[cn]++;
  742. }
  743. }
  744. }
  745. }
  746. }
  747. //cout << "size: " << selFeats.size() << endl;
  748. //getchar();
  749. map<int, int>::iterator mapit;
  750. double globent = 0.0;
  751. for ( mapit = e.begin() ; mapit != e.end(); mapit++ )
  752. {
  753. //cout << "class: " << mapit->first << ": " << mapit->second << endl;
  754. double p = ( double ) ( *mapit ).second / ( double ) featcounter;
  755. globent += p * log2 ( p );
  756. }
  757. globent = -globent;
  758. if ( globent < 0.5 )
  759. {
  760. //cout << "globent to small: " << globent << endl;
  761. return 0.0;
  762. }
  763. int classes = ( int ) forest[tree][0].dist.size();
  764. featsel.clear();
  765. for ( int i = 0; i < featsPerSplit; i++ )
  766. {
  767. int x1, x2, y1, y2;
  768. int ft = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ftypes );
  769. int tmpws = windowSize;
  770. if ( integralImgs[0].width() == 0 )
  771. ft = 0;
  772. if ( ft > 0 )
  773. {
  774. tmpws *= 4;
  775. }
  776. if ( useGaussian )
  777. {
  778. double sigma = ( double ) tmpws * 2.0;
  779. x1 = randGaussDouble ( sigma ) * ( double ) tmpws;
  780. x2 = randGaussDouble ( sigma ) * ( double ) tmpws;
  781. y1 = randGaussDouble ( sigma ) * ( double ) tmpws;
  782. y2 = randGaussDouble ( sigma ) * ( double ) tmpws;
  783. }
  784. else
  785. {
  786. x1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  787. x2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  788. y1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  789. y2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) tmpws ) - tmpws / 2;
  790. }
  791. if ( ft == 0 )
  792. {
  793. int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) featdim );
  794. int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) featdim );
  795. int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) ops.size() );
  796. Operation *op = ops[o]->clone();
  797. op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
  798. op->setContext ( false );
  799. featsel.push_back ( op );
  800. }
  801. else if ( ft == 1 )
  802. {
  803. int opssize = ( int ) ops.size();
  804. //opssize = 0;
  805. int o = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( ( ( double ) cops.size() ) + ( double ) opssize ) );
  806. Operation *op;
  807. if ( o < opssize )
  808. {
  809. int chans = ( int ) forest[0][0].dist.size();
  810. int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  811. int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  812. op = ops[o]->clone();
  813. op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
  814. op->setContext ( true );
  815. }
  816. else
  817. {
  818. int chans = integralImgs[0].channels();
  819. int f1 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  820. int f2 = ( int ) ( ( double ) rand() / ( double ) RAND_MAX * ( double ) chans );
  821. o -= opssize;
  822. op = cops[o]->clone();
  823. op->set ( x1, y1, x2, y2, f1, f2, calcVal[ft] );
  824. if ( f1 < forest[0][0].dist.size() )
  825. op->setContext ( true );
  826. else
  827. op->setContext ( false );
  828. }
  829. featsel.push_back ( op );
  830. }
  831. }
  832. #pragma omp parallel for private(mapit)
  833. for ( int f = 0; f < featsPerSplit; f++ )
  834. {
  835. double l_bestig = -numeric_limits< double >::max();
  836. double l_splitval = -1.0;
  837. set<vector<int> >::iterator it;
  838. vector<double> vals;
  839. double maxval = -numeric_limits<double>::max();
  840. double minval = numeric_limits<double>::max();
  841. for ( it = selFeats.begin() ; it != selFeats.end(); it++ )
  842. {
  843. Features feat;
  844. feat.feats = &feats[ ( *it ) [0]];
  845. feat.cfeats = &currentfeats[ ( *it ) [0]];
  846. feat.cTree = tree;
  847. feat.tree = &forest[tree];
  848. feat.integralImg = &integralImgs[ ( *it ) [0]];
  849. double val = featsel[f]->getVal ( feat, ( *it ) [1], ( *it ) [2] );
  850. vals.push_back ( val );
  851. maxval = std::max ( val, maxval );
  852. minval = std::min ( val, minval );
  853. }
  854. if ( minval == maxval )
  855. continue;
  856. double scale = maxval - minval;
  857. vector<double> splits;
  858. for ( int r = 0; r < randomTests; r++ )
  859. {
  860. splits.push_back ( ( ( double ) rand() / ( double ) RAND_MAX*scale ) + minval );
  861. }
  862. for ( int run = 0 ; run < randomTests; run++ )
  863. {
  864. set<vector<int> >::iterator it2;
  865. double val = splits[run];
  866. map<int, int> eL, eR;
  867. int counterL = 0, counterR = 0;
  868. int counter2 = 0;
  869. for ( it2 = selFeats.begin() ; it2 != selFeats.end(); it2++, counter2++ )
  870. {
  871. int cn = labels[ ( *it2 ) [0]] ( ( *it2 ) [1], ( *it2 ) [2] );
  872. //cout << "vals[counter2] " << vals[counter2] << " val: " << val << endl;
  873. if ( vals[counter2] < val )
  874. {
  875. //left entropie:
  876. eL[cn] = eL[cn] + 1;
  877. counterL++;
  878. }
  879. else
  880. {
  881. //right entropie:
  882. eR[cn] = eR[cn] + 1;
  883. counterR++;
  884. }
  885. }
  886. double leftent = 0.0;
  887. for ( mapit = eL.begin() ; mapit != eL.end(); mapit++ )
  888. {
  889. double p = ( double ) ( *mapit ).second / ( double ) counterL;
  890. leftent -= p * log2 ( p );
  891. }
  892. double rightent = 0.0;
  893. for ( mapit = eR.begin() ; mapit != eR.end(); mapit++ )
  894. {
  895. double p = ( double ) ( *mapit ).second / ( double ) counterR;
  896. rightent -= p * log2 ( p );
  897. }
  898. //cout << "rightent: " << rightent << " leftent: " << leftent << endl;
  899. double pl = ( double ) counterL / ( double ) ( counterL + counterR );
  900. double ig = globent - ( 1.0 - pl ) * rightent - pl * leftent;
  901. //double ig = globent - rightent - leftent;
  902. if ( useShannonEntropy )
  903. {
  904. double esplit = - ( pl * log ( pl ) + ( 1 - pl ) * log ( 1 - pl ) );
  905. ig = 2 * ig / ( globent + esplit );
  906. }
  907. if ( ig > l_bestig )
  908. {
  909. l_bestig = ig;
  910. l_splitval = val;
  911. }
  912. }
  913. #pragma omp critical
  914. {
  915. //cout << "globent: " << globent << " bestig " << bestig << " splitfeat: " << splitfeat << " splitval: " << splitval << endl;
  916. //cout << "globent: " << globent << " l_bestig " << l_bestig << " f: " << p << " l_splitval: " << l_splitval << endl;
  917. //cout << "p: " << featsubset[f] << endl;
  918. if ( l_bestig > bestig )
  919. {
  920. bestig = l_bestig;
  921. splitop = featsel[f];
  922. splitval = l_splitval;
  923. }
  924. }
  925. }
  926. //getchar();
  927. //splitop->writeInfos();
  928. //cout<< "ig: " << bestig << endl;
  929. //FIXME: delete all features!
  930. /*for(int i = 0; i < featsPerSplit; i++)
  931. {
  932. if(featsel[i] != splitop)
  933. delete featsel[i];
  934. }*/
  935. #ifdef debug
  936. cout << "globent: " << globent << " bestig " << bestig << " splitval: " << splitval << endl;
  937. #endif
  938. return bestig;
  939. }
  940. inline double SemSegContextTree::getMeanProb ( const int &x, const int &y, const int &channel, const MultiChannelImageT<unsigned short int> &currentfeats )
  941. {
  942. double val = 0.0;
  943. for ( int tree = 0; tree < nbTrees; tree++ )
  944. {
  945. val += forest[tree][currentfeats.get ( x,y,tree ) ].dist[channel];
  946. }
  947. return val / ( double ) nbTrees;
  948. }
  949. void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<SparseVectorInt> &infeats, NICE::MultiChannelImageT<SparseVectorInt> &integralImage )
  950. {
  951. int xsize = infeats.width();
  952. int ysize = infeats.height();
  953. integralImage ( 0, 0 ).add ( infeats.get ( 0, 0 ) );
  954. //first column
  955. for ( int y = 1; y < ysize; y++ )
  956. {
  957. integralImage ( 0, y ).add ( infeats.get ( 0, y ) );
  958. integralImage ( 0, y ).add ( integralImage ( 0, y - 1 ) );
  959. }
  960. //first row
  961. for ( int x = 1; x < xsize; x++ )
  962. {
  963. integralImage ( x, 0 ).add ( infeats.get ( x, 0 ) );
  964. integralImage ( x, 0 ).add ( integralImage ( x - 1, 0 ) );
  965. }
  966. //rest
  967. for ( int y = 1; y < ysize; y++ )
  968. {
  969. for ( int x = 1; x < xsize; x++ )
  970. {
  971. integralImage ( x, y ).add ( infeats.get ( x, y ) );
  972. integralImage ( x, y ).add ( integralImage ( x, y - 1 ) );
  973. integralImage ( x, y ).add ( integralImage ( x - 1, y ) );
  974. integralImage ( x, y ).sub ( integralImage ( x - 1, y - 1 ) );
  975. }
  976. }
  977. }
  978. void SemSegContextTree::computeIntegralImage ( const NICE::MultiChannelImageT<unsigned short int> &currentfeats, const NICE::MultiChannelImageT<double> &lfeats, NICE::MultiChannelImageT<double> &integralImage )
  979. {
  980. int xsize = currentfeats.width();
  981. int ysize = currentfeats.height();
  982. int channels = ( int ) forest[0][0].dist.size();
  983. #pragma omp parallel for
  984. for ( int c = 0; c < channels; c++ )
  985. {
  986. integralImage.set ( 0, 0, getMeanProb ( 0, 0, c, currentfeats ), c );
  987. //first column
  988. for ( int y = 1; y < ysize; y++ )
  989. {
  990. integralImage.set ( 0, y, getMeanProb ( 0, y, c, currentfeats ) + integralImage.get ( 0, y - 1, c ), c );
  991. }
  992. //first row
  993. for ( int x = 1; x < xsize; x++ )
  994. {
  995. integralImage.set ( x, 0, getMeanProb ( x, 0, c, currentfeats ) + integralImage.get ( x - 1, 0, c ), c );
  996. }
  997. //rest
  998. for ( int y = 1; y < ysize; y++ )
  999. {
  1000. for ( int x = 1; x < xsize; x++ )
  1001. {
  1002. double val = getMeanProb ( x, y, c, currentfeats ) + integralImage.get ( x, y - 1, c ) + integralImage.get ( x - 1, y, c ) - integralImage.get ( x - 1, y - 1, c );
  1003. integralImage.set ( x, y, val, c );
  1004. }
  1005. }
  1006. }
  1007. int channels2 = ( int ) lfeats.channels();
  1008. xsize = lfeats.width();
  1009. ysize = lfeats.height();
  1010. if ( integralImage.get ( xsize - 1, ysize - 1, channels ) == 0.0 )
  1011. {
  1012. #pragma omp parallel for
  1013. for ( int c1 = 0; c1 < channels2; c1++ )
  1014. {
  1015. int c = channels + c1;
  1016. integralImage.set ( 0, 0, lfeats.get ( 0, 0, c1 ), c );
  1017. //first column
  1018. for ( int y = 1; y < ysize; y++ )
  1019. {
  1020. integralImage.set ( 0, y, lfeats.get ( 0, y, c1 ) + integralImage.get ( 0, y, c ), c );
  1021. }
  1022. //first row
  1023. for ( int x = 1; x < xsize; x++ )
  1024. {
  1025. integralImage.set ( x, 0, lfeats.get ( x, 0, c1 ) + integralImage.get ( x, 0, c ), c );
  1026. }
  1027. //rest
  1028. for ( int y = 1; y < ysize; y++ )
  1029. {
  1030. for ( int x = 1; x < xsize; x++ )
  1031. {
  1032. double val = lfeats.get ( x, y, c1 ) + integralImage.get ( x, y - 1, c ) + integralImage.get ( x - 1, y, c ) - integralImage.get ( x - 1, y - 1, c );
  1033. integralImage.set ( x, y, val, c );
  1034. }
  1035. }
  1036. }
  1037. }
  1038. }
  1039. inline double computeWeight(const double &d, const double &dim)
  1040. {
  1041. return 1.0/(pow(2,(double)(dim-d+1)));
  1042. }
  1043. void SemSegContextTree::train ( const MultiDataset *md )
  1044. {
  1045. const LabeledSet train = * ( *md ) ["train"];
  1046. const LabeledSet *trainp = &train;
  1047. ProgressBar pb ( "compute feats" );
  1048. pb.show();
  1049. //TODO: Speichefresser!, lohnt sich sparse?
  1050. vector<MultiChannelImageT<double> > allfeats;
  1051. vector<MultiChannelImageT<unsigned short int> > currentfeats;
  1052. vector<MatrixT<int> > labels;
  1053. #ifdef TEXTONMAP
  1054. vector<MultiChannelImageT<SparseVectorInt> > textonMap;
  1055. #endif
  1056. vector<MultiChannelImageT<SparseVectorInt> > integralTexton;
  1057. std::string forbidden_classes_s = conf->gS ( "analysis", "donttrain", "" );
  1058. if ( forbidden_classes_s == "" )
  1059. {
  1060. forbidden_classes_s = conf->gS ( "analysis", "forbidden_classes", "" );
  1061. }
  1062. classnames.getSelection ( forbidden_classes_s, forbidden_classes );
  1063. int imgcounter = 0;
  1064. int amountPixels = 0;
  1065. LOOP_ALL_S ( *trainp )
  1066. {
  1067. EACH_INFO ( classno, info );
  1068. NICE::ColorImage img;
  1069. std::string currentFile = info.img();
  1070. CachedExample *ce = new CachedExample ( currentFile );
  1071. const LocalizationResult *locResult = info.localization();
  1072. if ( locResult->size() <= 0 )
  1073. {
  1074. fprintf ( stderr, "WARNING: NO ground truth polygons found for %s !\n",
  1075. currentFile.c_str() );
  1076. continue;
  1077. }
  1078. fprintf ( stderr, "SemSegCsurka: Collecting pixel examples from localization info: %s\n", currentFile.c_str() );
  1079. int xsize, ysize;
  1080. ce->getImageSize ( xsize, ysize );
  1081. amountPixels += xsize * ysize;
  1082. MatrixT<int> tmpMat ( xsize, ysize );
  1083. currentfeats.push_back ( MultiChannelImageT<unsigned short int> ( xsize, ysize, nbTrees ) );
  1084. currentfeats[imgcounter].setAll ( 0 );
  1085. #ifdef TEXTONMAP
  1086. textonMap.push_back ( MultiChannelImageT<SparseVectorInt> ( xsize / grid + 1, ysize / grid + 1, 1 ));
  1087. integralTexton.push_back ( MultiChannelImageT<SparseVectorInt> ( xsize / grid + 1, ysize / grid + 1, 1 ));
  1088. #endif
  1089. labels.push_back ( tmpMat );
  1090. try {
  1091. img = ColorImage ( currentFile );
  1092. } catch ( Exception ) {
  1093. cerr << "SemSeg: error opening image file <" << currentFile << ">" << endl;
  1094. continue;
  1095. }
  1096. Globals::setCurrentImgFN ( currentFile );
  1097. //TODO: resize image?!
  1098. MultiChannelImageT<double> feats;
  1099. allfeats.push_back ( feats );
  1100. #ifdef LOCALFEATS
  1101. lfcw->getFeats ( img, allfeats[imgcounter] );
  1102. #else
  1103. allfeats[imgcounter].reInit ( xsize, ysize, 3, true );
  1104. for ( int x = 0; x < xsize; x++ )
  1105. {
  1106. for ( int y = 0; y < ysize; y++ )
  1107. {
  1108. for ( int r = 0; r < 3; r++ )
  1109. {
  1110. allfeats[imgcounter].set ( x, y, img.getPixel ( x, y, r ), r );
  1111. }
  1112. }
  1113. }
  1114. allfeats[imgcounter] = ColorSpace::rgbtolab ( allfeats[imgcounter] );
  1115. #endif
  1116. // getting groundtruth
  1117. NICE::Image pixelLabels ( xsize, ysize );
  1118. pixelLabels.set ( 0 );
  1119. locResult->calcLabeledImage ( pixelLabels, ( *classNames ).getBackgroundClass() );
  1120. for ( int x = 0; x < xsize; x++ )
  1121. {
  1122. for ( int y = 0; y < ysize; y++ )
  1123. {
  1124. classno = pixelLabels.getPixel ( x, y );
  1125. labels[imgcounter] ( x, y ) = classno;
  1126. if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
  1127. continue;
  1128. labelcounter[classno]++;
  1129. }
  1130. }
  1131. imgcounter++;
  1132. pb.update ( trainp->count() );
  1133. delete ce;
  1134. }
  1135. pb.hide();
  1136. map<int, int>::iterator mapit;
  1137. int classes = 0;
  1138. for ( mapit = labelcounter.begin(); mapit != labelcounter.end(); mapit++ )
  1139. {
  1140. labelmap[mapit->first] = classes;
  1141. labelmapback[classes] = mapit->first;
  1142. classes++;
  1143. }
  1144. //balancing
  1145. int featcounter = 0;
  1146. a = vector<double> ( classes, 0.0 );
  1147. for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
  1148. {
  1149. int xsize = ( int ) currentfeats[iCounter].width();
  1150. int ysize = ( int ) currentfeats[iCounter].height();
  1151. for ( int x = 0; x < xsize; x++ )
  1152. {
  1153. for ( int y = 0; y < ysize; y++ )
  1154. {
  1155. featcounter++;
  1156. int cn = labels[iCounter] ( x, y );
  1157. if ( labelmap.find ( cn ) == labelmap.end() )
  1158. continue;
  1159. a[labelmap[cn]] ++;
  1160. }
  1161. }
  1162. }
  1163. for ( int i = 0; i < ( int ) a.size(); i++ )
  1164. {
  1165. a[i] /= ( double ) featcounter;
  1166. }
  1167. #ifdef DEBUG
  1168. for ( int i = 0; i < ( int ) a.size(); i++ )
  1169. {
  1170. cout << "a[" << i << "]: " << a[i] << endl;
  1171. }
  1172. cout << "a.size: " << a.size() << endl;
  1173. #endif
  1174. depth = 0;
  1175. int uniquenumber = 0;
  1176. for ( int t = 0; t < nbTrees; t++ )
  1177. {
  1178. vector<TreeNode> tree;
  1179. tree.push_back ( TreeNode() );
  1180. tree[0].dist = vector<double> ( classes, 0.0 );
  1181. tree[0].depth = depth;
  1182. tree[0].featcounter = amountPixels;
  1183. tree[0].nodeNumber = uniquenumber;
  1184. uniquenumber++;
  1185. forest.push_back ( tree );
  1186. }
  1187. vector<int> startnode ( nbTrees, 0 );
  1188. bool allleaf = false;
  1189. //int baseFeatSize = allfeats[0].size();
  1190. vector<MultiChannelImageT<double> > integralImgs ( imgcounter, MultiChannelImageT<double>() );
  1191. while ( !allleaf && depth < maxDepth )
  1192. {
  1193. depth++;
  1194. #ifdef DEBUG
  1195. cout << "depth: " << depth << endl;
  1196. #endif
  1197. allleaf = true;
  1198. vector<MultiChannelImageT<unsigned short int> > lastfeats = currentfeats;
  1199. #if 1
  1200. Timer timer;
  1201. timer.start();
  1202. #endif
  1203. double weight = computeWeight(depth,maxDepth) - computeWeight(depth-1,maxDepth);
  1204. if(depth == 1)
  1205. {
  1206. weight = computeWeight(1,maxDepth);
  1207. }
  1208. for ( int tree = 0; tree < nbTrees; tree++ )
  1209. {
  1210. int t = ( int ) forest[tree].size();
  1211. int s = startnode[tree];
  1212. startnode[tree] = t;
  1213. //TODO vielleicht parallel wenn nächste schleife trotzdem noch parallelsiert würde, die hat mehr gewicht
  1214. //#pragma omp parallel for
  1215. for ( int i = s; i < t; i++ )
  1216. {
  1217. if ( !forest[tree][i].isleaf && forest[tree][i].left < 0 )
  1218. {
  1219. #if 0
  1220. timer.stop();
  1221. cout << "time 1: " << timer.getLast() << endl;
  1222. timer.start();
  1223. #endif
  1224. Operation *splitfeat = NULL;
  1225. double splitval;
  1226. double bestig = getBestSplit ( allfeats, lastfeats, integralImgs, labels, i, splitfeat, splitval, tree );
  1227. #if 0
  1228. timer.stop();
  1229. double tl = timer.getLast();
  1230. if ( tl > 10.0 )
  1231. {
  1232. cout << "time 2: " << tl << endl;
  1233. cout << "slow split: " << splitfeat->writeInfos() << endl;
  1234. getchar();
  1235. }
  1236. timer.start();
  1237. #endif
  1238. forest[tree][i].feat = splitfeat;
  1239. forest[tree][i].decision = splitval;
  1240. if ( splitfeat != NULL )
  1241. {
  1242. allleaf = false;
  1243. int left = forest[tree].size();
  1244. forest[tree].push_back ( TreeNode() );
  1245. forest[tree].push_back ( TreeNode() );
  1246. int right = left + 1;
  1247. forest[tree][i].left = left;
  1248. forest[tree][i].right = right;
  1249. forest[tree][left].dist = vector<double> ( classes, 0.0 );
  1250. forest[tree][right].dist = vector<double> ( classes, 0.0 );
  1251. forest[tree][left].depth = depth;
  1252. forest[tree][right].depth = depth;
  1253. forest[tree][left].featcounter = 0;
  1254. forest[tree][right].featcounter = 0;
  1255. forest[tree][left].nodeNumber = uniquenumber;
  1256. int leftu = uniquenumber;
  1257. uniquenumber++;
  1258. forest[tree][right].nodeNumber = uniquenumber;
  1259. int rightu = uniquenumber;
  1260. uniquenumber++;
  1261. forest[tree][right].featcounter = 0;
  1262. #if 0
  1263. timer.stop();
  1264. cout << "time 3: " << timer.getLast() << endl;
  1265. timer.start();
  1266. #endif
  1267. #pragma omp parallel for
  1268. for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
  1269. {
  1270. int xsize = currentfeats[iCounter].width();
  1271. int ysize = currentfeats[iCounter].height();
  1272. for ( int x = 0; x < xsize; x++ )
  1273. {
  1274. for ( int y = 0; y < ysize; y++ )
  1275. {
  1276. if ( currentfeats[iCounter].get ( x, y, tree ) == i )
  1277. {
  1278. Features feat;
  1279. feat.feats = &allfeats[iCounter];
  1280. feat.cfeats = &lastfeats[iCounter];
  1281. feat.cTree = tree;
  1282. feat.tree = &forest[tree];
  1283. feat.integralImg = &integralImgs[iCounter];
  1284. double val = splitfeat->getVal ( feat, x, y );
  1285. int subx = x / grid;
  1286. int suby = y / grid;
  1287. #pragma omp critical
  1288. if ( val < splitval )
  1289. {
  1290. currentfeats[iCounter].set ( x, y, left, tree );
  1291. if ( labelmap.find ( labels[iCounter] ( x, y ) ) != labelmap.end() )
  1292. forest[tree][left].dist[labelmap[labels[iCounter] ( x, y ) ]]++;
  1293. forest[tree][left].featcounter++;
  1294. SparseVectorInt v;
  1295. v.insert ( pair<int, double> ( leftu, weight ) );
  1296. #ifdef TEXTONMAP
  1297. textonMap[iCounter] ( subx, suby ).add ( v );
  1298. #endif
  1299. }
  1300. else
  1301. {
  1302. currentfeats[iCounter].set ( x, y, right, tree );
  1303. if ( labelmap.find ( labels[iCounter] ( x, y ) ) != labelmap.end() )
  1304. forest[tree][right].dist[labelmap[labels[iCounter] ( x, y ) ]]++;
  1305. forest[tree][right].featcounter++;
  1306. //feld im subsampled finden und in diesem rechts hochzählen
  1307. SparseVectorInt v;
  1308. v.insert ( pair<int, double> ( rightu, weight ) );
  1309. #ifdef TEXTONMAP
  1310. textonMap[iCounter] ( subx, suby ).add ( v );
  1311. #endif
  1312. }
  1313. }
  1314. }
  1315. }
  1316. }
  1317. #if 0
  1318. timer.stop();
  1319. cout << "time 4: " << timer.getLast() << endl;
  1320. timer.start();
  1321. #endif
  1322. // forest[tree][right].featcounter = forest[tree][i].featcounter - forest[tree][left].featcounter;
  1323. double lcounter = 0.0, rcounter = 0.0;
  1324. for ( uint d = 0; d < forest[tree][left].dist.size(); d++ )
  1325. {
  1326. if ( forbidden_classes.find ( labelmapback[d] ) != forbidden_classes.end() )
  1327. {
  1328. forest[tree][left].dist[d] = 0;
  1329. forest[tree][right].dist[d] = 0;
  1330. }
  1331. else
  1332. {
  1333. forest[tree][left].dist[d] /= a[d];
  1334. lcounter += forest[tree][left].dist[d];
  1335. forest[tree][right].dist[d] /= a[d];
  1336. rcounter += forest[tree][right].dist[d];
  1337. }
  1338. }
  1339. #if 0
  1340. timer.stop();
  1341. cout << "time 5: " << timer.getLast() << endl;
  1342. timer.start();
  1343. #endif
  1344. if ( lcounter <= 0 || rcounter <= 0 )
  1345. {
  1346. cout << "lcounter : " << lcounter << " rcounter: " << rcounter << endl;
  1347. cout << "splitval: " << splitval << " splittype: " << splitfeat->writeInfos() << endl;
  1348. cout << "bestig: " << bestig << endl;
  1349. for ( int iCounter = 0; iCounter < imgcounter; iCounter++ )
  1350. {
  1351. int xsize = currentfeats[iCounter].width();
  1352. int ysize = currentfeats[iCounter].height();
  1353. int counter = 0;
  1354. for ( int x = 0; x < xsize; x++ )
  1355. {
  1356. for ( int y = 0; y < ysize; y++ )
  1357. {
  1358. if ( lastfeats[iCounter].get ( x, y, tree ) == i )
  1359. {
  1360. if ( ++counter > 30 )
  1361. break;
  1362. Features feat;
  1363. feat.feats = &allfeats[iCounter];
  1364. feat.cfeats = &lastfeats[iCounter];
  1365. feat.cTree = tree;
  1366. feat.tree = &forest[tree];
  1367. feat.integralImg = &integralImgs[iCounter];
  1368. double val = splitfeat->getVal ( feat, x, y );
  1369. cout << "splitval: " << splitval << " val: " << val << endl;
  1370. }
  1371. }
  1372. }
  1373. }
  1374. assert ( lcounter > 0 && rcounter > 0 );
  1375. }
  1376. for ( uint d = 0; d < forest[tree][left].dist.size(); d++ )
  1377. {
  1378. forest[tree][left].dist[d] /= lcounter;
  1379. forest[tree][right].dist[d] /= rcounter;
  1380. }
  1381. }
  1382. else
  1383. {
  1384. forest[tree][i].isleaf = true;
  1385. }
  1386. }
  1387. }
  1388. #if 0
  1389. timer.stop();
  1390. cout << "time after tree: " << timer.getLast() << endl;
  1391. timer.start();
  1392. #endif
  1393. }
  1394. //compute integral image
  1395. int channels = classes + allfeats[0].channels();
  1396. #if 0
  1397. timer.stop();
  1398. cout << "time for part0: " << timer.getLast() << endl;
  1399. timer.start();
  1400. #endif
  1401. if ( integralImgs[0].width() == 0 )
  1402. {
  1403. for ( int i = 0; i < imgcounter; i++ )
  1404. {
  1405. int xsize = allfeats[i].width();
  1406. int ysize = allfeats[i].height();
  1407. integralImgs[i].reInit ( xsize, ysize, channels );
  1408. integralImgs[i].setAll ( 0.0 );
  1409. }
  1410. }
  1411. #if 0
  1412. timer.stop();
  1413. cout << "time for part1: " << timer.getLast() << endl;
  1414. timer.start();
  1415. #endif
  1416. #pragma omp parallel for
  1417. for ( int i = 0; i < imgcounter; i++ )
  1418. {
  1419. computeIntegralImage ( currentfeats[i], allfeats[i], integralImgs[i] );
  1420. #ifdef TEXTONMAP
  1421. computeIntegralImage ( textonMap[i], integralTexton[i] );
  1422. #endif
  1423. }
  1424. #if 1
  1425. timer.stop();
  1426. cout << "time for depth " << depth << ": " << timer.getLast() << endl;
  1427. #endif
  1428. }
  1429. #ifdef WRITEGLOB
  1430. ofstream outstream("globtrain.feat");
  1431. for(int i = 0; i < textonMap.size(); i++)
  1432. {
  1433. set<int> usedclasses;
  1434. for ( uint x = 0; x < labels[i].rows(); x++ )
  1435. {
  1436. for ( uint y = 0; y < labels[i].cols(); y++ )
  1437. {
  1438. int classno = labels[i] ( x, y );
  1439. if ( forbidden_classes.find ( classno ) != forbidden_classes.end() )
  1440. continue;
  1441. usedclasses.insert(classno);
  1442. }
  1443. }
  1444. cout << "labels.cols: " << labels[i].cols() << " labels.rows " << labels[i].rows() << endl;
  1445. cout << "currentfeats : " << allfeats[i].width() << " allfeats[i].height(); " << allfeats[i].height() << endl;
  1446. set<int>::iterator it;
  1447. for ( it=usedclasses.begin() ; it != usedclasses.end(); it++ )
  1448. outstream << *it << " ";
  1449. outstream << endl;
  1450. integralTexton[i](integralTexton[i].width()-1, integralTexton[i].height()-1).store(outstream);
  1451. }
  1452. outstream.close();
  1453. #endif
  1454. cout << "uniquenumber " << uniquenumber << endl;
  1455. //getchar();
  1456. #ifdef DEBUG
  1457. for ( int tree = 0; tree < nbTrees; tree++ )
  1458. {
  1459. int t = ( int ) forest[tree].size();
  1460. for ( int i = 0; i < t; i++ )
  1461. {
  1462. printf ( "tree[%i]: left: %i, right: %i", i, forest[tree][i].left, forest[tree][i].right );
  1463. if ( !forest[tree][i].isleaf && forest[tree][i].left != -1 )
  1464. {
  1465. cout << ", feat: " << forest[tree][i].feat->writeInfos() << " ";
  1466. opOverview[forest[tree][i].feat->getOps() ]++;
  1467. contextOverview[forest[tree][i].depth][ ( int ) forest[tree][i].feat->getContext() ]++;
  1468. }
  1469. for ( int d = 0; d < ( int ) forest[tree][i].dist.size(); d++ )
  1470. {
  1471. cout << " " << forest[tree][i].dist[d];
  1472. }
  1473. cout << endl;
  1474. }
  1475. }
  1476. for ( uint c = 0; c < ops.size(); c++ )
  1477. {
  1478. cout << ops[c]->writeInfos() << ": " << opOverview[ops[c]->getOps() ] << endl;
  1479. }
  1480. for ( uint c = 0; c < cops.size(); c++ )
  1481. {
  1482. cout << cops[c]->writeInfos() << ": " << opOverview[cops[c]->getOps() ] << endl;
  1483. }
  1484. for ( int d = 0; d < maxDepth; d++ )
  1485. {
  1486. double sum = contextOverview[d][0] + contextOverview[d][1];
  1487. contextOverview[d][0] /= sum;
  1488. contextOverview[d][1] /= sum;
  1489. cout << "depth: " << d << " woContext: " << contextOverview[d][0] << " wContext: " << contextOverview[d][1] << endl;
  1490. }
  1491. #endif
  1492. }
  1493. void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult, NICE::MultiChannelImageT<double> & probabilities )
  1494. {
  1495. //int xpos = 8;
  1496. //int xpos = 15;
  1497. //int ypos = 78;
  1498. int xsize;
  1499. int ysize;
  1500. ce->getImageSize ( xsize, ysize );
  1501. int numClasses = classNames->numClasses();
  1502. fprintf ( stderr, "ContextTree classification !\n" );
  1503. probabilities.reInit ( xsize, ysize, numClasses, true );
  1504. probabilities.setAll ( 0 );
  1505. NICE::ColorImage img;
  1506. std::string currentFile = Globals::getCurrentImgFN();
  1507. try {
  1508. img = ColorImage ( currentFile );
  1509. } catch ( Exception ) {
  1510. cerr << "SemSeg: error opening image file <" << currentFile << ">" << endl;
  1511. return;
  1512. }
  1513. //TODO: resize image?!
  1514. MultiChannelImageT<double> feats;
  1515. #ifdef TEXTONMAP
  1516. MultiChannelImageT<SparseVectorInt> textonMap ( xsize / grid + 1, ysize / grid + 1, 1 );
  1517. MultiChannelImageT<SparseVectorInt> integralTexton ( xsize / grid + 1, ysize / grid + 1, 1 );
  1518. #endif
  1519. #ifdef LOCALFEATS
  1520. lfcw->getFeats ( img, feats );
  1521. #else
  1522. feats.reInit ( xsize, ysize, 3, true );
  1523. for ( int x = 0; x < xsize; x++ )
  1524. {
  1525. for ( int y = 0; y < ysize; y++ )
  1526. {
  1527. for ( int r = 0; r < 3; r++ )
  1528. {
  1529. feats.set ( x, y, img.getPixel ( x, y, r ), r );
  1530. }
  1531. }
  1532. }
  1533. feats = ColorSpace::rgbtolab ( feats );
  1534. #endif
  1535. bool allleaf = false;
  1536. MultiChannelImageT<double> integralImg;
  1537. MultiChannelImageT<unsigned short int> currentfeats ( xsize, ysize, nbTrees );
  1538. currentfeats.setAll ( 0 );
  1539. depth = 0;
  1540. for ( int d = 0; d < maxDepth && !allleaf; d++ )
  1541. {
  1542. depth++;
  1543. #ifdef TEXTONMAP
  1544. double weight = computeWeight(depth,maxDepth) - computeWeight(depth-1,maxDepth);
  1545. if(depth == 1)
  1546. {
  1547. weight = computeWeight(1,maxDepth);
  1548. }
  1549. #endif
  1550. allleaf = true;
  1551. MultiChannelImageT<unsigned short int> lastfeats = currentfeats;
  1552. #pragma omp parallel for
  1553. for ( int tree = 0; tree < nbTrees; tree++ )
  1554. {
  1555. for ( int x = 0; x < xsize; x++ )
  1556. {
  1557. for ( int y = 0; y < ysize; y++ )
  1558. {
  1559. int t = currentfeats.get ( x, y, tree );
  1560. if ( forest[tree][t].left > 0 )
  1561. {
  1562. allleaf = false;
  1563. Features feat;
  1564. feat.feats = &feats;
  1565. feat.cfeats = &lastfeats;
  1566. feat.cTree = tree;
  1567. feat.tree = &forest[tree];
  1568. feat.integralImg = &integralImg;
  1569. double val = forest[tree][t].feat->getVal ( feat, x, y );
  1570. int subx = x / grid;
  1571. int suby = y / grid;
  1572. if ( val < forest[tree][t].decision )
  1573. {
  1574. currentfeats.set ( x, y, forest[tree][t].left, tree );
  1575. #ifdef TEXTONMAP
  1576. #pragma omp critical
  1577. {
  1578. SparseVectorInt v;
  1579. v.insert ( pair<int, double> ( forest[tree][forest[tree][t].left].nodeNumber, weight ) );
  1580. textonMap ( subx, suby ).add ( v );
  1581. }
  1582. #endif
  1583. }
  1584. else
  1585. {
  1586. currentfeats.set ( x, y, forest[tree][t].right, tree );
  1587. #ifdef TEXTONMAP
  1588. #pragma omp critical
  1589. {
  1590. SparseVectorInt v;
  1591. v.insert ( pair<int, double> ( forest[tree][forest[tree][t].right].nodeNumber, weight ) );
  1592. textonMap ( subx, suby ).add ( v );
  1593. }
  1594. #endif
  1595. }
  1596. /*if ( x == xpos && y == ypos )
  1597. {
  1598. cout << "val: " << val << " decision: " << forest[tree][t].decision << " details: " << forest[tree][t].feat->writeInfos() << endl;
  1599. }*/
  1600. }
  1601. }
  1602. }
  1603. if(depth < maxDepth)
  1604. {
  1605. //compute integral image
  1606. int channels = ( int ) labelmap.size() + feats.channels();
  1607. if ( integralImg.width() == 0 )
  1608. {
  1609. int xsize = feats.width();
  1610. int ysize = feats.height();
  1611. integralImg.reInit ( xsize, ysize, channels );
  1612. integralImg.setAll ( 0.0 );
  1613. }
  1614. }
  1615. }
  1616. if(depth < maxDepth)
  1617. {
  1618. computeIntegralImage ( currentfeats, feats, integralImg );
  1619. #ifdef TEXTONMAP
  1620. computeIntegralImage ( textonMap, integralTexton );
  1621. #endif
  1622. }
  1623. }
  1624. // cout << forest[0][currentfeats.get ( xpos, ypos, 0 ) ].dist << endl;
  1625. #ifdef WRITEGLOB
  1626. ofstream outstream("globtest.feat",ofstream::app);
  1627. outstream << 0 << endl;
  1628. integralTexton(integralTexton.width()-1, integralTexton.height()-1).store(outstream);
  1629. outstream.close();
  1630. #endif
  1631. string cndir = conf->gS ( "SSContextTree", "cndir", "" );
  1632. int classes = ( int ) probabilities.numChannels;
  1633. vector<int> useclass ( classes, 1 );
  1634. #ifdef WRITEGLOB
  1635. std::vector< std::string > list;
  1636. StringTools::split ( currentFile, '/', list );
  1637. string orgname = list.back();
  1638. ofstream ostream("filelist.txt",ofstream::app);
  1639. ostream << orgname << ".dat" << endl;
  1640. ostream.close();
  1641. if ( cndir != "" )
  1642. {
  1643. useclass = vector<int> ( classes, 0 );
  1644. ifstream infile ( ( cndir + "/" + orgname + ".dat" ).c_str() );
  1645. #undef OLD
  1646. #ifdef OLD
  1647. while ( !infile.eof() && infile.good() )
  1648. {
  1649. int tmp;
  1650. infile >> tmp;
  1651. assert(tmp >= 0 && tmp < classes);
  1652. useclass[tmp] = 1;
  1653. }
  1654. #else
  1655. int c = 0;
  1656. vector<double> probs(classes, 0.0);
  1657. while ( !infile.eof() && infile.good() )
  1658. {
  1659. infile >> probs[c];
  1660. c++;
  1661. }
  1662. vector<double> sorted = probs;
  1663. sort (sorted.begin(), sorted.end());
  1664. double thr = sorted[10];
  1665. if(thr < 0.0)
  1666. thr = 0.0;
  1667. for(int c = 0; c < classes; c++)
  1668. {
  1669. if(probs[c] < thr)
  1670. {
  1671. useclass[c] = 1;
  1672. }
  1673. }
  1674. #endif
  1675. for(int c = 0; c < classes; c++)
  1676. {
  1677. if(useclass[c] == 0)
  1678. {
  1679. probabilities.set(-numeric_limits< double >::max(), c);
  1680. }
  1681. }
  1682. }
  1683. #endif
  1684. if ( pixelWiseLabeling )
  1685. {
  1686. //finales labeln:
  1687. //long int offset = 0;
  1688. for ( int x = 0; x < xsize; x++ )
  1689. {
  1690. for ( int y = 0; y < ysize; y++ )
  1691. {
  1692. double maxvalue = - numeric_limits<double>::max(); //TODO: das kann auch nur pro knoten gemacht werden, nicht pro pixel
  1693. int maxindex = 0;
  1694. uint s = forest[0][0].dist.size();
  1695. for ( uint i = 0; i < s; i++ )
  1696. {
  1697. int currentclass = labelmapback[i];
  1698. if(useclass[currentclass])
  1699. {
  1700. probabilities(x,y,currentclass) = getMeanProb ( x, y, i, currentfeats );
  1701. if ( probabilities(x,y,currentclass) > maxvalue )
  1702. {
  1703. maxvalue = probabilities(x,y,currentclass);
  1704. maxindex = currentclass;
  1705. }
  1706. }
  1707. }
  1708. segresult.setPixel ( x, y, maxindex );
  1709. if ( maxvalue > 1 )
  1710. cout << "maxvalue: " << maxvalue << endl;
  1711. }
  1712. }
  1713. #undef VISUALIZE
  1714. #ifdef VISUALIZE
  1715. for ( int j = 0 ; j < ( int ) probabilities.numChannels; j++ )
  1716. {
  1717. //cout << "class: " << j << endl;//" " << cn.text ( j ) << endl;
  1718. NICE::Matrix tmp ( probabilities.height(), probabilities.width() );
  1719. double maxval = -numeric_limits<double>::max();
  1720. double minval = numeric_limits<double>::max();
  1721. for ( int y = 0; y < probabilities.height(); y++ )
  1722. for ( int x = 0; x < probabilities.width(); x++ )
  1723. {
  1724. double val = probabilities( x, y, j );
  1725. tmp ( y, x ) = val;
  1726. maxval = std::max ( val, maxval );
  1727. minval = std::min ( val, minval );
  1728. }
  1729. tmp(0,0) = 1.0;
  1730. tmp(0,1) = 0.0;
  1731. NICE::ColorImage imgrgb ( probabilities.width(), probabilities.height() );
  1732. ICETools::convertToRGB ( tmp, imgrgb );
  1733. cout << "maxval = " << maxval << " minval: " << minval << " for class " << j << endl; //cn.text ( j ) << endl;
  1734. std::string s;
  1735. std::stringstream out;
  1736. out << "tmpprebmap" << j << ".ppm";
  1737. s = out.str();
  1738. imgrgb.write( s );
  1739. //showImage(imgrgb, "Ergebnis");
  1740. //getchar();
  1741. }
  1742. cout << "fertsch" << endl;
  1743. getchar();
  1744. cout << "weiter gehtsch" << endl;
  1745. #endif
  1746. }
  1747. else
  1748. {
  1749. //final labeling using segmentation
  1750. Matrix regions;
  1751. //showImage(img);
  1752. int regionNumber = segmentation->segRegions ( img, regions );
  1753. cout << "regions: " << regionNumber << endl;
  1754. int dSize = forest[0][0].dist.size();
  1755. vector<vector<double> > regionProbs ( regionNumber, vector<double> ( dSize, 0.0 ) );
  1756. vector<int> bestlabels ( regionNumber, 0 );
  1757. /*
  1758. for(int r = 0; r < regionNumber; r++)
  1759. {
  1760. Image over(img.width(), img.height());
  1761. for(int y = 0; y < img.height(); y++)
  1762. {
  1763. for(int x = 0; x < img.width(); x++)
  1764. {
  1765. if(((int)regions(x,y)) == r)
  1766. over.setPixel(x,y,1);
  1767. else
  1768. over.setPixel(x,y,0);
  1769. }
  1770. }
  1771. cout << "r: " << r << endl;
  1772. showImageOverlay(img, over);
  1773. }
  1774. */
  1775. for ( int y = 0; y < img.height(); y++ )
  1776. {
  1777. for ( int x = 0; x < img.width(); x++ )
  1778. {
  1779. int cregion = regions ( x, y );
  1780. for ( int d = 0; d < dSize; d++ )
  1781. {
  1782. regionProbs[cregion][d] += getMeanProb ( x, y, d, currentfeats );
  1783. }
  1784. }
  1785. }
  1786. for ( int r = 0; r < regionNumber; r++ )
  1787. {
  1788. double maxval = regionProbs[r][0];
  1789. bestlabels[r] = 0;
  1790. for ( int d = 1; d < dSize; d++ )
  1791. {
  1792. if ( maxval < regionProbs[r][d] )
  1793. {
  1794. maxval = regionProbs[r][d];
  1795. bestlabels[r] = d;
  1796. }
  1797. }
  1798. bestlabels[r] = labelmapback[bestlabels[r]];
  1799. }
  1800. for ( int y = 0; y < img.height(); y++ )
  1801. {
  1802. for ( int x = 0; x < img.width(); x++ )
  1803. {
  1804. segresult.setPixel ( x, y, bestlabels[regions ( x,y ) ] );
  1805. }
  1806. }
  1807. #define WRITEREGIONS
  1808. #ifdef WRITEREGIONS
  1809. RegionGraph rg;
  1810. segmentation->getGraphRepresentation ( img, regions, rg);
  1811. for ( uint pos = 0; pos < regionProbs.size(); pos++ )
  1812. {
  1813. rg[pos]->setProbs ( regionProbs[pos] );
  1814. }
  1815. std::string s;
  1816. std::stringstream out;
  1817. std::vector< std::string > list;
  1818. StringTools::split ( Globals::getCurrentImgFN (), '/', list );
  1819. out << "rgout/" << list.back() << ".graph";
  1820. string writefile = out.str();
  1821. rg.write ( writefile );
  1822. #endif
  1823. }
  1824. cout << "segmentation finished" << endl;
  1825. }
  1826. void SemSegContextTree::store ( std::ostream & os, int format ) const
  1827. {
  1828. os << nbTrees << endl;
  1829. classnames.store ( os );
  1830. map<int, int>::const_iterator it;
  1831. os << labelmap.size() << endl;
  1832. for ( it = labelmap.begin() ; it != labelmap.end(); it++ )
  1833. os << ( *it ).first << " " << ( *it ).second << endl;
  1834. os << labelmapback.size() << endl;
  1835. for ( it = labelmapback.begin() ; it != labelmapback.end(); it++ )
  1836. os << ( *it ).first << " " << ( *it ).second << endl;
  1837. int trees = forest.size();
  1838. os << trees << endl;
  1839. for ( int t = 0; t < trees; t++ )
  1840. {
  1841. int nodes = forest[t].size();
  1842. os << nodes << endl;
  1843. for ( int n = 0; n < nodes; n++ )
  1844. {
  1845. os << forest[t][n].left << " " << forest[t][n].right << " " << forest[t][n].decision << " " << forest[t][n].isleaf << " " << forest[t][n].depth << " " << forest[t][n].featcounter << " " << forest[t][n].nodeNumber << endl;
  1846. os << forest[t][n].dist << endl;
  1847. if ( forest[t][n].feat == NULL )
  1848. os << -1 << endl;
  1849. else
  1850. {
  1851. os << forest[t][n].feat->getOps() << endl;
  1852. forest[t][n].feat->store ( os );
  1853. }
  1854. }
  1855. }
  1856. }
  1857. void SemSegContextTree::restore ( std::istream & is, int format )
  1858. {
  1859. is >> nbTrees;
  1860. classnames.restore ( is );
  1861. int lsize;
  1862. is >> lsize;
  1863. labelmap.clear();
  1864. for ( int l = 0; l < lsize; l++ )
  1865. {
  1866. int first, second;
  1867. is >> first;
  1868. is >> second;
  1869. labelmap[first] = second;
  1870. }
  1871. is >> lsize;
  1872. labelmapback.clear();
  1873. for ( int l = 0; l < lsize; l++ )
  1874. {
  1875. int first, second;
  1876. is >> first;
  1877. is >> second;
  1878. labelmapback[first] = second;
  1879. }
  1880. int trees;
  1881. is >> trees;
  1882. forest.clear();
  1883. for ( int t = 0; t < trees; t++ )
  1884. {
  1885. vector<TreeNode> tmptree;
  1886. forest.push_back ( tmptree );
  1887. int nodes;
  1888. is >> nodes;
  1889. //cout << "nodes: " << nodes << endl;
  1890. for ( int n = 0; n < nodes; n++ )
  1891. {
  1892. TreeNode tmpnode;
  1893. forest[t].push_back ( tmpnode );
  1894. is >> forest[t][n].left;
  1895. is >> forest[t][n].right;
  1896. is >> forest[t][n].decision;
  1897. is >> forest[t][n].isleaf;
  1898. is >> forest[t][n].depth;
  1899. is >> forest[t][n].featcounter;
  1900. is >> forest[t][n].nodeNumber;
  1901. is >> forest[t][n].dist;
  1902. int feattype;
  1903. is >> feattype;
  1904. assert ( feattype < NBOPERATIONS );
  1905. forest[t][n].feat = NULL;
  1906. if ( feattype >= 0 )
  1907. {
  1908. for ( uint o = 0; o < ops.size(); o++ )
  1909. {
  1910. if ( ops[o]->getOps() == feattype )
  1911. {
  1912. forest[t][n].feat = ops[o]->clone();
  1913. break;
  1914. }
  1915. }
  1916. if ( forest[t][n].feat == NULL )
  1917. {
  1918. for ( uint o = 0; o < cops.size(); o++ )
  1919. {
  1920. if ( cops[o]->getOps() == feattype )
  1921. {
  1922. forest[t][n].feat = cops[o]->clone();
  1923. break;
  1924. }
  1925. }
  1926. }
  1927. assert ( forest[t][n].feat != NULL );
  1928. forest[t][n].feat->restore ( is );
  1929. }
  1930. }
  1931. }
  1932. }