FMKGPHyperparameterOptimization.cpp 63 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913
  1. /**
  2. * @file FMKGPHyperparameterOptimization.cpp
  3. * @brief Heart of the framework to set up everything, perform optimization, classification, and variance prediction (Implementation)
  4. * @author Erik Rodner, Alexander Freytag
  5. * @date 01/02/2012
  6. */
  7. // STL includes
  8. #include <iostream>
  9. #include <map>
  10. // NICE-core includes
  11. #include <core/algebra/ILSConjugateGradients.h>
  12. #include <core/algebra/ILSConjugateGradientsLanczos.h>
  13. #include <core/algebra/ILSSymmLqLanczos.h>
  14. #include <core/algebra/ILSMinResLanczos.h>
  15. #include <core/algebra/ILSPlainGradient.h>
  16. #include <core/algebra/EigValuesTRLAN.h>
  17. #include <core/algebra/CholeskyRobust.h>
  18. //
  19. #include <core/basics/Timer.h>
  20. #include <core/basics/ResourceStatistics.h>
  21. #include <core/basics/Exception.h>
  22. //
  23. #include <core/vector/Algorithms.h>
  24. #include <core/vector/Eigen.h>
  25. //
  26. #include <core/optimization/blackbox/DownhillSimplexOptimizer.h>
  27. // gp-hik-core includes
  28. #include "FMKGPHyperparameterOptimization.h"
  29. #include "FastMinKernel.h"
  30. #include "GMHIKernel.h"
  31. #include "IKMNoise.h"
  32. using namespace NICE;
  33. using namespace std;
  34. /////////////////////////////////////////////////////
  35. /////////////////////////////////////////////////////
  36. // PROTECTED METHODS
  37. /////////////////////////////////////////////////////
  38. /////////////////////////////////////////////////////
  39. void FMKGPHyperparameterOptimization::updateAfterIncrement (
  40. const std::set < int > newClasses,
  41. const bool & performOptimizationAfterIncrement )
  42. {
  43. if ( this->fmk == NULL )
  44. fthrow ( Exception, "FastMinKernel object was not initialized!" );
  45. std::map<int, NICE::Vector> binaryLabels;
  46. std::set<int> classesToUse;
  47. //TODO this could be made faster when storing the previous binary label vectors...
  48. this->prepareBinaryLabels ( binaryLabels, this->labels , classesToUse );
  49. if ( this->verbose )
  50. std::cerr << "labels.size() after increment: " << this->labels.size() << std::endl;
  51. NICE::Timer t1;
  52. NICE::GPLikelihoodApprox * gplike;
  53. uint parameterVectorSize;
  54. t1.start();
  55. this->setupGPLikelihoodApprox ( gplike, binaryLabels, parameterVectorSize );
  56. t1.stop();
  57. if ( this->verboseTime )
  58. std::cerr << "Time used for setting up the gplike-objects: " << t1.getLast() << std::endl;
  59. t1.start();
  60. if ( this->b_usePreviousAlphas && ( this->previousAlphas.size() > 0) )
  61. {
  62. //We initialize it with the same values as we use in GPLikelihoodApprox in batch training
  63. //default in GPLikelihoodApprox for the first time:
  64. // alpha = (binaryLabels[classCnt] * (1.0 / eigenmax[0]) );
  65. double factor ( 1.0 / this->eigenMax[0] );
  66. std::map<int, NICE::Vector>::const_iterator binaryLabelsIt = binaryLabels.begin();
  67. for ( std::map<int, NICE::Vector>::iterator prevAlphaIt = this->previousAlphas.begin();
  68. prevAlphaIt != this->previousAlphas.end();
  69. prevAlphaIt++
  70. )
  71. {
  72. int oldSize ( prevAlphaIt->second.size() );
  73. prevAlphaIt->second.resize ( oldSize + 1 );
  74. if ( binaryLabelsIt->second[oldSize] > 0 ) //we only have +1 and -1, so this might be benefitial in terms of speed
  75. prevAlphaIt->second[oldSize] = factor;
  76. else
  77. prevAlphaIt->second[oldSize] = -factor; //we follow the initialization as done in previous steps
  78. //prevAlphaIt->second[oldSize] = 0.0; // following the suggestion of Yeh and Darrell
  79. binaryLabelsIt++;
  80. }
  81. //compute unaffected alpha-vectors for the new classes
  82. for (std::set<int>::const_iterator newClIt = newClasses.begin(); newClIt != newClasses.end(); newClIt++)
  83. {
  84. NICE::Vector alphaVec = (binaryLabels[*newClIt] * factor ); //see GPLikelihoodApprox for an explanation
  85. previousAlphas.insert( std::pair<int, NICE::Vector>(*newClIt, alphaVec) );
  86. }
  87. gplike->setInitialAlphaGuess ( &previousAlphas );
  88. }
  89. else
  90. {
  91. //if we do not use previous alphas, we do not have to set up anything here
  92. gplike->setInitialAlphaGuess ( NULL );
  93. }
  94. t1.stop();
  95. if ( this->verboseTime )
  96. std::cerr << "Time used for setting up the alpha-objects: " << t1.getLast() << std::endl;
  97. if ( this->verbose )
  98. std::cerr << "update Eigendecomposition " << std::endl;
  99. t1.start();
  100. // we compute all needed eigenvectors for standard classification and variance prediction at ones.
  101. // nrOfEigenvaluesToConsiderForVarApprox should NOT be larger than 1 if a method different than approximate_fine is used!
  102. this->updateEigenDecomposition( std::max ( this->nrOfEigenvaluesToConsider, this->nrOfEigenvaluesToConsiderForVarApprox) );
  103. t1.stop();
  104. if ( this->verboseTime )
  105. std::cerr << "Time used for setting up the eigenvectors-objects: " << t1.getLast() << std::endl;
  106. ////////////////////// //////////////////////
  107. // RE-RUN THE OPTIMIZATION, IF DESIRED //
  108. ////////////////////// //////////////////////
  109. if ( this->verbose )
  110. std::cerr << "resulting eigenvalues for first class: " << eigenMax[0] << std::endl;
  111. // we can reuse the already given performOptimization-method:
  112. // OPT_GREEDY
  113. // for this strategy we can't reuse any of the previously computed scores
  114. // so come on, let's do the whole thing again...
  115. // OPT_DOWNHILLSIMPLEX
  116. // Here we can benefit from previous results, when we use them as initialization for our optimizer
  117. // ikmsums.begin()->second->getParameters ( currentParameters ); uses the previously computed optimal parameters
  118. // as initialization
  119. // OPT_NONE
  120. // nothing to do, obviously
  121. if ( this->verbose )
  122. std::cerr << "perform optimization after increment " << std::endl;
  123. int optimizationMethodTmpCopy;
  124. if ( !performOptimizationAfterIncrement )
  125. {
  126. // if no optimization shall be carried out, we simply set the optimization method to NONE but run the optimization
  127. // call nonetheless, thereby computing alpha vectors, etc. which would be not initialized
  128. optimizationMethodTmpCopy = this->optimizationMethod;
  129. this->optimizationMethod = OPT_NONE;
  130. }
  131. t1.start();
  132. this->performOptimization ( *gplike, parameterVectorSize);
  133. t1.stop();
  134. if ( this->verboseTime )
  135. std::cerr << "Time used for performing the optimization: " << t1.getLast() << std::endl;
  136. if ( this->verbose )
  137. std::cerr << "Preparing after retraining for classification ..." << std::endl;
  138. t1.start();
  139. this->transformFeaturesWithOptimalParameters ( *gplike, parameterVectorSize );
  140. t1.stop();
  141. if ( this->verboseTime)
  142. std::cerr << "Time used for transforming features with optimal parameters: " << t1.getLast() << std::endl;
  143. if ( !performOptimizationAfterIncrement )
  144. {
  145. this->optimizationMethod = optimizationMethodTmpCopy;
  146. }
  147. //NOTE unfortunately, the whole vector alpha differs, and not only its last entry.
  148. // If we knew any method, which could update this efficiently, we could also compute A and B more efficiently by updating them.
  149. // Since we are not aware of any such method, we have to compute them completely new
  150. // :/
  151. t1.start();
  152. this->computeMatricesAndLUTs ( *gplike );
  153. t1.stop();
  154. if ( this->verboseTime )
  155. std::cerr << "Time used for setting up the A'nB -objects: " << t1.getLast() << std::endl;
  156. //don't waste memory
  157. delete gplike;
  158. }
  159. /////////////////////////////////////////////////////
  160. /////////////////////////////////////////////////////
  161. // PUBLIC METHODS
  162. /////////////////////////////////////////////////////
  163. /////////////////////////////////////////////////////
  164. FMKGPHyperparameterOptimization::FMKGPHyperparameterOptimization()
  165. {
  166. // initialize pointer variables
  167. pf = NULL;
  168. eig = NULL;
  169. linsolver = NULL;
  170. fmk = NULL;
  171. q = NULL;
  172. precomputedTForVarEst = NULL;
  173. ikmsum = NULL;
  174. // initialize boolean flags
  175. verbose = false;
  176. verboseTime = false;
  177. debug = false;
  178. //stupid unneeded default values
  179. binaryLabelPositive = -1;
  180. binaryLabelNegative = -2;
  181. this->b_usePreviousAlphas = false;
  182. }
  183. FMKGPHyperparameterOptimization::FMKGPHyperparameterOptimization ( const Config *_conf, ParameterizedFunction *_pf, FastMinKernel *_fmk, const string & _confSection )
  184. {
  185. // initialize pointer variables
  186. pf = NULL;
  187. eig = NULL;
  188. linsolver = NULL;
  189. fmk = NULL;
  190. q = NULL;
  191. precomputedTForVarEst = NULL;
  192. ikmsum = NULL;
  193. //stupid unneeded default values
  194. binaryLabelPositive = -1;
  195. binaryLabelNegative = -2;
  196. knownClasses.clear();
  197. if ( _fmk == NULL )
  198. this->initialize ( _conf, _pf ); //then the confSection is also the default value
  199. else
  200. this->initialize ( _conf, _pf, _fmk, _confSection );
  201. }
  202. FMKGPHyperparameterOptimization::~FMKGPHyperparameterOptimization()
  203. {
  204. //pf will delete from outer program
  205. if ( this->eig != NULL )
  206. delete this->eig;
  207. if ( this->linsolver != NULL )
  208. delete this->linsolver;
  209. if ( this->fmk != NULL )
  210. delete this->fmk;
  211. if ( this->q != NULL )
  212. delete this->q;
  213. for ( uint i = 0 ; i < precomputedT.size(); i++ )
  214. delete [] ( precomputedT[i] );
  215. if ( precomputedTForVarEst != NULL )
  216. delete precomputedTForVarEst;
  217. if ( ikmsum != NULL )
  218. delete ikmsum;
  219. }
  220. void FMKGPHyperparameterOptimization::initialize ( const Config *_conf, ParameterizedFunction *_pf, FastMinKernel *_fmk, const std::string & _confSection )
  221. {
  222. if ( _fmk != NULL )
  223. {
  224. if ( this->fmk != NULL )
  225. {
  226. delete this->fmk;
  227. fmk = NULL;
  228. }
  229. this->fmk = _fmk;
  230. }
  231. this->pf = _pf;
  232. this->verbose = _conf->gB ( _confSection, "verbose", false );
  233. this->verboseTime = _conf->gB ( _confSection, "verboseTime", false );
  234. this->debug = _conf->gB ( _confSection, "debug", false );
  235. if ( verbose )
  236. {
  237. std::cerr << "------------" << std::endl;
  238. std::cerr << "| set-up |" << std::endl;
  239. std::cerr << "------------" << std::endl;
  240. }
  241. // this->eig = new EigValuesTRLAN();
  242. // My time measurements show that both methods use equal time, a comparision
  243. // of their numerical performance has not been done yet
  244. this->eig = new EVArnoldi ( _conf->gB ( _confSection, "eig_verbose", false ) /* verbose flag */, 10 );
  245. this->parameterUpperBound = _conf->gD ( _confSection, "parameter_upper_bound", 2.5 );
  246. this->parameterLowerBound = _conf->gD ( _confSection, "parameter_lower_bound", 1.0 );
  247. this->parameterStepSize = _conf->gD ( _confSection, "parameter_step_size", 0.1 );
  248. this->verifyApproximation = _conf->gB ( _confSection, "verify_approximation", false );
  249. this->nrOfEigenvaluesToConsider = _conf->gI ( _confSection, "nrOfEigenvaluesToConsider", 1 );
  250. this->nrOfEigenvaluesToConsiderForVarApprox = _conf->gI ( _confSection, "nrOfEigenvaluesToConsiderForVarApprox", 1 );
  251. bool useQuantization = _conf->gB ( _confSection, "use_quantization", false );
  252. if ( verbose )
  253. {
  254. std::cerr << "_confSection: " << _confSection << std::endl;
  255. std::cerr << "use_quantization: " << useQuantization << std::endl;
  256. }
  257. if ( _conf->gB ( _confSection, "use_quantization", false ) ) {
  258. int numBins = _conf->gI ( _confSection, "num_bins", 100 );
  259. if ( verbose )
  260. std::cerr << "FMKGPHyperparameterOptimization: quantization initialized with " << numBins << " bins." << std::endl;
  261. this->q = new Quantization ( numBins );
  262. } else {
  263. this->q = NULL;
  264. }
  265. bool ils_verbose = _conf->gB ( _confSection, "ils_verbose", false );
  266. ils_max_iterations = _conf->gI ( _confSection, "ils_max_iterations", 1000 );
  267. if ( verbose )
  268. std::cerr << "FMKGPHyperparameterOptimization: maximum number of iterations is " << ils_max_iterations << std::endl;
  269. double ils_min_delta = _conf->gD ( _confSection, "ils_min_delta", 1e-7 );
  270. double ils_min_residual = _conf->gD ( _confSection, "ils_min_residual", 1e-7/*1e-2 */ );
  271. string ils_method = _conf->gS ( _confSection, "ils_method", "CG" );
  272. if ( ils_method.compare ( "CG" ) == 0 )
  273. {
  274. if ( verbose )
  275. std::cerr << "We use CG with " << ils_max_iterations << " iterations, " << ils_min_delta << " as min delta, and " << ils_min_residual << " as min res " << std::endl;
  276. this->linsolver = new ILSConjugateGradients ( ils_verbose , ils_max_iterations, ils_min_delta, ils_min_residual );
  277. if ( verbose )
  278. std::cerr << "FMKGPHyperparameterOptimization: using ILS ConjugateGradients" << std::endl;
  279. }
  280. else if ( ils_method.compare ( "CGL" ) == 0 )
  281. {
  282. this->linsolver = new ILSConjugateGradientsLanczos ( ils_verbose , ils_max_iterations );
  283. if ( verbose )
  284. std::cerr << "FMKGPHyperparameterOptimization: using ILS ConjugateGradients (Lanczos)" << std::endl;
  285. }
  286. else if ( ils_method.compare ( "SYMMLQ" ) == 0 )
  287. {
  288. this->linsolver = new ILSSymmLqLanczos ( ils_verbose , ils_max_iterations );
  289. if ( verbose )
  290. std::cerr << "FMKGPHyperparameterOptimization: using ILS SYMMLQ" << std::endl;
  291. }
  292. else if ( ils_method.compare ( "MINRES" ) == 0 )
  293. {
  294. this->linsolver = new ILSMinResLanczos ( ils_verbose , ils_max_iterations );
  295. if ( verbose )
  296. std::cerr << "FMKGPHyperparameterOptimization: using ILS MINRES" << std::endl;
  297. }
  298. else
  299. {
  300. std::cerr << "FMKGPHyperparameterOptimization: " << _confSection << ":ils_method (" << ils_method << ") does not match any type (CG,CGL,SYMMLQ,MINRES), I will use CG" << std::endl;
  301. this->linsolver = new ILSConjugateGradients ( ils_verbose , ils_max_iterations, ils_min_delta, ils_min_residual );
  302. }
  303. string optimizationMethod_s = _conf->gS ( _confSection, "optimization_method", "greedy" );
  304. if ( optimizationMethod_s == "greedy" )
  305. optimizationMethod = OPT_GREEDY;
  306. else if ( optimizationMethod_s == "downhillsimplex" )
  307. optimizationMethod = OPT_DOWNHILLSIMPLEX;
  308. else if ( optimizationMethod_s == "none" )
  309. optimizationMethod = OPT_NONE;
  310. else
  311. fthrow ( Exception, "Optimization method " << optimizationMethod_s << " is not known." );
  312. if ( verbose )
  313. std::cerr << "Using optimization method: " << optimizationMethod_s << std::endl;
  314. downhillSimplexMaxIterations = _conf->gI ( _confSection, "downhillsimplex_max_iterations", 20 );
  315. // do not run longer than a day :)
  316. downhillSimplexTimeLimit = _conf->gD ( _confSection, "downhillsimplex_time_limit", 24 * 60 * 60 );
  317. downhillSimplexParamTol = _conf->gD ( _confSection, "downhillsimplex_delta", 0.01 );
  318. optimizeNoise = _conf->gB ( _confSection, "optimize_noise", false );
  319. if ( verbose )
  320. std::cerr << "Optimize noise: " << ( optimizeNoise ? "on" : "off" ) << std::endl;
  321. this->b_usePreviousAlphas = _conf->gB ( _confSection, "b_usePreviousAlphas", true );
  322. if ( verbose )
  323. {
  324. std::cerr << "------------" << std::endl;
  325. std::cerr << "| start |" << std::endl;
  326. std::cerr << "------------" << std::endl;
  327. }
  328. }
  329. ///////////////////// ///////////////////// /////////////////////
  330. // GET / SET
  331. ///////////////////// ///////////////////// /////////////////////
  332. void FMKGPHyperparameterOptimization::setParameterUpperBound ( const double & _parameterUpperBound )
  333. {
  334. parameterUpperBound = _parameterUpperBound;
  335. }
  336. void FMKGPHyperparameterOptimization::setParameterLowerBound ( const double & _parameterLowerBound )
  337. {
  338. parameterLowerBound = _parameterLowerBound;
  339. }
  340. std::set<int> FMKGPHyperparameterOptimization::getKnownClassNumbers ( ) const
  341. {
  342. return this->knownClasses;
  343. }
  344. ///////////////////// ///////////////////// /////////////////////
  345. // CLASSIFIER STUFF
  346. ///////////////////// ///////////////////// /////////////////////
  347. inline void FMKGPHyperparameterOptimization::setupGPLikelihoodApprox ( GPLikelihoodApprox * & gplike, const std::map<int, NICE::Vector> & binaryLabels, uint & parameterVectorSize )
  348. {
  349. gplike = new GPLikelihoodApprox ( binaryLabels, ikmsum, linsolver, eig, verifyApproximation, nrOfEigenvaluesToConsider );
  350. gplike->setDebug( debug );
  351. gplike->setVerbose( verbose );
  352. parameterVectorSize = ikmsum->getNumParameters();
  353. }
  354. void FMKGPHyperparameterOptimization::updateEigenDecomposition( const int & i_noEigenValues )
  355. {
  356. //compute the largest eigenvalue of K + noise
  357. try
  358. {
  359. eig->getEigenvalues ( *ikmsum, eigenMax, eigenMaxVectors, i_noEigenValues );
  360. }
  361. catch ( char const* exceptionMsg)
  362. {
  363. std::cerr << exceptionMsg << std::endl;
  364. throw("Problem in calculating Eigendecomposition of kernel matrix. Abort program...");
  365. }
  366. //NOTE EigenValue computation extracts EV and EW per default in decreasing order.
  367. }
  368. void FMKGPHyperparameterOptimization::performOptimization ( GPLikelihoodApprox & gplike, const uint & parameterVectorSize )
  369. {
  370. if (verbose)
  371. std::cerr << "perform optimization" << std::endl;
  372. if ( optimizationMethod == OPT_GREEDY )
  373. {
  374. if ( verbose )
  375. std::cerr << "OPT_GREEDY!!! " << std::endl;
  376. // simple greedy strategy
  377. if ( ikmsum->getNumParameters() != 1 )
  378. fthrow ( Exception, "Reduce size of the parameter vector or use downhill simplex!" );
  379. NICE::Vector lB = ikmsum->getParameterLowerBounds();
  380. NICE::Vector uB = ikmsum->getParameterUpperBounds();
  381. if ( verbose )
  382. std::cerr << "lower bound " << lB << " upper bound " << uB << " parameterStepSize: " << parameterStepSize << std::endl;
  383. for ( double mypara = lB[0]; mypara <= uB[0]; mypara += this->parameterStepSize )
  384. {
  385. OPTIMIZATION::matrix_type hyperp ( 1, 1, mypara );
  386. gplike.evaluate ( hyperp );
  387. }
  388. }
  389. else if ( optimizationMethod == OPT_DOWNHILLSIMPLEX )
  390. {
  391. //standard as before, normal optimization
  392. if ( verbose )
  393. std::cerr << "DOWNHILLSIMPLEX!!! " << std::endl;
  394. // downhill simplex strategy
  395. OPTIMIZATION::DownhillSimplexOptimizer optimizer;
  396. OPTIMIZATION::matrix_type initialParams ( parameterVectorSize, 1 );
  397. NICE::Vector currentParameters;
  398. ikmsum->getParameters ( currentParameters );
  399. for ( uint i = 0 ; i < parameterVectorSize; i++ )
  400. initialParams(i,0) = currentParameters[ i ];
  401. if ( verbose )
  402. std::cerr << "Initial parameters: " << initialParams << std::endl;
  403. //the scales object does not really matter in the actual implementation of Downhill Simplex
  404. // OPTIMIZATION::matrix_type scales ( parameterVectorSize, 1);
  405. // scales.set(1.0);
  406. OPTIMIZATION::SimpleOptProblem optProblem ( &gplike, initialParams, initialParams /* scales */ );
  407. optimizer.setMaxNumIter ( true, downhillSimplexMaxIterations );
  408. optimizer.setTimeLimit ( true, downhillSimplexTimeLimit );
  409. optimizer.setParamTol ( true, downhillSimplexParamTol );
  410. optimizer.optimizeProb ( optProblem );
  411. }
  412. else if ( optimizationMethod == OPT_NONE )
  413. {
  414. if ( verbose )
  415. std::cerr << "NO OPTIMIZATION!!! " << std::endl;
  416. // without optimization
  417. if ( optimizeNoise )
  418. fthrow ( Exception, "Deactivate optimize_noise!" );
  419. if ( verbose )
  420. std::cerr << "Optimization is deactivated!" << std::endl;
  421. double value (1.0);
  422. if ( this->parameterLowerBound == this->parameterUpperBound)
  423. value = this->parameterLowerBound;
  424. pf->setParameterLowerBounds ( NICE::Vector ( 1, value ) );
  425. pf->setParameterUpperBounds ( NICE::Vector ( 1, value ) );
  426. // we use the standard value
  427. OPTIMIZATION::matrix_type hyperp ( 1, 1, value );
  428. gplike.setParameterLowerBound ( value );
  429. gplike.setParameterUpperBound ( value );
  430. //we do not need to compute the likelihood here - we are only interested in directly obtaining alpha vectors
  431. gplike.computeAlphaDirect( hyperp, eigenMax );
  432. }
  433. if ( verbose )
  434. {
  435. std::cerr << "Optimal hyperparameter was: " << gplike.getBestParameters() << std::endl;
  436. }
  437. }
  438. void FMKGPHyperparameterOptimization::transformFeaturesWithOptimalParameters ( const GPLikelihoodApprox & gplike, const uint & parameterVectorSize )
  439. {
  440. // transform all features with the currently "optimal" parameter
  441. ikmsum->setParameters ( gplike.getBestParameters() );
  442. }
  443. void FMKGPHyperparameterOptimization::computeMatricesAndLUTs ( const GPLikelihoodApprox & gplike )
  444. {
  445. precomputedA.clear();
  446. precomputedB.clear();
  447. for ( std::map<int, NICE::Vector>::const_iterator i = gplike.getBestAlphas().begin(); i != gplike.getBestAlphas().end(); i++ )
  448. {
  449. PrecomputedType A;
  450. PrecomputedType B;
  451. fmk->hik_prepare_alpha_multiplications ( i->second, A, B );
  452. A.setIoUntilEndOfFile ( false );
  453. B.setIoUntilEndOfFile ( false );
  454. precomputedA[ i->first ] = A;
  455. precomputedB[ i->first ] = B;
  456. if ( q != NULL )
  457. {
  458. double *T = fmk->hik_prepare_alpha_multiplications_fast ( A, B, *q, pf );
  459. //just to be sure that we do not waste space here
  460. if ( precomputedT[ i->first ] != NULL )
  461. delete precomputedT[ i->first ];
  462. precomputedT[ i->first ] = T;
  463. }
  464. }
  465. if ( this->precomputedTForVarEst != NULL )
  466. {
  467. this->prepareVarianceApproximationRough();
  468. }
  469. else if ( this->nrOfEigenvaluesToConsiderForVarApprox > 0)
  470. {
  471. this->prepareVarianceApproximationFine();
  472. }
  473. // in case that we should want to store the alpha vectors for incremental extensions
  474. if ( this->b_usePreviousAlphas )
  475. this->previousAlphas = gplike.getBestAlphas();
  476. }
  477. #ifdef NICE_USELIB_MATIO
  478. void FMKGPHyperparameterOptimization::optimizeBinary ( const sparse_t & data, const NICE::Vector & yl, const std::set<int> & positives, const std::set<int> & negatives, double noise )
  479. {
  480. std::map<int, int> examples;
  481. NICE::Vector y ( yl.size() );
  482. int ind = 0;
  483. for ( uint i = 0 ; i < yl.size(); i++ )
  484. {
  485. if ( positives.find ( i ) != positives.end() ) {
  486. y[ examples.size() ] = 1.0;
  487. examples.insert ( pair<int, int> ( i, ind ) );
  488. ind++;
  489. } else if ( negatives.find ( i ) != negatives.end() ) {
  490. y[ examples.size() ] = -1.0;
  491. examples.insert ( pair<int, int> ( i, ind ) );
  492. ind++;
  493. }
  494. }
  495. y.resize ( examples.size() );
  496. std::cerr << "Examples: " << examples.size() << std::endl;
  497. optimize ( data, y, examples, noise );
  498. }
  499. void FMKGPHyperparameterOptimization::optimize ( const sparse_t & data, const NICE::Vector & y, const std::map<int, int> & examples, double noise )
  500. {
  501. NICE::Timer t;
  502. t.start();
  503. std::cerr << "Initializing data structure ..." << std::endl;
  504. if ( fmk != NULL ) delete fmk;
  505. fmk = new FastMinKernel ( data, noise, examples );
  506. t.stop();
  507. if (verboseTime)
  508. std::cerr << "Time used for initializing the FastMinKernel structure: " << t.getLast() << std::endl;
  509. optimize ( y );
  510. }
  511. #endif
  512. int FMKGPHyperparameterOptimization::prepareBinaryLabels ( std::map<int, NICE::Vector> & binaryLabels, const NICE::Vector & y , std::set<int> & myClasses )
  513. {
  514. myClasses.clear();
  515. // determine which classes we have in our label vector
  516. // -> MATLAB: myClasses = unique(y);
  517. for ( NICE::Vector::const_iterator it = y.begin(); it != y.end(); it++ )
  518. {
  519. if ( myClasses.find ( *it ) == myClasses.end() )
  520. {
  521. myClasses.insert ( *it );
  522. }
  523. }
  524. //count how many different classes appear in our data
  525. int nrOfClasses = myClasses.size();
  526. binaryLabels.clear();
  527. //compute the corresponding binary label vectors
  528. if ( nrOfClasses > 2 )
  529. {
  530. //resize every labelVector and set all entries to -1.0
  531. for ( set<int>::const_iterator k = myClasses.begin(); k != myClasses.end(); k++ )
  532. {
  533. binaryLabels[ *k ].resize ( y.size() );
  534. binaryLabels[ *k ].set ( -1.0 );
  535. }
  536. // now look on every example and set the entry of its corresponding label vector to 1.0
  537. // proper existance should not be a problem
  538. for ( int i = 0 ; i < ( int ) y.size(); i++ )
  539. binaryLabels[ y[i] ][i] = 1.0;
  540. }
  541. else if ( nrOfClasses == 2 )
  542. {
  543. //binary setting -- prepare two binary label vectors with opposite signs
  544. NICE::Vector yb ( y );
  545. binaryLabelNegative = *(myClasses.begin());
  546. std::set<int>::const_iterator classIt = myClasses.begin(); classIt++;
  547. binaryLabelPositive = *classIt;
  548. if ( verbose )
  549. std::cerr << "positiveClass : " << binaryLabelPositive << " negativeClass: " << binaryLabelNegative << std::endl;
  550. for ( uint i = 0 ; i < yb.size() ; i++ )
  551. yb[i] = ( y[i] == binaryLabelNegative ) ? -1.0 : 1.0;
  552. binaryLabels[ binaryLabelPositive ] = yb;
  553. //we do NOT do real binary computation, but an implicite one with only a single object
  554. nrOfClasses--;
  555. }
  556. else //OCC setting
  557. {
  558. //we set the labels to 1, independent of the previously given class number
  559. //however, the original class numbers are stored and returned in classification
  560. NICE::Vector yOne ( y.size(), 1 );
  561. binaryLabels[ *(myClasses.begin()) ] = yOne;
  562. //we have to indicate, that we are in an OCC setting
  563. nrOfClasses--;
  564. }
  565. return nrOfClasses;
  566. }
  567. void FMKGPHyperparameterOptimization::optimize ( const NICE::Vector & y )
  568. {
  569. if ( fmk == NULL )
  570. fthrow ( Exception, "FastMinKernel object was not initialized!" );
  571. this->labels = y;
  572. std::map<int, NICE::Vector> binaryLabels;
  573. prepareBinaryLabels ( binaryLabels, y , knownClasses );
  574. //now call the main function :)
  575. this->optimize(binaryLabels);
  576. }
  577. void FMKGPHyperparameterOptimization::optimize ( std::map<int, NICE::Vector> & binaryLabels )
  578. {
  579. Timer t;
  580. t.start();
  581. //how many different classes do we have right now?
  582. int nrOfClasses = binaryLabels.size();
  583. if (verbose)
  584. {
  585. std::cerr << "Initial noise level: " << fmk->getNoise() << std::endl;
  586. std::cerr << "Number of classes (=1 means we have a binary setting):" << nrOfClasses << std::endl;
  587. std::cerr << "Effective number of classes (neglecting classes without positive examples): " << knownClasses.size() << std::endl;
  588. }
  589. // combine standard model and noise model
  590. Timer t1;
  591. t1.start();
  592. //setup the kernel combination
  593. ikmsum = new IKMLinearCombination ();
  594. if ( verbose )
  595. {
  596. std::cerr << "binaryLabels.size(): " << binaryLabels.size() << std::endl;
  597. }
  598. //First model: noise
  599. ikmsum->addModel ( new IKMNoise ( fmk->get_n(), fmk->getNoise(), optimizeNoise ) );
  600. // set pretty low built-in noise, because we explicitely add the noise with the IKMNoise
  601. fmk->setNoise ( 0.0 );
  602. ikmsum->addModel ( new GMHIKernel ( fmk, pf, NULL /* no quantization */ ) );
  603. t1.stop();
  604. if (verboseTime)
  605. std::cerr << "Time used for setting up the ikm-objects: " << t1.getLast() << std::endl;
  606. GPLikelihoodApprox * gplike;
  607. uint parameterVectorSize;
  608. t1.start();
  609. this->setupGPLikelihoodApprox ( gplike, binaryLabels, parameterVectorSize );
  610. t1.stop();
  611. if (verboseTime)
  612. std::cerr << "Time used for setting up the gplike-objects: " << t1.getLast() << std::endl;
  613. if (verbose)
  614. {
  615. std::cerr << "parameterVectorSize: " << parameterVectorSize << std::endl;
  616. }
  617. t1.start();
  618. // we compute all needed eigenvectors for standard classification and variance prediction at ones.
  619. // nrOfEigenvaluesToConsiderForVarApprox should NOT be larger than 1 if a method different than approximate_fine is used!
  620. this->updateEigenDecomposition( std::max ( this->nrOfEigenvaluesToConsider, this->nrOfEigenvaluesToConsiderForVarApprox) );
  621. t1.stop();
  622. if (verboseTime)
  623. std::cerr << "Time used for setting up the eigenvectors-objects: " << t1.getLast() << std::endl;
  624. if ( verbose )
  625. std::cerr << "resulting eigenvalues for first class: " << eigenMax[0] << std::endl;
  626. t1.start();
  627. this->performOptimization ( *gplike, parameterVectorSize );
  628. t1.stop();
  629. if (verboseTime)
  630. std::cerr << "Time used for performing the optimization: " << t1.getLast() << std::endl;
  631. if ( verbose )
  632. std::cerr << "Preparing classification ..." << std::endl;
  633. t1.start();
  634. this->transformFeaturesWithOptimalParameters ( *gplike, parameterVectorSize );
  635. t1.stop();
  636. if (verboseTime)
  637. std::cerr << "Time used for transforming features with optimal parameters: " << t1.getLast() << std::endl;
  638. t1.start();
  639. this->computeMatricesAndLUTs ( *gplike );
  640. t1.stop();
  641. if (verboseTime)
  642. std::cerr << "Time used for setting up the A'nB -objects: " << t1.getLast() << std::endl;
  643. t.stop();
  644. ResourceStatistics rs;
  645. std::cerr << "Time used for learning: " << t.getLast() << std::endl;
  646. long maxMemory;
  647. rs.getMaximumMemory ( maxMemory );
  648. std::cerr << "Maximum memory used: " << maxMemory << " KB" << std::endl;
  649. //don't waste memory
  650. delete gplike;
  651. }
  652. void FMKGPHyperparameterOptimization::prepareVarianceApproximationRough()
  653. {
  654. PrecomputedType AVar;
  655. fmk->hikPrepareKVNApproximation ( AVar );
  656. precomputedAForVarEst = AVar;
  657. precomputedAForVarEst.setIoUntilEndOfFile ( false );
  658. if ( q != NULL )
  659. {
  660. double *T = fmk->hikPrepareLookupTableForKVNApproximation ( *q, pf );
  661. this->precomputedTForVarEst = T;
  662. }
  663. }
  664. void FMKGPHyperparameterOptimization::prepareVarianceApproximationFine()
  665. {
  666. if ( this->eigenMax.size() < (uint) this->nrOfEigenvaluesToConsiderForVarApprox )
  667. {
  668. std::cerr << "not enough eigenvectors computed for fine approximation of predictive variance. " <<std::endl;
  669. std::cerr << "Current number of EV: " << this->eigenMax.size() << " but required: " << (uint) this->nrOfEigenvaluesToConsiderForVarApprox << std::endl;
  670. this->updateEigenDecomposition( this->nrOfEigenvaluesToConsiderForVarApprox );
  671. }
  672. }
  673. int FMKGPHyperparameterOptimization::classify ( const NICE::SparseVector & xstar, NICE::SparseVector & scores ) const
  674. {
  675. // loop through all classes
  676. if ( precomputedA.size() == 0 )
  677. {
  678. fthrow ( Exception, "The precomputation vector is zero...have you trained this classifier?" );
  679. }
  680. uint maxClassNo = 0;
  681. for ( std::map<int, PrecomputedType>::const_iterator i = precomputedA.begin() ; i != precomputedA.end(); i++ )
  682. {
  683. uint classno = i->first;
  684. maxClassNo = std::max ( maxClassNo, classno );
  685. double beta;
  686. if ( q != NULL ) {
  687. map<int, double *>::const_iterator j = precomputedT.find ( classno );
  688. double *T = j->second;
  689. fmk->hik_kernel_sum_fast ( T, *q, xstar, beta );
  690. } else {
  691. const PrecomputedType & A = i->second;
  692. std::map<int, PrecomputedType>::const_iterator j = precomputedB.find ( classno );
  693. const PrecomputedType & B = j->second;
  694. // fmk->hik_kernel_sum ( A, B, xstar, beta ); if A, B are of type Matrix
  695. // Giving the transformation pf as an additional
  696. // argument is necessary due to the following reason:
  697. // FeatureMatrixT is sorted according to the original values, therefore,
  698. // searching for upper and lower bounds ( findFirst... functions ) require original feature
  699. // values as inputs. However, for calculation we need the transformed features values.
  700. fmk->hik_kernel_sum ( A, B, xstar, beta, pf );
  701. }
  702. scores[ classno ] = beta;
  703. }
  704. scores.setDim ( maxClassNo + 1 );
  705. if ( precomputedA.size() > 1 )
  706. { // multi-class classification
  707. return scores.maxElement();
  708. }
  709. else
  710. { // binary setting
  711. scores[binaryLabelNegative] = -scores[binaryLabelPositive];
  712. return scores[ binaryLabelPositive ] <= 0.0 ? binaryLabelNegative : binaryLabelPositive;
  713. }
  714. }
  715. int FMKGPHyperparameterOptimization::classify ( const NICE::Vector & xstar, NICE::SparseVector & scores ) const
  716. {
  717. // loop through all classes
  718. if ( precomputedA.size() == 0 )
  719. {
  720. fthrow ( Exception, "The precomputation vector is zero...have you trained this classifier?" );
  721. }
  722. uint maxClassNo = 0;
  723. for ( std::map<int, PrecomputedType>::const_iterator i = precomputedA.begin() ; i != precomputedA.end(); i++ )
  724. {
  725. uint classno = i->first;
  726. maxClassNo = std::max ( maxClassNo, classno );
  727. double beta;
  728. if ( q != NULL ) {
  729. std::map<int, double *>::const_iterator j = precomputedT.find ( classno );
  730. double *T = j->second;
  731. fmk->hik_kernel_sum_fast ( T, *q, xstar, beta );
  732. } else {
  733. const PrecomputedType & A = i->second;
  734. std::map<int, PrecomputedType>::const_iterator j = precomputedB.find ( classno );
  735. const PrecomputedType & B = j->second;
  736. // fmk->hik_kernel_sum ( A, B, xstar, beta ); if A, B are of type Matrix
  737. // Giving the transformation pf as an additional
  738. // argument is necessary due to the following reason:
  739. // FeatureMatrixT is sorted according to the original values, therefore,
  740. // searching for upper and lower bounds ( findFirst... functions ) require original feature
  741. // values as inputs. However, for calculation we need the transformed features values.
  742. fmk->hik_kernel_sum ( A, B, xstar, beta, pf );
  743. }
  744. scores[ classno ] = beta;
  745. }
  746. scores.setDim ( maxClassNo + 1 );
  747. if ( precomputedA.size() > 1 )
  748. { // multi-class classification
  749. return scores.maxElement();
  750. }
  751. else
  752. { // binary setting
  753. scores[binaryLabelNegative] = -scores[binaryLabelPositive];
  754. return scores[ binaryLabelPositive ] <= 0.0 ? binaryLabelNegative : binaryLabelPositive;
  755. }
  756. }
  757. //////////////////////////////////////////
  758. // variance computation: sparse inputs
  759. //////////////////////////////////////////
  760. void FMKGPHyperparameterOptimization::computePredictiveVarianceApproximateRough ( const NICE::SparseVector & x, double & predVariance ) const
  761. {
  762. // security check!
  763. if ( pf == NULL )
  764. fthrow ( Exception, "pf is NULL...have you prepared the uncertainty prediction? Aborting..." );
  765. // ---------------- compute the first term --------------------
  766. double kSelf ( 0.0 );
  767. for ( NICE::SparseVector::const_iterator it = x.begin(); it != x.end(); it++ )
  768. {
  769. kSelf += pf->f ( 0, it->second );
  770. // if weighted dimensions:
  771. //kSelf += pf->f(it->first,it->second);
  772. }
  773. // ---------------- compute the approximation of the second term --------------------
  774. double normKStar;
  775. if ( q != NULL )
  776. {
  777. if ( precomputedTForVarEst == NULL )
  778. {
  779. fthrow ( Exception, "The precomputed LUT for uncertainty prediction is NULL...have you prepared the uncertainty prediction? Aborting..." );
  780. }
  781. fmk->hikComputeKVNApproximationFast ( precomputedTForVarEst, *q, x, normKStar );
  782. }
  783. else
  784. {
  785. if ( precomputedAForVarEst.size () == 0 )
  786. {
  787. fthrow ( Exception, "The precomputedAForVarEst is empty...have you trained this classifer? Aborting..." );
  788. }
  789. fmk->hikComputeKVNApproximation ( precomputedAForVarEst, x, normKStar, pf );
  790. }
  791. predVariance = kSelf - ( 1.0 / eigenMax[0] )* normKStar;
  792. }
  793. void FMKGPHyperparameterOptimization::computePredictiveVarianceApproximateFine ( const NICE::SparseVector & x, double & predVariance ) const
  794. {
  795. // security check!
  796. if ( eigenMaxVectors.rows() == 0 )
  797. {
  798. fthrow ( Exception, "eigenMaxVectors is empty...have you trained this classifer? Aborting..." );
  799. }
  800. // ---------------- compute the first term --------------------
  801. // Timer t;
  802. // t.start();
  803. double kSelf ( 0.0 );
  804. for ( NICE::SparseVector::const_iterator it = x.begin(); it != x.end(); it++ )
  805. {
  806. kSelf += pf->f ( 0, it->second );
  807. // if weighted dimensions:
  808. //kSelf += pf->f(it->first,it->second);
  809. }
  810. // ---------------- compute the approximation of the second term --------------------
  811. // t.stop();
  812. // std::cerr << "ApproxFine -- time for first term: " << t.getLast() << std::endl;
  813. // t.start();
  814. NICE::Vector kStar;
  815. fmk->hikComputeKernelVector ( x, kStar );
  816. /* t.stop();
  817. std::cerr << "ApproxFine -- time for kernel vector: " << t.getLast() << std::endl;*/
  818. // NICE::Vector multiplicationResults; // will contain nrOfEigenvaluesToConsiderForVarApprox many entries
  819. // multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
  820. NICE::Vector multiplicationResults( nrOfEigenvaluesToConsiderForVarApprox-1, 0.0 );
  821. //ok, there seems to be a nasty thing in computing multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
  822. //wherefor it takes aeons...
  823. //so we compute it by ourselves
  824. // for ( uint tmpI = 0; tmpI < kStar.size(); tmpI++)
  825. NICE::Matrix::const_iterator eigenVecIt = eigenMaxVectors.begin();
  826. // double kStarI ( kStar[tmpI] );
  827. for ( int tmpJ = 0; tmpJ < nrOfEigenvaluesToConsiderForVarApprox-1; tmpJ++)
  828. {
  829. for ( NICE::Vector::const_iterator kStarIt = kStar.begin(); kStarIt != kStar.end(); kStarIt++,eigenVecIt++)
  830. {
  831. multiplicationResults[tmpJ] += (*kStarIt) * (*eigenVecIt);//eigenMaxVectors(tmpI,tmpJ);
  832. }
  833. }
  834. double projectionLength ( 0.0 );
  835. double currentSecondTerm ( 0.0 );
  836. double sumOfProjectionLengths ( 0.0 );
  837. int cnt ( 0 );
  838. NICE::Vector::const_iterator it = multiplicationResults.begin();
  839. while ( cnt < ( nrOfEigenvaluesToConsiderForVarApprox - 1 ) )
  840. {
  841. projectionLength = ( *it );
  842. currentSecondTerm += ( 1.0 / eigenMax[cnt] ) * pow ( projectionLength, 2 );
  843. sumOfProjectionLengths += pow ( projectionLength, 2 );
  844. it++;
  845. cnt++;
  846. }
  847. double normKStar ( pow ( kStar.normL2 (), 2 ) );
  848. currentSecondTerm += ( 1.0 / eigenMax[nrOfEigenvaluesToConsiderForVarApprox-1] ) * ( normKStar - sumOfProjectionLengths );
  849. if ( ( normKStar - sumOfProjectionLengths ) < 0 )
  850. {
  851. std::cerr << "Attention: normKStar - sumOfProjectionLengths is smaller than zero -- strange!" << std::endl;
  852. }
  853. predVariance = kSelf - currentSecondTerm;
  854. }
  855. void FMKGPHyperparameterOptimization::computePredictiveVarianceExact ( const NICE::SparseVector & x, double & predVariance ) const
  856. {
  857. // security check!
  858. if ( ikmsum->getNumberOfModels() == 0 )
  859. {
  860. fthrow ( Exception, "ikmsum is empty... have you trained this classifer? Aborting..." );
  861. }
  862. Timer t;
  863. // t.start();
  864. // ---------------- compute the first term --------------------
  865. double kSelf ( 0.0 );
  866. for ( NICE::SparseVector::const_iterator it = x.begin(); it != x.end(); it++ )
  867. {
  868. kSelf += pf->f ( 0, it->second );
  869. // if weighted dimensions:
  870. //kSelf += pf->f(it->first,it->second);
  871. }
  872. // ---------------- compute the second term --------------------
  873. NICE::Vector kStar;
  874. fmk->hikComputeKernelVector ( x, kStar );
  875. //now run the ILS method
  876. NICE::Vector diagonalElements;
  877. ikmsum->getDiagonalElements ( diagonalElements );
  878. // init simple jacobi pre-conditioning
  879. ILSConjugateGradients *linsolver_cg = dynamic_cast<ILSConjugateGradients *> ( linsolver );
  880. //TODO what to do for other solver techniques?
  881. //perform pre-conditioning
  882. if ( linsolver_cg != NULL )
  883. linsolver_cg->setJacobiPreconditioner ( diagonalElements );
  884. NICE::Vector beta;
  885. /** About finding a good initial solution (see also GPLikelihoodApproximation)
  886. * K~ = K + sigma^2 I
  887. *
  888. * K~ \approx lambda_max v v^T
  889. * \lambda_max v v^T * alpha = k_* | multiply with v^T from left
  890. * => \lambda_max v^T alpha = v^T k_*
  891. * => alpha = k_* / lambda_max could be a good initial start
  892. * If we put everything in the first equation this gives us
  893. * v = k_*
  894. * This reduces the number of iterations by 5 or 8
  895. */
  896. beta = (kStar * (1.0 / eigenMax[0]) );
  897. linsolver->solveLin ( *ikmsum, kStar, beta );
  898. beta *= kStar;
  899. double currentSecondTerm( beta.Sum() );
  900. predVariance = kSelf - currentSecondTerm;
  901. }
  902. //////////////////////////////////////////
  903. // variance computation: non-sparse inputs
  904. //////////////////////////////////////////
  905. void FMKGPHyperparameterOptimization::computePredictiveVarianceApproximateRough ( const NICE::Vector & x, double & predVariance ) const
  906. {
  907. // security check!
  908. if ( pf == NULL )
  909. fthrow ( Exception, "pf is NULL...have you prepared the uncertainty prediction? Aborting..." );
  910. // ---------------- compute the first term --------------------
  911. double kSelf ( 0.0 );
  912. int dim ( 0 );
  913. for ( NICE::Vector::const_iterator it = x.begin(); it != x.end(); it++, dim++ )
  914. {
  915. kSelf += pf->f ( 0, *it );
  916. // if weighted dimensions:
  917. //kSelf += pf->f(dim,*it);
  918. }
  919. // ---------------- compute the approximation of the second term --------------------
  920. double normKStar;
  921. if ( q != NULL )
  922. {
  923. if ( precomputedTForVarEst == NULL )
  924. {
  925. fthrow ( Exception, "The precomputed LUT for uncertainty prediction is NULL...have you prepared the uncertainty prediction? Aborting..." );
  926. }
  927. fmk->hikComputeKVNApproximationFast ( precomputedTForVarEst, *q, x, normKStar );
  928. }
  929. else
  930. {
  931. if ( precomputedAForVarEst.size () == 0 )
  932. {
  933. fthrow ( Exception, "The precomputedAForVarEst is empty...have you trained this classifer? Aborting..." );
  934. }
  935. fmk->hikComputeKVNApproximation ( precomputedAForVarEst, x, normKStar, pf );
  936. }
  937. predVariance = kSelf - ( 1.0 / eigenMax[0] )* normKStar;
  938. }
  939. void FMKGPHyperparameterOptimization::computePredictiveVarianceApproximateFine ( const NICE::Vector & x, double & predVariance ) const
  940. {
  941. // security check!
  942. if ( eigenMaxVectors.rows() == 0 )
  943. {
  944. fthrow ( Exception, "eigenMaxVectors is empty...have you trained this classifer? Aborting..." );
  945. }
  946. // ---------------- compute the first term --------------------
  947. double kSelf ( 0.0 );
  948. int dim ( 0 );
  949. for ( NICE::Vector::const_iterator it = x.begin(); it != x.end(); it++, dim++ )
  950. {
  951. kSelf += pf->f ( 0, *it );
  952. // if weighted dimensions:
  953. //kSelf += pf->f(dim,*it);
  954. }
  955. // ---------------- compute the approximation of the second term --------------------
  956. NICE::Vector kStar;
  957. fmk->hikComputeKernelVector ( x, kStar );
  958. //ok, there seems to be a nasty thing in computing multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
  959. //wherefor it takes aeons...
  960. //so we compute it by ourselves
  961. // NICE::Vector multiplicationResults; // will contain nrOfEigenvaluesToConsiderForVarApprox many entries
  962. // multiplicationResults.multiply ( *eigenMaxVectorIt, kStar, true/* transpose */ );
  963. NICE::Vector multiplicationResults( nrOfEigenvaluesToConsiderForVarApprox-1, 0.0 );
  964. NICE::Matrix::const_iterator eigenVecIt = eigenMaxVectors.begin();
  965. for ( int tmpJ = 0; tmpJ < nrOfEigenvaluesToConsiderForVarApprox-1; tmpJ++)
  966. {
  967. for ( NICE::Vector::const_iterator kStarIt = kStar.begin(); kStarIt != kStar.end(); kStarIt++,eigenVecIt++)
  968. {
  969. multiplicationResults[tmpJ] += (*kStarIt) * (*eigenVecIt);//eigenMaxVectors(tmpI,tmpJ);
  970. }
  971. }
  972. double projectionLength ( 0.0 );
  973. double currentSecondTerm ( 0.0 );
  974. double sumOfProjectionLengths ( 0.0 );
  975. int cnt ( 0 );
  976. NICE::Vector::const_iterator it = multiplicationResults.begin();
  977. while ( cnt < ( nrOfEigenvaluesToConsiderForVarApprox - 1 ) )
  978. {
  979. projectionLength = ( *it );
  980. currentSecondTerm += ( 1.0 / eigenMax[cnt] ) * pow ( projectionLength, 2 );
  981. sumOfProjectionLengths += pow ( projectionLength, 2 );
  982. it++;
  983. cnt++;
  984. }
  985. double normKStar ( pow ( kStar.normL2 (), 2 ) );
  986. currentSecondTerm += ( 1.0 / eigenMax[nrOfEigenvaluesToConsiderForVarApprox-1] ) * ( normKStar - sumOfProjectionLengths );
  987. if ( ( normKStar - sumOfProjectionLengths ) < 0 )
  988. {
  989. std::cerr << "Attention: normKStar - sumOfProjectionLengths is smaller than zero -- strange!" << std::endl;
  990. }
  991. predVariance = kSelf - currentSecondTerm;
  992. }
  993. void FMKGPHyperparameterOptimization::computePredictiveVarianceExact ( const NICE::Vector & x, double & predVariance ) const
  994. {
  995. if ( ikmsum->getNumberOfModels() == 0 )
  996. {
  997. fthrow ( Exception, "ikmsum is empty... have you trained this classifer? Aborting..." );
  998. }
  999. // ---------------- compute the first term --------------------
  1000. double kSelf ( 0.0 );
  1001. int dim ( 0 );
  1002. for ( NICE::Vector::const_iterator it = x.begin(); it != x.end(); it++, dim++ )
  1003. {
  1004. kSelf += pf->f ( 0, *it );
  1005. // if weighted dimensions:
  1006. //kSelf += pf->f(dim,*it);
  1007. }
  1008. // ---------------- compute the second term --------------------
  1009. NICE::Vector kStar;
  1010. fmk->hikComputeKernelVector ( x, kStar );
  1011. //now run the ILS method
  1012. NICE::Vector diagonalElements;
  1013. ikmsum->getDiagonalElements ( diagonalElements );
  1014. // init simple jacobi pre-conditioning
  1015. ILSConjugateGradients *linsolver_cg = dynamic_cast<ILSConjugateGradients *> ( linsolver );
  1016. //perform pre-conditioning
  1017. if ( linsolver_cg != NULL )
  1018. linsolver_cg->setJacobiPreconditioner ( diagonalElements );
  1019. NICE::Vector beta;
  1020. /** About finding a good initial solution (see also GPLikelihoodApproximation)
  1021. * K~ = K + sigma^2 I
  1022. *
  1023. * K~ \approx lambda_max v v^T
  1024. * \lambda_max v v^T * alpha = k_* | multiply with v^T from left
  1025. * => \lambda_max v^T alpha = v^T k_*
  1026. * => alpha = k_* / lambda_max could be a good initial start
  1027. * If we put everything in the first equation this gives us
  1028. * v = k_*
  1029. * This reduces the number of iterations by 5 or 8
  1030. */
  1031. beta = (kStar * (1.0 / eigenMax[0]) );
  1032. linsolver->solveLin ( *ikmsum, kStar, beta );
  1033. beta *= kStar;
  1034. double currentSecondTerm( beta.Sum() );
  1035. predVariance = kSelf - currentSecondTerm;
  1036. }
  1037. ///////////////////// INTERFACE PERSISTENT /////////////////////
  1038. // interface specific methods for store and restore
  1039. ///////////////////// INTERFACE PERSISTENT /////////////////////
  1040. void FMKGPHyperparameterOptimization::restore ( std::istream & is, int format )
  1041. {
  1042. bool b_restoreVerbose ( false );
  1043. #ifdef B_RESTOREVERBOSE
  1044. b_restoreVerbose = true;
  1045. #endif
  1046. if ( is.good() )
  1047. {
  1048. if ( b_restoreVerbose )
  1049. std::cerr << " in FMKGP restore" << std::endl;
  1050. std::string tmp;
  1051. is >> tmp; //class name
  1052. if ( ! this->isStartTag( tmp, "FMKGPHyperparameterOptimization" ) )
  1053. {
  1054. std::cerr << " WARNING - attempt to restore FMKGPHyperparameterOptimization, but start flag " << tmp << " does not match! Aborting... " << std::endl;
  1055. throw;
  1056. }
  1057. if (fmk != NULL)
  1058. {
  1059. delete fmk;
  1060. fmk = NULL;
  1061. }
  1062. if ( ikmsum != NULL )
  1063. {
  1064. delete ikmsum;
  1065. }
  1066. ikmsum = new IKMLinearCombination ();
  1067. if ( b_restoreVerbose )
  1068. std::cerr << "ikmsum object created" << std::endl;
  1069. is.precision ( numeric_limits<double>::digits10 + 1 );
  1070. bool b_endOfBlock ( false ) ;
  1071. while ( !b_endOfBlock )
  1072. {
  1073. is >> tmp; // start of block
  1074. if ( this->isEndTag( tmp, "FMKGPHyperparameterOptimization" ) )
  1075. {
  1076. b_endOfBlock = true;
  1077. continue;
  1078. }
  1079. tmp = this->removeStartTag ( tmp );
  1080. if ( b_restoreVerbose )
  1081. std::cerr << " currently restore section " << tmp << " in FMKGPHyperparameterOptimization" << std::endl;
  1082. if ( tmp.compare("fmk") == 0 )
  1083. {
  1084. fmk = new FastMinKernel;
  1085. fmk->restore( is, format );
  1086. is >> tmp; // end of block
  1087. tmp = this->removeEndTag ( tmp );
  1088. }
  1089. else if ( tmp.compare("precomputedA") == 0 )
  1090. {
  1091. is >> tmp; // size
  1092. int preCompSize ( 0 );
  1093. is >> preCompSize;
  1094. precomputedA.clear();
  1095. if ( b_restoreVerbose )
  1096. std::cerr << "restore precomputedA with size: " << preCompSize << std::endl;
  1097. for ( int i = 0; i < preCompSize; i++ )
  1098. {
  1099. int nr;
  1100. is >> nr;
  1101. PrecomputedType pct;
  1102. pct.setIoUntilEndOfFile ( false );
  1103. pct.restore ( is, format );
  1104. precomputedA.insert ( std::pair<int, PrecomputedType> ( nr, pct ) );
  1105. }
  1106. is >> tmp; // end of block
  1107. tmp = this->removeEndTag ( tmp );
  1108. }
  1109. else if ( tmp.compare("precomputedB") == 0 )
  1110. {
  1111. is >> tmp; // size
  1112. int preCompSize ( 0 );
  1113. is >> preCompSize;
  1114. precomputedB.clear();
  1115. if ( b_restoreVerbose )
  1116. std::cerr << "restore precomputedB with size: " << preCompSize << std::endl;
  1117. for ( int i = 0; i < preCompSize; i++ )
  1118. {
  1119. int nr;
  1120. is >> nr;
  1121. PrecomputedType pct;
  1122. pct.setIoUntilEndOfFile ( false );
  1123. pct.restore ( is, format );
  1124. precomputedB.insert ( std::pair<int, PrecomputedType> ( nr, pct ) );
  1125. }
  1126. is >> tmp; // end of block
  1127. tmp = this->removeEndTag ( tmp );
  1128. }
  1129. else if ( tmp.compare("precomputedT") == 0 )
  1130. {
  1131. is >> tmp; // size
  1132. int precomputedTSize ( 0 );
  1133. is >> precomputedTSize;
  1134. precomputedT.clear();
  1135. if ( b_restoreVerbose )
  1136. std::cerr << "restore precomputedT with size: " << precomputedTSize << std::endl;
  1137. if ( precomputedTSize > 0 )
  1138. {
  1139. if ( b_restoreVerbose )
  1140. std::cerr << " restore precomputedT" << std::endl;
  1141. is >> tmp;
  1142. int sizeOfLUT;
  1143. is >> sizeOfLUT;
  1144. for (int i = 0; i < precomputedTSize; i++)
  1145. {
  1146. is >> tmp;
  1147. int index;
  1148. is >> index;
  1149. double * array = new double [ sizeOfLUT];
  1150. for ( int i = 0; i < sizeOfLUT; i++ )
  1151. {
  1152. is >> array[i];
  1153. }
  1154. precomputedT.insert ( std::pair<int, double*> ( index, array ) );
  1155. }
  1156. }
  1157. else
  1158. {
  1159. if ( b_restoreVerbose )
  1160. std::cerr << " skip restoring precomputedT" << std::endl;
  1161. }
  1162. is >> tmp; // end of block
  1163. tmp = this->removeEndTag ( tmp );
  1164. }
  1165. else if ( tmp.compare("precomputedAForVarEst") == 0 )
  1166. {
  1167. int sizeOfAForVarEst;
  1168. is >> sizeOfAForVarEst;
  1169. if ( b_restoreVerbose )
  1170. std::cerr << "restore precomputedAForVarEst with size: " << sizeOfAForVarEst << std::endl;
  1171. if (sizeOfAForVarEst > 0)
  1172. {
  1173. precomputedAForVarEst.clear();
  1174. precomputedAForVarEst.setIoUntilEndOfFile ( false );
  1175. precomputedAForVarEst.restore ( is, format );
  1176. }
  1177. is >> tmp; // end of block
  1178. tmp = this->removeEndTag ( tmp );
  1179. }
  1180. else if ( tmp.compare("precomputedTForVarEst") == 0 )
  1181. {
  1182. std::string isNull;
  1183. is >> isNull; // NOTNULL or NULL
  1184. if ( b_restoreVerbose )
  1185. std::cerr << "content of isNull: " << isNull << std::endl;
  1186. if (isNull.compare("NOTNULL") == 0)
  1187. {
  1188. if ( b_restoreVerbose )
  1189. std::cerr << "restore precomputedTForVarEst" << std::endl;
  1190. int sizeOfLUT;
  1191. is >> sizeOfLUT;
  1192. precomputedTForVarEst = new double [ sizeOfLUT ];
  1193. for ( int i = 0; i < sizeOfLUT; i++ )
  1194. {
  1195. is >> precomputedTForVarEst[i];
  1196. }
  1197. }
  1198. else
  1199. {
  1200. if ( b_restoreVerbose )
  1201. std::cerr << "skip restoring of precomputedTForVarEst" << std::endl;
  1202. if (precomputedTForVarEst != NULL)
  1203. delete precomputedTForVarEst;
  1204. }
  1205. is >> tmp; // end of block
  1206. tmp = this->removeEndTag ( tmp );
  1207. }
  1208. else if ( tmp.compare("eigenMax") == 0 )
  1209. {
  1210. is >> eigenMax;
  1211. is >> tmp; // end of block
  1212. tmp = this->removeEndTag ( tmp );
  1213. }
  1214. else if ( tmp.compare("eigenMaxVectors") == 0 )
  1215. {
  1216. is >> eigenMaxVectors;
  1217. is >> tmp; // end of block
  1218. tmp = this->removeEndTag ( tmp );
  1219. }
  1220. else if ( tmp.compare("ikmsum") == 0 )
  1221. {
  1222. bool b_endOfBlock ( false ) ;
  1223. while ( !b_endOfBlock )
  1224. {
  1225. is >> tmp; // start of block
  1226. if ( this->isEndTag( tmp, "ikmsum" ) )
  1227. {
  1228. b_endOfBlock = true;
  1229. continue;
  1230. }
  1231. tmp = this->removeStartTag ( tmp );
  1232. if ( tmp.compare("IKMNoise") == 0 )
  1233. {
  1234. IKMNoise * ikmnoise = new IKMNoise ();
  1235. ikmnoise->restore ( is, format );
  1236. if ( b_restoreVerbose )
  1237. std::cerr << " add ikmnoise to ikmsum object " << std::endl;
  1238. ikmsum->addModel ( ikmnoise );
  1239. }
  1240. else
  1241. {
  1242. std::cerr << "WARNING -- unexpected ikmsum object -- " << tmp << " -- for restoration... aborting" << std::endl;
  1243. throw;
  1244. }
  1245. }
  1246. }
  1247. else if ( tmp.compare("binaryLabelPositive") == 0 )
  1248. {
  1249. is >> binaryLabelPositive;
  1250. is >> tmp; // end of block
  1251. tmp = this->removeEndTag ( tmp );
  1252. }
  1253. else if ( tmp.compare("binaryLabelNegative") == 0 )
  1254. {
  1255. is >> binaryLabelNegative;
  1256. is >> tmp; // end of block
  1257. tmp = this->removeEndTag ( tmp );
  1258. }
  1259. else if ( tmp.compare("labels") == 0 )
  1260. {
  1261. is >> labels;
  1262. is >> tmp; // end of block
  1263. tmp = this->removeEndTag ( tmp );
  1264. }
  1265. else if ( tmp.compare("b_usePreviousAlphas") == 0 )
  1266. {
  1267. is >> b_usePreviousAlphas;
  1268. is >> tmp; // end of block
  1269. tmp = this->removeEndTag ( tmp );
  1270. }
  1271. else if ( tmp.compare("previousAlphas") == 0 )
  1272. {
  1273. is >> tmp; // size
  1274. int sizeOfPreviousAlphas ( 0 );
  1275. is >> sizeOfPreviousAlphas;
  1276. previousAlphas.clear();
  1277. if ( b_restoreVerbose )
  1278. std::cerr << "restore previousAlphas with size: " << sizeOfPreviousAlphas << std::endl;
  1279. for ( int i = 0; i < sizeOfPreviousAlphas; i++ )
  1280. {
  1281. int classNo;
  1282. is >> classNo;
  1283. NICE::Vector classAlpha;
  1284. is >> classAlpha;
  1285. previousAlphas.insert ( std::pair< int, NICE::Vector > ( classNo, classAlpha ) );
  1286. }
  1287. is >> tmp; // end of block
  1288. tmp = this->removeEndTag ( tmp );
  1289. }
  1290. else
  1291. {
  1292. std::cerr << "WARNING -- unexpected FMKGPHyper object -- " << tmp << " -- for restoration... aborting" << std::endl;
  1293. throw;
  1294. }
  1295. }
  1296. //NOTE are there any more models you added? then add them here respectively in the correct order
  1297. //.....
  1298. //the last one is the GHIK - which we do not have to restore, but simply reset it
  1299. if ( b_restoreVerbose )
  1300. std::cerr << " add GMHIKernel" << std::endl;
  1301. ikmsum->addModel ( new GMHIKernel ( fmk, this->pf, this->q ) );
  1302. if ( b_restoreVerbose )
  1303. std::cerr << " restore positive and negative label" << std::endl;
  1304. knownClasses.clear();
  1305. if ( b_restoreVerbose )
  1306. std::cerr << " fill known classes object " << std::endl;
  1307. if ( precomputedA.size() == 1)
  1308. {
  1309. knownClasses.insert( binaryLabelPositive );
  1310. knownClasses.insert( binaryLabelNegative );
  1311. if ( b_restoreVerbose )
  1312. std::cerr << " binary setting - added corresp. two class numbers" << std::endl;
  1313. }
  1314. else
  1315. {
  1316. for ( std::map<int, PrecomputedType>::const_iterator itA = precomputedA.begin(); itA != precomputedA.end(); itA++)
  1317. knownClasses.insert ( itA->first );
  1318. if ( b_restoreVerbose )
  1319. std::cerr << " multi class setting - added corresp. multiple class numbers" << std::endl;
  1320. }
  1321. }
  1322. else
  1323. {
  1324. std::cerr << "InStream not initialized - restoring not possible!" << std::endl;
  1325. throw;
  1326. }
  1327. }
  1328. void FMKGPHyperparameterOptimization::store ( std::ostream & os, int format ) const
  1329. {
  1330. if ( os.good() )
  1331. {
  1332. // show starting point
  1333. os << this->createStartTag( "FMKGPHyperparameterOptimization" ) << std::endl;
  1334. os << this->createStartTag( "fmk" ) << std::endl;
  1335. fmk->store ( os, format );
  1336. os << this->createEndTag( "fmk" ) << std::endl;
  1337. os.precision ( numeric_limits<double>::digits10 + 1 );
  1338. //we only have to store the things we computed, since the remaining settings come with the config file afterwards
  1339. os << this->createStartTag( "precomputedA" ) << std::endl;
  1340. os << "size: " << precomputedA.size() << std::endl;
  1341. std::map< int, PrecomputedType >::const_iterator preCompIt = precomputedA.begin();
  1342. for ( uint i = 0; i < precomputedA.size(); i++ )
  1343. {
  1344. os << preCompIt->first << std::endl;
  1345. ( preCompIt->second ).store ( os, format );
  1346. preCompIt++;
  1347. }
  1348. os << this->createEndTag( "precomputedA" ) << std::endl;
  1349. os << this->createStartTag( "precomputedB" ) << std::endl;
  1350. os << "size: " << precomputedB.size() << std::endl;
  1351. preCompIt = precomputedB.begin();
  1352. for ( uint i = 0; i < precomputedB.size(); i++ )
  1353. {
  1354. os << preCompIt->first << std::endl;
  1355. ( preCompIt->second ).store ( os, format );
  1356. preCompIt++;
  1357. }
  1358. os << this->createEndTag( "precomputedB" ) << std::endl;
  1359. os << this->createStartTag( "precomputedT" ) << std::endl;
  1360. os << "size: " << precomputedT.size() << std::endl;
  1361. if ( precomputedT.size() > 0 )
  1362. {
  1363. int sizeOfLUT ( 0 );
  1364. if ( q != NULL )
  1365. sizeOfLUT = q->size() * this->fmk->get_d();
  1366. os << "SizeOfLUTs: " << sizeOfLUT << std::endl;
  1367. for ( std::map< int, double * >::const_iterator it = precomputedT.begin(); it != precomputedT.end(); it++ )
  1368. {
  1369. os << "index: " << it->first << std::endl;
  1370. for ( int i = 0; i < sizeOfLUT; i++ )
  1371. {
  1372. os << ( it->second ) [i] << " ";
  1373. }
  1374. os << std::endl;
  1375. }
  1376. }
  1377. os << this->createEndTag( "precomputedT" ) << std::endl;
  1378. //now store the things needed for the variance estimation
  1379. os << this->createStartTag( "precomputedAForVarEst" ) << std::endl;
  1380. os << precomputedAForVarEst.size() << std::endl;
  1381. if (precomputedAForVarEst.size() > 0)
  1382. {
  1383. precomputedAForVarEst.store ( os, format );
  1384. os << std::endl;
  1385. }
  1386. os << this->createEndTag( "precomputedAForVarEst" ) << std::endl;
  1387. os << this->createStartTag( "precomputedTForVarEst" ) << std::endl;
  1388. if ( precomputedTForVarEst != NULL )
  1389. {
  1390. os << "NOTNULL" << std::endl;
  1391. int sizeOfLUT ( 0 );
  1392. if ( q != NULL )
  1393. sizeOfLUT = q->size() * this->fmk->get_d();
  1394. os << sizeOfLUT << std::endl;
  1395. for ( int i = 0; i < sizeOfLUT; i++ )
  1396. {
  1397. os << precomputedTForVarEst[i] << " ";
  1398. }
  1399. os << std::endl;
  1400. }
  1401. else
  1402. {
  1403. os << "NULL" << std::endl;
  1404. }
  1405. os << this->createEndTag( "precomputedTForVarEst" ) << std::endl;
  1406. //store the eigenvalues and eigenvectors
  1407. os << this->createStartTag( "eigenMax" ) << std::endl;
  1408. os << eigenMax << std::endl;
  1409. os << this->createEndTag( "eigenMax" ) << std::endl;
  1410. os << this->createStartTag( "eigenMaxVectors" ) << std::endl;
  1411. os << eigenMaxVectors << std::endl;
  1412. os << this->createEndTag( "eigenMaxVectors" ) << std::endl;
  1413. os << this->createStartTag( "ikmsum" ) << std::endl;
  1414. for ( int j = 0; j < ikmsum->getNumberOfModels() - 1; j++ )
  1415. {
  1416. ( ikmsum->getModel ( j ) )->store ( os, format );
  1417. }
  1418. os << this->createEndTag( "ikmsum" ) << std::endl;
  1419. //store the class numbers for binary settings (if mc-settings, these values will be negative by default)
  1420. os << this->createStartTag( "binaryLabelPositive" ) << std::endl;
  1421. os << binaryLabelPositive << std::endl;
  1422. os << this->createEndTag( "binaryLabelPositive" ) << std::endl;
  1423. os << this->createStartTag( "binaryLabelNegative" ) << std::endl;
  1424. os << binaryLabelNegative << std::endl;
  1425. os << this->createEndTag( "binaryLabelNegative" ) << std::endl;
  1426. os << this->createStartTag( "labels" ) << std::endl;
  1427. os << labels << std::endl;
  1428. os << this->createEndTag( "labels" ) << std::endl;
  1429. os << this->createStartTag( "b_usePreviousAlphas" ) << std::endl;
  1430. os << b_usePreviousAlphas << std::endl;
  1431. os << this->createEndTag( "b_usePreviousAlphas" ) << std::endl;
  1432. os << this->createStartTag( "previousAlphas" ) << std::endl;
  1433. os << "size: " << previousAlphas.size() << std::endl;
  1434. std::map< int, NICE::Vector >::const_iterator prevAlphaIt = previousAlphas.begin();
  1435. for ( uint i = 0; i < previousAlphas.size(); i++ )
  1436. {
  1437. os << prevAlphaIt->first << std::endl;
  1438. os << prevAlphaIt->second << std::endl;
  1439. prevAlphaIt++;
  1440. }
  1441. os << this->createEndTag( "previousAlphas" ) << std::endl;
  1442. // done
  1443. os << this->createEndTag( "FMKGPHyperparameterOptimization" ) << std::endl;
  1444. }
  1445. else
  1446. {
  1447. std::cerr << "OutStream not initialized - storing not possible!" << std::endl;
  1448. }
  1449. }
  1450. void FMKGPHyperparameterOptimization::clear ( ) {};
  1451. ///////////////////// INTERFACE ONLINE LEARNABLE /////////////////////
  1452. // interface specific methods for incremental extensions
  1453. ///////////////////// INTERFACE ONLINE LEARNABLE /////////////////////
  1454. void FMKGPHyperparameterOptimization::addExample( const NICE::SparseVector * example,
  1455. const double & label,
  1456. const bool & performOptimizationAfterIncrement
  1457. )
  1458. {
  1459. if ( this->verbose )
  1460. std::cerr << " --- FMKGPHyperparameterOptimization::addExample --- " << std::endl;
  1461. NICE::Timer t;
  1462. t.start();
  1463. std::set< int > newClasses;
  1464. this->labels.append ( label );
  1465. //have we seen this class already?
  1466. if ( this->knownClasses.find( label ) == this->knownClasses.end() )
  1467. {
  1468. this->knownClasses.insert( label );
  1469. newClasses.insert( label );
  1470. }
  1471. // add the new example to our data structure
  1472. // It is necessary to do this already here and not lateron for internal reasons (see GMHIKernel for more details)
  1473. NICE::Timer tFmk;
  1474. tFmk.start();
  1475. this->fmk->addExample ( example, pf );
  1476. tFmk.stop();
  1477. if ( this->verboseTime)
  1478. std::cerr << "Time used for adding the data to the fmk object: " << tFmk.getLast() << std::endl;
  1479. // add examples to all implicite kernel matrices we currently use
  1480. this->ikmsum->addExample ( example, label, performOptimizationAfterIncrement );
  1481. // update the corresponding matrices A, B and lookup tables T
  1482. // optional: do the optimization again using the previously known solutions as initialization
  1483. this->updateAfterIncrement ( newClasses, performOptimizationAfterIncrement );
  1484. //clean up
  1485. newClasses.clear();
  1486. t.stop();
  1487. NICE::ResourceStatistics rs;
  1488. std::cerr << "Time used for re-learning: " << t.getLast() << std::endl;
  1489. long maxMemory;
  1490. rs.getMaximumMemory ( maxMemory );
  1491. if ( this->verbose )
  1492. std::cerr << "Maximum memory used: " << maxMemory << " KB" << std::endl;
  1493. if ( this->verbose )
  1494. std::cerr << " --- FMKGPHyperparameterOptimization::addExample done --- " << std::endl;
  1495. }
  1496. void FMKGPHyperparameterOptimization::addMultipleExamples( const std::vector< const NICE::SparseVector * > & newExamples,
  1497. const NICE::Vector & newLabels,
  1498. const bool & performOptimizationAfterIncrement
  1499. )
  1500. {
  1501. if ( this->verbose )
  1502. std::cerr << " --- FMKGPHyperparameterOptimization::addMultipleExamples --- " << std::endl;
  1503. NICE::Timer t;
  1504. t.start();
  1505. std::set< int > newClasses;
  1506. this->labels.append ( newLabels );
  1507. //have we seen this class already?
  1508. for ( NICE::Vector::const_iterator vecIt = newLabels.begin();
  1509. vecIt != newLabels.end(); vecIt++
  1510. )
  1511. {
  1512. if ( this->knownClasses.find( *vecIt ) == this->knownClasses.end() )
  1513. {
  1514. this->knownClasses.insert( *vecIt );
  1515. newClasses.insert( *vecIt );
  1516. }
  1517. }
  1518. // add the new example to our data structure
  1519. // It is necessary to do this already here and not lateron for internal reasons (see GMHIKernel for more details)
  1520. NICE::Timer tFmk;
  1521. tFmk.start();
  1522. this->fmk->addMultipleExamples ( newExamples, pf );
  1523. tFmk.stop();
  1524. if ( this->verboseTime)
  1525. std::cerr << "Time used for adding the data to the fmk object: " << tFmk.getLast() << std::endl;
  1526. // add examples to all implicite kernel matrices we currently use
  1527. this->ikmsum->addMultipleExamples ( newExamples, newLabels, performOptimizationAfterIncrement );
  1528. // update the corresponding matrices A, B and lookup tables T
  1529. // optional: do the optimization again using the previously known solutions as initialization
  1530. this->updateAfterIncrement ( newClasses, performOptimizationAfterIncrement );
  1531. //clean up
  1532. newClasses.clear();
  1533. t.stop();
  1534. NICE::ResourceStatistics rs;
  1535. std::cerr << "Time used for re-learning: " << t.getLast() << std::endl;
  1536. long maxMemory;
  1537. rs.getMaximumMemory ( maxMemory );
  1538. if ( this->verbose )
  1539. std::cerr << "Maximum memory used: " << maxMemory << " KB" << std::endl;
  1540. if ( this->verbose )
  1541. std::cerr << " --- FMKGPHyperparameterOptimization::addMultipleExamples done --- " << std::endl;
  1542. }