Matlab_GPMultiClass/evaluateIncrementalLearning.m

function evaluateIncrementalLearning ( settings )
% compare multi-class incremental learning with Gaussian processes using
% either efficient model updates or batch training from scratch
%
% Copyright (c) by Alexander Freytag, 2013-11-13.


    if ( (nargin < 1) || isempty ( settings) )
        settings = [];
    end

    %setup variables for experiments

    if ( ~isfield(  settings, 'i_numDim') || isempty(settings.i_numDim) )
        settings.i_numDim = 10;
    end
    if ( ~isfield(  settings, 'd_mean') || isempty(settings.d_mean) )
        settings.d_mean = 3.0;
    end
    if ( ~isfield(  settings, 'd_var') || isempty(settings.d_var) )
        settings.d_var = 1.0;
    end

    if ( ~isfield(  settings, 'offsetSpecific') || isempty(settings.offsetSpecific) )
        settings.offsetSpecific = [ 0.0, 2.0, 4.0 ];
    end
    if ( ~isfield(  settings, 'varSpecific') || isempty(settings.varSpecific) )
        settings.varSpecific = [2.0, 0.5, 1.0];
    end
    if ( ~isfield(  settings, 'dimToChange') || isempty(settings.dimToChange) )
        settings.dimToChange = [ [1,3] , [2,4] , [5] ];
    end

    if ( ~isfield(  settings, 'i_numClasses') || isempty(settings.i_numClasses) )
        settings.i_numClasses = 3;
    end
    
    if ( ~isfield(  settings, 'i_numSamplesPerClassTrain') || isempty(settings.i_numSamplesPerClassTrain) )
        settings.i_numSamplesPerClassTrain = 10;    
    end
    if ( ~isfield(  settings, 'i_numSamplesPerClassUnlabeled') || isempty(settings.i_numSamplesPerClassUnlabeled) )
        settings.i_numSamplesPerClassUnlabeled = 50;
    end
    if ( ~isfield(  settings, 'i_numSamplesPerClassTest') || isempty(settings.i_numSamplesPerClassTest) )
        settings.i_numSamplesPerClassTest = 10;
    end

    % how many experiments do you want to average?
    if ( ~isfield(  settings, 'i_numRepetitions') || isempty(settings.i_numRepetitions) )
        i_numRepetitions = 10;
    end


    timesBatch = zeros ( i_numRepetitions, (settings.i_numClasses*settings.i_numSamplesPerClassUnlabeled+1) );
    timesEfficient = zeros ( i_numRepetitions, (settings.i_numClasses*settings.i_numSamplesPerClassUnlabeled+1) );
    
    arrBatch = zeros ( i_numRepetitions, (settings.i_numClasses*settings.i_numSamplesPerClassUnlabeled+1) );
    arrEfficient = zeros ( i_numRepetitions, (settings.i_numClasses*settings.i_numSamplesPerClassUnlabeled+1) );    
    
    for i_idxRep = 1:i_numRepetitions
        
        %%  (1) sample some features for initial train set, separate test set, and additional separate set to be added incrementally

        % sample training data
        settings.i_numSamplesPerClass = settings.i_numSamplesPerClassTrain;
        labeledData = sampleData( settings );


        % sample unlabeled data
        settings.i_numSamplesPerClass = settings.i_numSamplesPerClassUnlabeled;
        unlabeledData = sampleData( settings );


        % sample test data
        settings.i_numSamplesPerClass = settings.i_numSamplesPerClassTest;
        testData = sampleData( settings );

        % in which order to we want to add the 'unlabeled' sampels?
        idxToAdd = randperm( length(unlabeledData.y) ); 



        %%  (2) run both techniques : efficient updates versus batch training

        b_efficientUpdates = false;
        resultsBatch = simulateIncrementalLearning ( labeledData, unlabeledData, testData, b_efficientUpdates, idxToAdd );
        
        timesBatch(i_idxRep, :) = resultsBatch.times;
        arrBatch(i_idxRep, :) = resultsBatch.ARR;
        clear ( 'resultsBatch' );
        
        
        

        b_efficientUpdates = true;
        resultsEfficient = simulateIncrementalLearning ( labeledData, unlabeledData, testData, b_efficientUpdates, idxToAdd );
        
        timesEfficient(i_idxRep, :) = resultsEfficient.times;
        arrEfficient(i_idxRep, :) = resultsEfficient.ARR;
        clear ( 'resultsEfficient' );
        
        
    end


    %%  (3) evaluation
    
    % compute relevant values
    timesBatch = mean ( timesBatch ) ;
    timesEfficient = mean ( timesEfficient ) ;
    
    arrBatch = mean( arrBatch );
    arrEfficient = mean ( arrEfficient );      
    
    % setup variables
    if ( ( ~isfield(settings,'s_legendLocation'))  || isempty(settings.s_legendLocation) )
       s_legendLocation = 'NorthEast';
    else
       s_legendLocation = settings.s_legendLocation;
    end     
  
    if ( ( ~isfield(settings,'i_fontSize'))  || isempty(settings.i_fontSize) )
       i_fontSize = 12;
    else
       i_fontSize = settings.i_fontSize;
    end  
  
    if ( ( ~isfield(settings,'i_fontSizeAxis'))  || isempty(settings.i_fontSizeAxis) )
       i_fontSizeAxis = 16;
    else
       i_fontSizeAxis = settings.i_fontSizeAxis;
    end
    
    if ( ( ~isfield(settings,'c'))  || isempty(settings.c) )
        c={[0,0,1], [1,0,0]};
    else
        c = settings.c;
    end  
  
    if ( ( ~isfield(settings,'lineStyle'))  || isempty(settings.lineStyle) )
        lineStyle={'-',   '--' };
    else
        lineStyle = settings.lineStyle;
    end 
  
    if ( ( ~isfield(settings,'marker'))  || isempty(settings.marker) )
        marker={'none', 'none'};
    else
        marker = settings.marker;
    end 
    
    if ( ( ~isfield(settings,'linewidth'))  || isempty(settings.linewidth) )
        linewidth=3;
    else
        linewidth = settings.linewidth;
    end      
    
    %% plot computation times (blue should be lower than red)
    fig_timeComp = figure;
    set ( fig_timeComp, 'name', 'Computation Times for Model Updates');
    hold on;

    plot ( timesEfficient, 'Color', c{ 1 }, 'LineStyle', lineStyle{ 1 },  'Marker', marker{ 1 }, ...
                'LineWidth', linewidth, 'MarkerSize',8 );
    plot ( timesBatch, 'Color', c{ 2 }, 'LineStyle', lineStyle{ 2 },  'Marker', marker{ 2 }, ...
                'LineWidth', linewidth, 'MarkerSize',8);
    
    leg=legend( {'Efficient', 'Batch'}, 'Location', s_legendLocation,'fontSize', i_fontSize,'LineWidth', 3);
    xlabel('Number of samples added');
    ylabel({'Time spent for model update [s]'});    
    
    text_h=findobj(gca,'type','text');  
    set(text_h,'FontSize',i_fontSize);
    set(gca, 'FontSize', i_fontSize);
    set(get(gca,'YLabel'), 'FontSize', i_fontSizeAxis);
    set(get(gca,'XLabel'), 'FontSize', i_fontSizeAxis);    

    hold off;


    %% plot accuracies (blue and red should be identical)
    fig_accuracyComp = figure;
    set ( fig_accuracyComp, 'name', 'Accuracy over time');
    hold on;

    plot ( arrEfficient, 'Color', c{ 1 }, 'LineStyle', lineStyle{ 1 },  'Marker', marker{ 1 }, ...
                'LineWidth', linewidth, 'MarkerSize',8 );
    plot ( arrBatch, 'Color', c{ 2 }, 'LineStyle', lineStyle{ 2 },  'Marker', marker{ 2 }, ...
                'LineWidth', linewidth, 'MarkerSize',8);
    leg=legend( {'Efficient', 'Batch'}, 'Location', s_legendLocation,'fontSize', i_fontSize,'LineWidth', 3);
    xlabel('Number of samples added');
    ylabel({'Time spent for model update [s]'});    
    
    text_h=findobj(gca,'type','text');  
    set(text_h,'FontSize',i_fontSize);
    set(gca, 'FontSize', i_fontSize);
    set(get(gca,'YLabel'), 'FontSize', i_fontSizeAxis);
    set(get(gca,'XLabel'), 'FontSize', i_fontSizeAxis);    

    hold off;


    pause;
    close ( fig_timeComp );
    close ( fig_accuracyComp );
    
end    
    
function data = sampleData( settings )

    if ( isfield(  settings, 'i_numDim') && ~isempty(settings.i_numDim) )
        i_numDim = settings.i_numDim;
    else
        i_numDim = 5;
    end
    

    if ( isfield(  settings, 'd_mean') && ~isempty(settings.d_mean) )
        d_mean = settings.d_mean;
    else
        d_mean = 3.0;
    end
    

    if ( isfield(  settings, 'd_var') && ~isempty(settings.d_var) )
        d_var = settings.d_var;
    else
        d_var = 1.0;
    end
    
    if ( isfield(  settings, 'i_numClasses') && ~isempty(settings.i_numClasses) )
        i_numClasses = settings.i_numClasses;
    else
        i_numClasses = 3;
    end
    
    if ( isfield(  settings, 'offsetSpecific') && ~isempty(settings.offsetSpecific) )
        offsetSpecific = settings.offsetSpecific;
    else
        offsetSpecific = [ 0.0, 2.0, 4.0 ];
    end
 

    if ( isfield(  settings, 'varSpecific') && ~isempty(settings.varSpecific) )
        varSpecific = settings.varSpecific;
    else
        varSpecific = [2.0, 0.5, 1.0];
    end
    

    if ( isfield(  settings, 'dimToChange') && ~isempty(settings.dimToChange) )
        dimToChange = settings.dimToChange;
    else
        dimToChange = [ [1,3] , [2,4] , [5] ];
    end
    


    
    if ( isfield(  settings, 'i_numSamplesPerClass') && ~isempty(settings.i_numSamplesPerClass) )
        i_numSamplesPerClass = settings.i_numSamplesPerClass;
    else
        i_numSamplesPerClass = 2;
    end    
    

    % randomly compute some features
    data.X = abs ( d_mean + d_var.*randn(i_numClasses*i_numSamplesPerClass,i_numDim) );

    % disturbe features for specific classes
    for i_clIdx = 1:i_numClasses

        i_idxStart = (i_clIdx-1)*i_numSamplesPerClass+1;
        i_idxEnd = i_clIdx*i_numSamplesPerClass;

        data.X(  i_idxStart:i_idxEnd,  dimToChange(i_clIdx) ) = ...
                    abs ( data.X(i_idxStart:i_idxEnd,1)  + ... 
                    offsetSpecific( i_clIdx) + ... %add class offset
                    varSpecific ( i_clIdx) .*randn(i_numSamplesPerClass,1) ... % add class variance
                 );
    end


    % normalize features, thereby simulate histograms, which commonly occure in 
    % computer vision applications
    data.X = bsxfun(@times, data.X, 1./(sum(data.X, 2)));

    % adapt class labels
    data.y =  bsxfun(@times, ones(i_numSamplesPerClass,1), 1:i_numClasses );
    data.y = data.y(:);

end