comprehensive metrics and explantions in intro notebook

README.md

@@ -6,7 +6,10 @@ Further the camera trap data and the way it was handled is described. The corres
 
 After cloning the package, create a new envionrment with python 3.8:
 
-conda create --name <nameofenvironement> python=3.8
+conda create --name *nameofenvironement* python=3.8
+
+Then activate the environment
+ conda activate *nameofenvironement*
 
 Next, navigate the avalanche folder containing the setup.py script. Use the following commad to get dependencies and install avalanche the environment in developer mode, meaning that changes made in the avalanche folder are seen by the environment:
 pip install -e . 
@@ -21,5 +24,5 @@ https://pytorch.org/get-started/locally/
 
 Lastly, use this command to use the envionrment in a jupyter notebook:
 
-python -m ipykernel install --user --name=<nameofenvironement>
+python -m ipykernel install --user --name=*nameofenvironement*
 

avalanche/avalanche/__init__.py

@@ -1,6 +1,6 @@
 import os
 from .logging import *
-from .evaluation import *
+from .evaluation import * 
 from .benchmarks import *
 from .models import *
 from .training import *

avalanche/avalanche/benchmarks/scenarios/generic_benchmark_creation.py

@@ -555,20 +555,15 @@ def create_generic_benchmark_from_paths(
 
     for stream_name, lists_of_files in input_streams.items():
         stream_datasets = []
+
         for exp_id, list_of_files in enumerate(lists_of_files):
-            #_, exp_paths_list = common_paths_root(list_of_files)
             paths_dataset = path_dataset_class(root=common_root,files=list_of_files)
-            #print('paths.dataset has field paths: ')
-            #print(paths_dataset.paths)
-            #print(dir(paths_dataset))
 
             stream_datasets.append(AvalancheDataset(
                 paths_dataset,
                 task_labels=task_labels[exp_id],
                 paths = paths_dataset.paths))
-            #print('in avalanche dataset wrap')
-            #print(stream_datasets[-1].paths)
-            #print(dir(stream_datasets[-1]._dataset))
+
             
 
         stream_definitions[stream_name] = stream_datasets

avalanche/avalanche/core.py

@@ -16,6 +16,8 @@ Strategy Callbacks definition that can be used by the :py:mod:`training`
 module for defining new continual learning strategies and by the
 :py:mod:`evaluation` module for defining new evaluation plugin metrics.
 """
+__all__ = ["StrategyCallbacks"]
+
 
 from abc import ABC
 from typing import Generic, TypeVar

avalanche/avalanche/evaluation/__init__.py

@@ -6,3 +6,4 @@ the `EvaluationPlugin`.
 """
 
 from .metric_definitions import *
+from avalanche.evaluation.metrics import *

avalanche/avalanche/evaluation/metric_definitions.py

@@ -13,7 +13,7 @@ from abc import ABC, abstractmethod
 from typing import TypeVar, Optional, TYPE_CHECKING
 from typing_extensions import Protocol
 from .metric_results import MetricValue
-from .metric_utils import get_metric_name, phase_and_task
+from .metric_utils import get_metric_name, phase_and_task, custom_get_metric_name
 from ..core import StrategyCallbacks
 
 if TYPE_CHECKING:
@@ -237,19 +237,33 @@ class GenericPluginMetric(PluginMetric[TResult]):
         plot_x_position = self.get_global_counter()
 
         if isinstance(metric_value, dict):
-            metrics = []
-            for k, v in metric_value.items():
-                metric_name = get_metric_name(
-                    self, strategy, add_experience=add_exp, add_task=k)
-                metrics.append(MetricValue(self, metric_name, v,
-                                           plot_x_position))
+            metrics =[]
+
+            if -1 in metric_value.keys():
+                for k, v in metric_value.items():
+                    if k != -1:
+                        metric_name = custom_get_metric_name(self, strategy, experience_or_class=k)
+                        if len(v.items()) >0:
+                            metrics.append(MetricValue(self, metric_name, v[0],plot_x_position))
+                        else: 
+                            metrics.append(MetricValue(self, metric_name, 0,plot_x_position))
+            else:
+                for k, v in metric_value.items():
+                    metric_name = get_metric_name(
+                        self, strategy, add_experience=add_exp, add_task=k)
+
+                    metrics.append(MetricValue(self, metric_name, v,
+                                               plot_x_position))
             return metrics
         else:
             metric_name = get_metric_name(self, strategy,
                                           add_experience=add_exp,
                                           add_task=True)
-            return [MetricValue(self, metric_name, metric_value,
-                                plot_x_position)]
+            mv =metric_value
+            metric_value =MetricValue(self, metric_name, metric_value,
+                                plot_x_position)
+
+            return [metric_value]
 
     def before_training(self, strategy: 'BaseStrategy'):
         super().before_training(strategy)

avalanche/avalanche/evaluation/metric_utils.py

@@ -323,10 +323,29 @@ def get_metric_name(metric: 'PluginMetric',
         return base_name + task_name
 
 
+def custom_get_metric_name(metric: 'PluginMetric',
+                    strategy: 'BaseStrategy',
+                    experience_or_class=False):
+
+    phase_name, task_label = phase_and_task(strategy)
+    stream = stream_type(strategy.experience)
+    base_name = '{}/{}_phase/{}_stream'.format(str(metric),
+                                               phase_name, stream)
+    
+
+
+    if experience_or_class is not None:
+        return base_name + '/'+ str(experience_or_class)
+    else:
+        return base_name
+
+
+
 __all__ = [
     "default_cm_image_creator",
     "phase_and_task",
     "get_metric_name",
+    "custom_get_metric_name",
     "stream_type",
     "bytes2human",
     "default_history_repartition_image_creator",

avalanche/avalanche/logging/__init__.py

@@ -16,4 +16,4 @@ from .tensorboard_logger import *
 from .wandb_logger import *
 from .text_logging import TextLogger
 from .interactive_logging import InteractiveLogger
-from .csv_logger import CSVLogger
+from .csv_logger import CSVLogger, GenericCSVLogger

avalanche/avalanche/logging/csv_logger.py

@@ -13,6 +13,7 @@ from typing import List, TYPE_CHECKING
 
 import torch
 import os
+import datetime 
 
 from avalanche.evaluation.metric_results import MetricValue
 from avalanche.logging import StrategyLogger
@@ -173,3 +174,265 @@ class CSVLogger(StrategyLogger):
     def close(self):
         self.training_file.close()
         self.eval_file.close()
+
+
+
+
+class GenericCSVLogger(StrategyLogger):
+    #  A more comprehensive CSV Logger capable of logging new metrics
+    
+
+
+    def __init__(self, log_folder=None):
+
+
+        super().__init__()
+        self.log_folder = log_folder if log_folder is not None else "csvlogs"
+        os.makedirs(self.log_folder, exist_ok=True)
+
+
+        datetime_stap = str(datetime.datetime.now()).replace(' ','_').replace(':','-').replace('.','-')[:-4]
+        self.training_epoch_file = open(os.path.join(self.log_folder, '_training_epochs.csv'), 'w')
+        self.test_stream_file = open(os.path.join(self.log_folder,'_test_stream.csv'), 'w')
+        self.validation_stream_file = open(os.path.join(self.log_folder,'_validation_stream.csv'), 'w')
+        self.training_stream_file = open(os.path.join(self.log_folder, '_training_stream.csv'), 'w')
+        self.transfer_file = open(os.path.join(self.log_folder,'_training_transfer.csv'), 'w')
+        
+        os.makedirs(self.log_folder, exist_ok=True)
+
+        # current training experience id
+        self.training_exp_id = None
+
+        # if we are currently training or evaluating
+        # evaluation within training will not change this flag
+        self.in_train_phase = None
+
+        # validation metrics computed during training
+        self.val_acc, self.val_loss = 0, 0
+
+
+    def log_metric(self, metric_value: 'MetricValue', callback: str) -> None:
+        pass
+
+    def _val_to_str(self, m_val):
+        if isinstance(m_val, torch.Tensor):
+            return '\n' + str(m_val)
+        elif isinstance(m_val, float):
+            return f'{m_val:.4f}'
+        else:
+            return str(m_val)
+
+    def print_vals_to_file(self, val_list, mode, exp, epoch, strategy):
+        add_classes = False
+        if mode =='test':
+            classes_in_last_exp = [str(i)+' ' for i in self.classes_in_last_exp]
+            classes_in_last_exp = ''.join(classes_in_last_exp)
+            classes_in_last_exp = '['+classes_in_last_exp+']'
+            add_classes = True
+            log_file = self.test_stream_file
+
+        elif mode =='validation':
+            classes_in_last_exp = [str(i)+' ' for i in self.classes_in_last_exp]
+            classes_in_last_exp = ''.join(classes_in_last_exp)
+            classes_in_last_exp = '['+classes_in_last_exp+']'
+            add_classes = True
+            log_file = self.validation_stream_file
+        
+        elif mode =='train_epoch':
+            log_file = self.training_epoch_file
+
+        elif mode== 'train_stream':
+            classes_in_last_exp = [str(i)+' ' for i in self.classes_in_last_exp]
+            classes_in_last_exp = ''.join(classes_in_last_exp)
+            classes_in_last_exp = '['+classes_in_last_exp+']'
+            add_classes = True
+            log_file = self.training_stream_file
+            
+        elif mode == 'transfer':
+            log_file = self.transfer_file
+
+            if exp == 0 and (epoch+1)==strategy.train_epochs:
+                nr_exp = len(val_list)
+                exp_list = torch.arange(nr_exp)
+                exp_list = ['Exp '+ str(i.item()) for i in exp_list]
+                exp_list = ['train_exp', 'epoch']+ exp_list
+                print(*exp_list, sep=',', file=log_file, flush=True)
+     
+
+        val_list = [self._val_to_str(i) for i in val_list]
+        if add_classes: 
+            log_list = [exp, epoch, classes_in_last_exp ]+ val_list
+        else:
+            log_list = [exp, epoch ]+ val_list
+        print(*log_list, sep=',', file=log_file, flush=True)
+
+
+
+    def after_training_epoch(self, strategy: 'BaseStrategy',
+                             metric_values: List['MetricValue'], **kwargs):
+        super().after_training_epoch(strategy, metric_values, **kwargs)
+        train_vals = [None] * len(self.train_metrics)
+        train_exp = False
+        metrics_found = False
+        for val in metric_values:
+            if 'train_stream' in val.name:
+                train_exp = True
+                metrics_found = True
+                val_index = self.train_metrics.index(val.name.split('/')[0])
+                train_vals[val_index] = val.value
+
+        if train_exp:
+            self.print_vals_to_file(train_vals, mode='train_epoch', exp = self.training_exp_id,epoch= strategy.epoch, strategy=strategy)
+
+
+
+    def after_eval_exp(self, strategy: 'BaseStrategy',
+                       metric_values: List['MetricValue'], **kwargs):
+        super().after_eval_exp(strategy, metric_values, **kwargs)
+
+        metrics_found = False
+        if metrics_found:
+            if not train_eval_exp :
+                self.print_vals_to_file(eval_vals, mode ='eval', exp = self.training_exp_id,epoch= strategy.epoch, strategy=strategy)
+            else: 
+                self.print_vals_to_file(train_vals, mode ='train', exp = self.training_exp_id,epoch= strategy.epoch, strategy=strategy)
+
+    def after_eval(self, strategy: 'BaseStrategy',
+                       metric_values: List['MetricValue'], **kwargs):
+        super().after_eval_exp(strategy, metric_values, **kwargs)
+        train_metrics_found = False
+        validation_metrics_found = False
+        test_metrics_found = False
+        train_stream_accs = []
+        train_stream_vals = [None]*len(self.eval_metrics)
+        test_stream_vals = [None]*len(self.eval_metrics)
+        validation_stream_vals = [None]*len(self.eval_metrics)
+
+        classes_acc = []
+        seq_classes_acc = []
+        for val in metric_values:
+            if 'eval_phase' in val.name: # validation within training
+                if 'Transfer' in val.name :
+                    if 'train_stream' in val.name:
+                            train_metrics_found = True
+                            train_stream_accs.append(val.value)
+
+
+                else:
+                    if 'train_stream' in val.name:
+                        train_metrics_found = True
+                        if 'SeqClasswise' in val.name:
+                            seq_classes_acc.append(val.value)
+                        elif 'Classwise' in val.name:
+                            classes_acc.append(val.value)
+                        else:
+                            val_index = self.eval_metrics.index(val.name.split('/')[0])
+                            train_stream_vals[val_index] = val.value
+        
+                    if 'test_stream' in val.name:
+                        test_metrics_found = True
+                        if 'SeqClasswise' in val.name:
+                            seq_classes_acc.append(val.value)
+                        elif 'Classwise' in val.name:
+                            classes_acc.append(val.value)
+                        else:
+                            val_index = self.eval_metrics.index(val.name.split('/')[0])
+                            test_stream_vals[val_index] = val.value 
+
+                    if 'validation_stream' in val.name:
+                        validation_metrics_found = True
+                        if 'SeqClasswise' in val.name:
+                            seq_classes_acc.append(val.value)
+                        elif 'Classwise' in val.name:
+                            classes_acc.append(val.value)
+                        else:
+                            val_index = self.eval_metrics.index(val.name.split('/')[0])
+                            validation_stream_vals[val_index] = val.value 
+            
+        if test_metrics_found:
+            if classes_acc != []:
+                    test_stream_vals = test_stream_vals+classes_acc
+            if seq_classes_acc != []:
+                    test_stream_vals = test_stream_vals+seq_classes_acc
+            self.print_vals_to_file(test_stream_vals, mode ='test', exp = self.training_exp_id,epoch= strategy.epoch, strategy=strategy)
+
+        if validation_metrics_found:
+            if classes_acc != []:
+                    validation_stream_vals = validation_stream_vals+classes_acc
+            if seq_classes_acc != []:
+                    validation_stream_vals = validation_stream_vals+seq_classes_acc
+            self.print_vals_to_file(validation_stream_vals, mode ='validation', exp = self.training_exp_id,epoch= strategy.epoch, strategy=strategy)
+
+        if train_metrics_found:
+            if  train_stream_accs != []:
+                self.print_vals_to_file(train_stream_accs,  mode ='transfer', exp=self.training_exp_id, epoch=strategy.epoch, strategy=strategy)
+            if not None in train_stream_vals:
+                if classes_acc != []:
+                    train_stream_vals = train_stream_vals+classes_acc
+                if seq_classes_acc != []:
+                    train_stream_vals = train_stream_vals+seq_classes_acc
+                self.print_vals_to_file(train_stream_vals, mode='train_stream', exp=self.training_exp_id, epoch=strategy.epoch, strategy=strategy)
+
+    def before_training_exp(self, strategy: 'BaseStrategy',
+                            metric_values: List['MetricValue'], **kwargs):
+        super().before_training(strategy, metric_values, **kwargs)
+        self.training_exp_id = strategy.experience.current_experience
+        strategy.current_train_exp_seen = strategy.experience.current_experience
+        self.classes_in_last_exp = strategy.experience.classes_in_this_experience
+
+    def before_eval(self, strategy: 'BaseStrategy',
+                    metric_values: List['MetricValue'], **kwargs):
+        """
+        Manage the case in which `eval` is first called before `train`
+        """
+
+        if self.in_train_phase is None:
+            self.in_train_phase = False
+
+    def before_training(self, strategy: 'BaseStrategy',
+                        metric_values: List['MetricValue'], **kwargs):
+        if strategy.experience == None:
+
+            self.train_metrics =[]
+            self.eval_metrics = []
+            logging_classes_accs = False
+            seq_logging_classes_accs = False
+            for metric in self.all_metrics:
+
+                if metric._mode == 'train':
+                    self.train_metrics.append(str(metric))
+                else: 
+                    if 'SeqClasswise' in str(metric):
+                        seq_logging_classes_accs =True
+                    elif 'Classwise' in str(metric):
+                        logging_classes_accs =True
+                    elif not 'Transfer' in str(metric) :
+                        self.eval_metrics.append(str(metric))
+
+            train_head = ['train_exp', 'epoch']+ self.train_metrics
+            eval_head = ['train_exp', 'epoch', 'train exp classes']+ self.eval_metrics
+            if logging_classes_accs:
+                classes_head = ['Cls'+str(i.item()) for i in torch.arange(strategy.model.num_classes)]
+                eval_head = eval_head +classes_head
+            if seq_logging_classes_accs:
+                classes_head = ['SeqCls'+str(i.item()) for i in torch.arange(strategy.model.num_classes)]
+                eval_head = eval_head +classes_head
+
+            print(*train_head, sep=',', file=self.training_epoch_file, flush=True)
+            print(*eval_head, sep=',', file=self.validation_stream_file, flush=True)
+            print(*eval_head, sep=',', file=self.test_stream_file, flush=True)
+            print(*eval_head, sep=',', file=self.training_stream_file, flush=True)
+
+
+        self.in_train_phase = True
+
+    def after_training(self, strategy: 'BaseStrategy',
+                       metric_values: List['MetricValue'], **kwargs):
+        self.in_train_phase = False
+
+    def close(self):
+        self.training_epoch_file.close()
+        self.validation_stream_file.close()
+        self.test_stream_file.close()
+        self.training_stream_file.close()
+        self.transfer_file.close()

avalanche/avalanche/training/plugins/__init__.py

@@ -1,6 +1,6 @@
 from .agem import AGEMPlugin
 from .cwr_star import CWRStarPlugin
-from .evaluation import EvaluationPlugin
+from .evaluation_plugin import EvaluationPlugin
 from .ewc import EWCPlugin
 from .gdumb import GDumbPlugin
 from .gem import GEMPlugin

avalanche/avalanche/training/plugins/evaluation.py → avalanche/avalanche/training/plugins/evaluation_plugin.py

@@ -5,14 +5,18 @@ from typing import Union, Sequence, TYPE_CHECKING
 
 from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics
 from avalanche.training.plugins.strategy_plugin import StrategyPlugin
-from avalanche.logging import StrategyLogger, InteractiveLogger
+from avalanche.logging import StrategyLogger, InteractiveLogger, CSVLogger, GenericCSVLogger
 
 if TYPE_CHECKING:
     from avalanche.evaluation import PluginMetric
     from avalanche.logging import StrategyLogger
     from avalanche.training.strategies import BaseStrategy
 
-
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import pickle as pkl
+import datetime
 class EvaluationPlugin(StrategyPlugin):
     """
     An evaluation plugin that obtains relevant data from the
@@ -29,8 +33,7 @@ class EvaluationPlugin(StrategyPlugin):
 
     def __init__(self,
                  *metrics: Union['PluginMetric', Sequence['PluginMetric']],
-                 loggers: Union['StrategyLogger',
-                                Sequence['StrategyLogger']] = None,
+                 loggers: Union['StrategyLogger', Sequence['StrategyLogger']] = None,
                  collect_all=True,
                  benchmark=None,
                  strict_checks=False,
@@ -62,11 +65,22 @@ class EvaluationPlugin(StrategyPlugin):
         self.strict_checks = strict_checks
         self.suppress_warnings = suppress_warnings
         flat_metrics_list = []
+        time_metric = None
+
         for metric in metrics:
             if isinstance(metric, Sequence):
-                flat_metrics_list += list(metric)
+                for m in metric:
+                    if 'Time' in str(m):
+                        time_metric = m
+                    else:
+                        flat_metrics_list.append(m)
             else:
-                flat_metrics_list.append(metric)
+                if 'Time' in str(metric):
+                    time_metric = m
+                else:
+                    flat_metrics_list.append(metric)
+        if time_metric != None:
+            flat_metrics_list.append(time_metric)
         self.metrics = flat_metrics_list
 
         if loggers is None:
@@ -92,6 +106,10 @@ class EvaluationPlugin(StrategyPlugin):
         if len(self.loggers) == 0:
             warnings.warn('No loggers specified, metrics will not be logged')
 
+        for logger in loggers:
+            if isinstance(logger, CSVLogger) or isinstance(logger, GenericCSVLogger):
+                #print(logger)
+                logger.all_metrics = self.metrics
         if self.collect_all:
             # for each curve collect all emitted values.
             # dictionary key is full metric name.
@@ -122,15 +140,19 @@ class EvaluationPlugin(StrategyPlugin):
     def _update_metrics(self, strategy: 'BaseStrategy', callback: str):
         if not self._active:
             return []
-
         metric_values = []
+        non_none_counter = 0
         for metric in self.metrics:
             metric_result = getattr(metric, callback)(strategy)
             if isinstance(metric_result, Sequence):
+                non_none_counter +=1
                 metric_values += list(metric_result)
+
             elif metric_result is not None:
-                metric_values.append(metric_result)
+                non_none_counter +=1
 
+                metric_values.append(metric_result)
+            
         for metric_value in metric_values:
             name = metric_value.name
             x = metric_value.x_plot
@@ -143,6 +165,7 @@ class EvaluationPlugin(StrategyPlugin):
 
         for logger in self.loggers:
             getattr(logger, callback)(strategy, metric_values)
+
         return metric_values
 
     def get_last_metrics(self):
@@ -229,7 +252,7 @@ class EvaluationPlugin(StrategyPlugin):
 
     def after_training(self, strategy: 'BaseStrategy', **kwargs):
         self._update_metrics(strategy, 'after_training')
-
+        
     def before_eval(self, strategy: 'BaseStrategy', **kwargs):
         self._update_metrics(strategy, 'before_eval')
         msgw = "Evaluation stream is not equal to the complete test stream. " \
@@ -268,6 +291,103 @@ class EvaluationPlugin(StrategyPlugin):
     def after_eval(self, strategy: 'BaseStrategy', **kwargs):
         self._update_metrics(strategy, 'after_eval')
 
+        if strategy.last_eval:
+            plt.figure(figsize=(27,18))
+            tick_marks = np.arange(strategy.model.num_classes)
+            confusion_data_dict = {'tick_marks': tick_marks}
+
+            title =  'Acc: '+ str(np.round(strategy.top1_acc*100, 2))+' ClsAvgAcc: '+str(np.round(strategy.cls_top1_avg_acc*100, 2))
+            title = title +'\n'+ 'SeqAcc: '+str(np.round(strategy.seq_top1_acc*100, 2)) + ' SeqClsAcc: '+ str(np.round(strategy.seq_cls_top1_avg_acc*100, 2))
+            title = title + ' SeqAnyAcc'+str(np.round(strategy.seq_any_acc*100, 2))
+
+            cmt = np.zeros((strategy.model.num_classes,strategy.model.num_classes))
+            strategy.all_preds = strategy.all_preds.cpu()
+            strategy.all_targets = strategy.all_targets.cpu()
+            
+            if strategy.experience.origin_stream.name =='validation': 
+                file_name = '_after_exp'+str(strategy.current_train_exp_seen)+'_validation'
+                class_acc_dict = strategy.val_cls_acc_dict
+            if strategy.experience.origin_stream.name =='test': 
+                file_name = '_after_exp'+str(strategy.current_train_exp_seen)+'_test'
+                class_acc_dict = strategy.test_cls_acc_dict
+            if strategy.experience.origin_stream.name =='train': 
+                file_name = '_after_exp'+str(strategy.current_train_exp_seen)+'_train'
+                class_acc_dict = strategy.train_cls_acc_dict
+            
+
+            confusion_data_dict['title'] = title
+            confusion_data_dict['top1_acc'] = strategy.top1_acc
+            confusion_data_dict['cls_top1_avg_acc'] = strategy.cls_top1_avg_acc
+            confusion_data_dict['seq_top1_acc'] = strategy.seq_top1_acc
+            confusion_data_dict['seq_cls_top1_avg_acc'] = strategy.seq_cls_top1_avg_acc
+            confusion_data_dict['seq_any_acc'] = strategy.seq_any_acc
+
+            confusion_data_dict['all_preds'] = strategy.all_preds
+            confusion_data_dict['all_targets'] = strategy.all_targets
+            confusion_data_dict['label_dict'] = strategy.label_dict
+            confusion_data_dict['cls_acc_dict'] = class_acc_dict
+            confusion_data_dict['unique_train_cls_dict'] = strategy.unique_train_cls_dict
+            confusion_data_dict['total_train_cls_dict'] = strategy.total_train_cls_dict
+            confusion_data_dict['total_test_cls_dict'] = strategy.total_test_cls_dict
+            confusion_data_dict['total_validation_cls_dict'] = strategy.total_validation_cls_dict
+            confusion_data_dict['cumulative_dataset_paths']=strategy.cumulative_dataset_paths
+            confusion_data_dict['rehearsal_indicies_picked']=strategy.rehearsal_indicies_picked # corresponding to cumulative datset paths
+            confusion_data_dict['mir_losses_dict'] = strategy.mir_losses_dict
+            confusion_data_dict['memory_dataset_paths']= strategy.memory_dataset_paths
+            confusion_data_dict['memory_dataset_targets'] = strategy.memory_dataset_targets
+            for pl, tl in zip(strategy.all_preds, strategy.all_targets):
+
+                cmt[int(tl), int(pl)] = cmt[int(tl), int(pl)] + 1
+            cmt = cmt.astype('float') / cmt.sum(axis=1)[:, np.newaxis]
+
+            classes = []
+            class_labels = []
+      
+            for i in range(strategy.model.num_classes):
+                cls_acc = -1
+
+
+                if isinstance(class_acc_dict[i].result(), dict) and (0 in class_acc_dict[i].result().keys()):
+                    if isinstance(class_acc_dict[i].result()[0], dict):
+                        if 0 in class_acc_dict[i].result()[0].keys():
+                            cls_acc = class_acc_dict[i].result()[0][0]
+                    elif  isinstance(class_acc_dict[i].result()[0], float):
+                        cls_acc = class_acc_dict[i].result()[0]
+
+                if i in strategy.label_dict.keys():
+                    classes.append(strategy.label_dict[i])
+                else:
+                    classes.append('Unknown Class')
+                cls_label = strategy.label_dict[i] +''+' cls_acc: '+str(np.round(cls_acc*100,2)) +'\ntrain/cumu/test/validation: '
+                cls_label = cls_label +str(self.get_int_value(strategy.unique_train_cls_dict[i]))+'/'+str(self.get_int_value(strategy.total_train_cls_dict[i]))+'/'+str(self.get_int_value(strategy.total_test_cls_dict[i]))+'/'+str(self.get_int_value(strategy.total_validation_cls_dict[i]))
+                class_labels.append(cls_label)
+
+
+
+            plt.imshow(cmt, interpolation='nearest', cmap=plt.cm.Blues)
+            plt.title(title, fontsize=21)
+            plt.colorbar()
+            plt.xticks(tick_marks, classes, rotation=90,  fontsize=18)
+            plt.yticks(tick_marks, class_labels, fontsize=18)
+
+            plt.ylabel('True label', fontsize=18)
+            plt.xlabel('Predicted label', fontsize=18)
+            print(strategy.log_dir)
+            plt.tight_layout()
+            datetime_stap = str(datetime.datetime.now()).replace(' ','_').replace(':','-').replace('.','-')[:-4]
+            plt.savefig(str(strategy.log_dir)+datetime_stap+file_name+'_confusion.png', facecolor='white')
+            plt.close()
+
+            with open(str(strategy.log_dir)+datetime_stap+file_name+'_confusion_data_dict.pkl', 'wb') as handle:
+                pkl.dump(confusion_data_dict, handle, protocol=pkl.HIGHEST_PROTOCOL)
+    
+    def get_int_value(self, value):
+        if torch.is_tensor(value):
+            return value.item()
+        return int(value)
+
+
+
     def before_eval_iteration(self, strategy: 'BaseStrategy', **kwargs):
         self._update_metrics(strategy, 'before_eval_iteration')
 
@@ -280,7 +400,6 @@ class EvaluationPlugin(StrategyPlugin):
     def after_eval_iteration(self, strategy: 'BaseStrategy', **kwargs):
         self._update_metrics(strategy, 'after_eval_iteration')
 
-
 default_logger = EvaluationPlugin(
     accuracy_metrics(minibatch=False, epoch=True, experience=True, stream=True),
     loss_metrics(minibatch=False, epoch=True, experience=True, stream=True),

avalanche/avalanche/training/strategies/ar1.py

@@ -15,7 +15,7 @@ from avalanche.training.plugins import StrategyPlugin, EvaluationPlugin, \
 from avalanche.training.strategies import BaseStrategy
 from avalanche.training.utils import replace_bn_with_brn, get_last_fc_layer, \
     freeze_up_to, change_brn_pars, examples_per_class, LayerAndParameter
-from avalanche.training.plugins.evaluation import default_logger
+from avalanche.training.plugins.evaluation_plugin import default_logger
 
 
 class AR1(BaseStrategy):

avalanche/avalanche/training/strategies/base_strategy.py

@@ -22,7 +22,7 @@ from avalanche.benchmarks.utils.data_loader import TaskBalancedDataLoader, _seq_
 from avalanche.models import DynamicModule
 from avalanche.models.dynamic_optimizers import reset_optimizer
 from avalanche.models.utils import avalanche_forward
-from avalanche.training.plugins.evaluation import default_logger
+from avalanche.training.plugins.evaluation_plugin import default_logger
 from typing import TYPE_CHECKING
 
 from avalanche.training.plugins import EvaluationPlugin
@@ -131,7 +131,6 @@ class BaseStrategy:
         if evaluator is None:
             evaluator = EvaluationPlugin()
         self.plugins.append(evaluator)
-
         self.evaluator = evaluator
         """ EvaluationPlugin used for logging and metric computations. """
 
@@ -365,7 +364,6 @@ class BaseStrategy:
             # in the first case we are outside epoch loop
             # in the second case we are within epoch loop
 
-            #print('starting eval() from here')
             for exp in eval_streams:
                 self.eval(exp)
 
@@ -676,7 +674,6 @@ class BaseStrategy:
 
     def after_eval(self, **kwargs):
         for p in self.plugins:
-
             p.after_eval(self, **kwargs)
 
     def before_eval_iteration(self, **kwargs):

avalanche/avalanche/training/strategies/cumulative.py

@@ -5,7 +5,7 @@ from torch.optim import Optimizer
 from torch.utils.data import ConcatDataset
 
 from avalanche.benchmarks.utils import AvalancheConcatDataset
-from avalanche.training.plugins.evaluation import default_logger
+from avalanche.training.plugins.evaluation_plugin import default_logger
 from avalanche.training.plugins import StrategyPlugin, EvaluationPlugin
 from avalanche.training.strategies import BaseStrategy
 

avalanche/avalanche/training/strategies/deep_slda.py

@@ -6,7 +6,7 @@ import torch
 
 from avalanche.training.plugins import StrategyPlugin
 from avalanche.training.strategies import BaseStrategy
-from avalanche.training.plugins.evaluation import default_logger
+from avalanche.training.plugins.evaluation_plugin import default_logger
 from avalanche.models.dynamic_modules import MultiTaskModule
 from avalanche.models import FeatureExtractorBackbone
 

avalanche/avalanche/training/strategies/joint_training.py

@@ -17,7 +17,7 @@ from torch.utils.data import ConcatDataset
 
 from avalanche.benchmarks.scenarios import Experience
 from avalanche.benchmarks.utils import AvalancheConcatDataset
-from avalanche.training.plugins.evaluation import default_logger
+from avalanche.training.plugins.evaluation_plugin import default_logger
 from avalanche.training.strategies import BaseStrategy
 
 if TYPE_CHECKING:

avalanche/avalanche/training/strategies/strategy_wrappers.py

@@ -14,7 +14,7 @@ from torch.nn import Module, CrossEntropyLoss
 from torch.optim import Optimizer, SGD
 
 from avalanche.models.pnn import PNN
-from avalanche.training.plugins.evaluation import default_logger
+from avalanche.training.plugins.evaluation_plugin import default_logger
 from avalanche.training.plugins import StrategyPlugin, CWRStarPlugin, \
     ReplayPlugin, GDumbPlugin, LwFPlugin, AGEMPlugin, GEMPlugin, EWCPlugin, \
     EvaluationPlugin, SynapticIntelligencePlugin, CoPEPlugin

scripts/jupyter_notbooks/introduction_to_avalanche.ipynb

@@ -0,0 +1,958 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/boehlke/programs/anaconda3/envs/ava-test5/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import pickle as pkl\n",
+    "import numpy as np\n",
+    "import random\n",
+    "import torch\n",
+    "import torchvision\n",
+    "import avalanche as ava"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Introduction to Avalance\n",
+    "This Notebook provides some example code which should act as a start off point to understand the inner workings of the avalnche framework and extentsions made. The way in which thecode in this notebook is written should make it easy to follow where the different Classes are impoted from and thereby understand the code and its application. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Checking if GPU is recognized: "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "device(type='cuda', index=0)"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "use_cuda = torch.cuda.is_available()\n",
+    "detected_device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
+    "detected_device"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Importing Data:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "images_root = '/home/AMMOD_data/camera_traps/BayerWald/G-Fallen/MDcrops/'\n",
+    "data_dir_path = os.path.join(os.getcwd(), '../../data/')\n",
+    "train_stream_file = data_dir_path+'data_stream_files/BW_stream_files/cv0_expsize128_crop_train_stream.pkl'\n",
+    "test_stream_file = data_dir_path+'data_stream_files/BW_stream_files/cv0_expsize128_crop_test_stream.pkl'\n",
+    "label_dict_file = data_dir_path+'label_dictionaries/BIRDS_11_Species.pkl'\n",
+    "with open(train_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
+    "    train_stream = pkl.load(handle)\n",
+    "with open(test_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
+    "    test_stream = pkl.load(handle)\n",
+    "with open(label_dict_file, 'rb') as handle:\n",
+    "    label_dict = pkl.load(handle)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Defining Model Parameters and Data Augmentations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "resize = int(224/0.875)\n",
+    "data_transforms = {\n",
+    "    'train': torchvision.transforms.Compose([\n",
+    "        torchvision.transforms.Resize((resize,resize)),\n",
+    "        torchvision.transforms.RandomCrop(size=(224,224)),\n",
+    "        torchvision.transforms.RandomHorizontalFlip(), \n",
+    "        torchvision.transforms.ToTensor(),\n",
+    "        torchvision.transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1), \n",
+    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
+    "    ]),\n",
+    "    'val': torchvision.transforms.Compose([\n",
+    "        torchvision.transforms.Resize((224,224)),\n",
+    "        torchvision.transforms.ToTensor(),\n",
+    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
+    "    ]),\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "model = ava.models.pytorchcv_wrapper.resnet('imagenet', depth=18, pretrained=True)\n",
+    "model.num_classes = 11\n",
+    "num_input_ftrs = model.output.in_features\n",
+    "model.output = torch.nn.Linear(num_input_ftrs , model.num_classes)\n",
+    "criterion = torch.nn.CrossEntropyLoss()\n",
+    "optimizer = torch.optim.AdamW(model.parameters(), eps=1e-4, weight_decay=75e-3)\n",
+    "batchsize = 128"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "###  Instantiating Avalanche's Basic Metrics and Logging Classes \n",
+    "#### (combined in the Evaluation Plugin)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/without_seq_id/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/without_seq_id/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "\n",
+    "# This following avalanche function returns a list of accuracy metrics \n",
+    "# that are evaluated at different invtervals during the training/evaluation loop\n",
+    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
+    "\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Datastream Without Sequence Information\n",
+    "\n",
+    "The data loaded from the data_stream_files folder is camerat trap data, where each piece of data comes with the path, label and sequence identifier. In order to simplify things at the start and demonstrate the usage in cases without sequence ids, we are removing the sequence_id from the stream data. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_stream_without_seq = [[(tupel[0], tupel[1]) for tupel in sublist] for sublist in train_stream]\n",
+    "test_stream_without_seq = [(tupel[0], tupel[1]) for tupel in test_stream]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Defining the Strategy\n",
+    "Next, we are instanciating the Basestrategy class (Naive) or the child of the Basestrategy class (Cumulative) in case we want to cumulatively train the Model. The Basestrategy class defines a training loop with several callback functions at different stages in the training/evaluation cycle that are called by 'Plugins'. At this stage, the plugins list is empty, but the evaluator is basically a plugin, that calls the different callback functions, and checks whether any of the metrics have to be updated at each point in the loop."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
+    "\n",
+    "cumulative_strategy = ava.training.Cumulative(\n",
+    "    model, optimizer, criterion, train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Avalanche Scenario\n",
+    "Next, the so-called scenario instance is created, by which avalanche combines the data and augmentations. The paths_benchmark function is actually called create_generic_benchmark_from_paths (defined in benchmarks/scenarios/generic_benchmark_creation.py). It is renamed in benchmarks/generators/benchmark_generators.py for some reason...\n",
+    "\n",
+    "Theoriginal Avalanche function was modified to take a path_dataset variable, which should be a child class of the Pytorch Dataset class. In this case we use the PathsDataset class defined in benchmarks/utils/datasets_from_filelists.py "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# In many continuous learning applications, a model should learn different 'tasks' (groups of classes).\n",
+    "# In our case this is not relavant, so all pieces of data belong to the same task: 0\n",
+    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
+    "\n",
+    "# generic_scenario = ava.benchmarks.scenarios.generic_benchmark_creation.create_generic_benchmark_from_paths(...\n",
+    "# calls the same funciton as:\n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream_without_seq,\n",
+    "   [test_stream_without_seq],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Here the train function from the strategy is called for each sublist in the train_stream. \n",
+    "# Then the eval function is called on the test stream. This function could also be called on the train stream,\n",
+    "# which would evaluate all metrics on the entire train stream and might take a long time, since the train stream\n",
+    "# is three times as large as the test_stream. \n",
+    "\n",
+    "number_workers = 4\n",
+    "cl_strategy = naive_strategy # or cumulative_strategy\n",
+    "\n",
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Standard Joint Training As A Baseline\n",
+    "When we want to tain a simple model in the standard way, i.e, all data is available to the model at once, the Naive (Basestrategy) instance is used and the training loop is modified slightly.  Here it is important to instantiate the Naive Basestrategy with train_epochs=1 and define the number of epochs used in the loop seperately. Surely, there is a different way to do this too, but this works with the logging the way I want it to. Also, the train_stream is modified to be a list with a single list of data tuples."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "joint_train_stream_without_seq = [item for sublist in train_stream_without_seq for item in sublist ]\n",
+    "random.shuffle(joint_train_stream_without_seq)\n",
+    "\n",
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/joint_training/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/joint_training/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "\n",
+    "# This following avalanche function returns a list of accuracy metrics \n",
+    "# that are evaluated at different invtervals during the training/evaluation loop\n",
+    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
+    "\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "\n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   joint_train_stream_without_seq,\n",
+    "   [test_stream_without_seq],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n",
+    "\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
+    "\n",
+    "nr_of_epochs = 9\n",
+    "number_workers = 4\n",
+    "cl_strategy = naive_strategy # or cumulative_strategy\n",
+    "\n",
+    "for i in range(nr_of_epochs):\n",
+    "    cl_strategy.train(joint_train_stream_without_seq, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Track Sequnece Information in Avalanche\n",
+    "In order to track sequence wise metrics, the information about the sequence affiliations of each image need to be available in the training/testing loop. This was implemented using the SeqPathsDataset class, which is an extension of the Python Dataset class that handles a third field (addditionally to image path and labell), the sequence id. Further, the SeqDataPlugin was added with a modified \\_unpack\\_minibatch callback function that is called during the training/evaluation loop to handle the third seq_id field that the Dataloader of the SeqPathsDataset returns. This plugin also keeps track  of class occurance statistics and saves the targets and predictions during evaluation.\n",
+    "This data is collected here to produce comprehensive visualisations of results in form of confusion matricies later on."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/seq_id_data/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/seq_id_data/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "\n",
+    "# This following avalanche function returns a list of accuracy metrics \n",
+    "# that are evaluated at different invtervals during the training/evaluation loop\n",
+    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
+    "\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
+    "\n",
+    "cumulative_strategy = ava.training.Cumulative(\n",
+    "    model, optimizer, criterion, train_mb_size=48, train_epochs=1, eval_mb_size=96, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
+    "\n",
+    "\n",
+    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream,\n",
+    "   [test_stream],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n",
+    "\n",
+    "\n",
+    "\n",
+    "number_workers = 4\n",
+    "cl_strategy = naive_strategy # or cumulative_strategy\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Comprehensive Metrics and Generic Logger\n",
+    "\n",
+    "In order to get a better understanding of the performance of different continual leaning / rehearsal strategies, several different metrics were implemented. This includes sequence-wise metrics, class-wise metrics, season-based evaluations and several more. See the /evaluation/metrics/accuracy.py implementation comments for more details.\n",
+    "\n",
+    "Each metric inherits from the GenericPluginMetric class defined in /metrics/metric\\_definitions.py which coordinates the update()-/reset()-/result()- function call of the metrics at different stages during the training, depending on whether the metric is calculated over an iteration, experience, epoch or stream. More precisely, the EvaluationPlugin has a funciton called \\_update\\_metrics() which calls the callback function of the GenericPluginMetric which again calls the appropriate update()-/reset()-/result()- function for each metric and collects the results in a list called metric_values at the appropriate stage. Then the evalutaion plugin also calls all loggers and passes the metric_values list as an argument. \n",
+    "\n",
+    "The GenericPluginMetric was written to handle the newly added accuracy metrics. It is unfortunately not as generic as the name makes it out to be. It could definately be improved upon. Also the implementation of the metrics could be improved to avoid for example the looping over sequence ids or computing the same updates more than once for similar metrics like class-wise accuracies for each class and class avg accuracy. The metrics that are evaluated for every epoch, experience or iteration are quite slow. Doing this after each epoch/experience/iteration would mean that the whole training-evaluation process would take a very long time. \n",
+    "\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "file = 'cv0_expsize128_crop'\n",
+    "data_streams_path = os.path.join(os.getcwd(), '../../data/data_stream_files/BW_stream_files/')\n",
+    "with open(data_streams_path+file.replace('384','128')+'_winter_val_stream.pkl', 'rb') as handle:\n",
+    "     validation_stream_winter = pkl.load(handle)\n",
+    "\n",
+    "with open(data_streams_path+file.replace('384','128')+'_winter_test_stream.pkl', 'rb') as handle:\n",
+    "    test_stream_winter = pkl.load(handle)\n",
+    "\n",
+    "with open(data_streams_path+file.replace('_crop','').replace('384','128')+'_exp_season_split_dict.pkl', 'rb') as handle:\n",
+    "    exp_season_split_dict = pkl.load(handle)\n",
+    "    \n",
+    "validation_stream_winter_files = np.array(validation_stream_winter)[:,0]\n",
+    "validation_stream_winter_files = [images_root+s[0:] for s in validation_stream_winter_files ]\n",
+    "test_stream_winter_files = np.array(test_stream_winter)[:,0]\n",
+    "test_stream_winter_files = [images_root+s[0:] for s in test_stream_winter_files ]\n",
+    "\n",
+    "winter_exp = exp_season_split_dict['winter']\n",
+    "summer_exp = exp_season_split_dict['summer']\n",
+    "winter_train_exp = [train_stream[i] for i in winter_exp]\n",
+    "all_winter_train_data = [i for sublist in winter_train_exp for i in sublist]\n",
+    "all_winter_train_files = np.array(all_winter_train_data)[:,0]\n",
+    "all_winter_train_files = [images_root+s[0:] for s in all_winter_train_files ]\n",
+    "all_winter_files = all_winter_train_files +test_stream_winter_files+validation_stream_winter_files\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nc =11 #number of classes\n",
+    "\n",
+    "metrics = [ava.metrics.accuracy.Top1AccTransfer(nr_train_exp=len(train_stream), reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.ClassTop_nAvgAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.ClassTop_nAvgAcc(nr_classes=nc, n=2, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.ClasswiseTop_nAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.Top_nAcc(n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonTop_nAcc(n=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonTop_nAcc(n=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.Top_nAcc(n=2, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeqTop_nAcc(n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeqMaxAcc( reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonSeqTop_nAcc(num=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonSeqTop_nAcc(num=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeqClasswiseTop_nAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeqClassTop_nAvgAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeqClassAvgMaxAcc(nr_classes=nc, reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonSeqClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeasonSeqClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
+    "           ava.metrics.accuracy.SeqAnyAcc(reset_at='stream', emit_at='stream', mode='eval')]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/comprehnesive_metrics/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.GenericCSVLogger(log_folder='./logging_example/comprehnesive_metrics/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
+    "\n",
+    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream,\n",
+    "   [test_stream],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
+    "   common_root = images_root\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  0\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 0 (Task 0) from train stream --\n",
+      "5it [52:28, 629.60s/it]                                                         \n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:05<00:00,  1.17it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1927\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7653\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  1\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 1 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.39it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:11<00:00,  1.15it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1963\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7742\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  2\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 2 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.46it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:09<00:00,  1.16it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1900\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7751\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  3\n",
+      "Current Classes:  [10, 4]\n",
+      "-- Starting training on experience 3 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.63it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:15<00:00,  1.14it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1926\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7650\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = nan\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  4\n",
+      "Current Classes:  [10, 4]\n",
+      "-- Starting training on experience 4 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.34it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:10<00:00,  1.15it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1928\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7087\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  5\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 5 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.66it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:19<00:00,  1.12it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1878\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.6948\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  6\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 6 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.72it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|█████████████████████████████████████████| 358/358 [05:18<00:00,  1.13it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "found validation stream name\n",
+      "-- >> End of eval phase << --\n",
+      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
+      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1818\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
+      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
+      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
+      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
+      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
+      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.6661\n",
+      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
+      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
+      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  7\n",
+      "Current Classes:  [3, 10, 2, 4]\n",
+      "-- Starting training on experience 7 (Task 0) from train stream --\n",
+      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.92it/s]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      " 73%|██████████████████████████████           | 262/358 [04:06<01:26,  1.11it/s]"
+     ]
+    }
+   ],
+   "source": [
+    "cl_strategy = naive_strategy\n",
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ava-test5",
+   "language": "python",
+   "name": "ava-test5"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scripts/jupyter_notbooks/introduction_to_avlanche.ipynb

@@ -1,429 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/boehlke/programs/anaconda3/envs/ava-test5/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
-   "source": [
-    "import os\n",
-    "import pickle as pkl\n",
-    "import numpy as np\n",
-    "import random\n",
-    "import torch\n",
-    "import torchvision\n",
-    "import avalanche as ava"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Introduction to Avalance\n",
-    "This Notebook provides some example code which should act as a start off point to understand the inner workings of the avalnche framework and extentsions made. The way in which thecode in this notebook is written should make it easy to follow where the different Classes are impoted from and thereby understand the code and its application. "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Checking if GPU is recognized: "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "device(type='cuda', index=0)"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "use_cuda = torch.cuda.is_available()\n",
-    "detected_device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
-    "detected_device"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Importing Data:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "images_root = '/home/AMMOD_data/camera_traps/BayerWald/G-Fallen/MDcrops/'\n",
-    "data_dir_path = os.path.join(os.getcwd(), '../data/')\n",
-    "train_stream_file = data_dir_path+'data_stream_files/cv0_expsize128_crop_train_stream.pkl'\n",
-    "test_stream_file = data_dir_path+'data_stream_files/cv0_expsize128_crop_test_stream.pkl'\n",
-    "label_dict_file = data_dir_path+'label_dictionaries/BIRDS_11_Species.pkl'\n",
-    "with open(train_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
-    "    train_stream = pkl.load(handle)\n",
-    "with open(test_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
-    "    test_stream = pkl.load(handle)\n",
-    "with open(label_dict_file, 'rb') as handle:\n",
-    "    label_dict = pkl.load(handle)\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Defining Model Parameters and Data Augmentations"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "resize = int(224/0.875)\n",
-    "data_transforms = {\n",
-    "    'train': torchvision.transforms.Compose([\n",
-    "        torchvision.transforms.Resize((resize,resize)),\n",
-    "        torchvision.transforms.RandomCrop(size=(224,224)),\n",
-    "        torchvision.transforms.RandomHorizontalFlip(), \n",
-    "        torchvision.transforms.ToTensor(),\n",
-    "        torchvision.transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1), \n",
-    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
-    "    ]),\n",
-    "    'val': torchvision.transforms.Compose([\n",
-    "        torchvision.transforms.Resize((224,224)),\n",
-    "        torchvision.transforms.ToTensor(),\n",
-    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
-    "    ]),\n",
-    "    }\n",
-    "\n",
-    "\n",
-    "model = ava.models.pytorchcv_wrapper.resnet('imagenet', depth=18, pretrained=True)\n",
-    "model.num_classes = 11\n",
-    "num_input_ftrs = model.output.in_features\n",
-    "model.output = torch.nn.Linear(num_input_ftrs , model.num_classes)\n",
-    "criterion = torch.nn.CrossEntropyLoss()\n",
-    "optimizer = torch.optim.AdamW(model.parameters(), eps=1e-4, weight_decay=75e-3)\n",
-    "batchsize = 128"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "###  Instantiating Avalanche's Basic Metrics and Logging Classes \n",
-    "#### (combined in the Evaluation Plugin)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/without_seq_id/tb_data\", filename_suffix='test_run')\n",
-    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/without_seq_id/csvlogs')\n",
-    "interactive_logger = ava.logging.InteractiveLogger()\n",
-    "\n",
-    "# This following avalanche function returns a list of accuracy metrics \n",
-    "# that are evaluated at different invtervals during the training/evaluation loop\n",
-    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
-    "\n",
-    "combined_logger = ava.training.EvaluationPlugin(\n",
-    "    metrics,\n",
-    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
-    "    suppress_warnings=True)\n",
-    "\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Datastream Without Sequence Information\n",
-    "\n",
-    "The data loaded from the data_stream_files folder is camerat trap data, where each piece of data comes with the path, label and sequence identifier. In order to simplify things at the start and demonstrate the usage in cases without sequence ids, we are removing the sequence_id from the stream data. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "train_stream_without_seq = [[(tupel[0], tupel[1]) for tupel in sublist] for sublist in train_stream]\n",
-    "test_stream_without_seq = [(tupel[0], tupel[1]) for tupel in test_stream]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Defining the Strategy\n",
-    "Next, we are instanciating the Basestrategy class (Naive) or the child of the Basestrategy class (Cumulative) in case we want to cumulatively train the Model. The Basestrategy class defines a training loop with several callback functions at different stages in the training/evaluation cycle that are called by 'Plugins'. At this stage, the plugins list is empty, but the evaluator is basically a plugin, that calls the different callback functions, and checks whether any of the metrics have to be updated at each point in the loop."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "naive_strategy = ava.training.Naive(\n",
-    "    model, optimizer, criterion,  \n",
-    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
-    "\n",
-    "cumulative_strategy = ava.training.Cumulative(\n",
-    "    model, optimizer, criterion, train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Avalanche Scenario\n",
-    "Next, the so-called scenario instance is created, by which avalanche combines the data and augmentations. The paths_benchmark function is actually called create_generic_benchmark_from_paths (defined in benchmarks/scenarios/generic_benchmark_creation.py). It is renamed in benchmarks/generators/benchmark_generators.py for some reason...\n",
-    "\n",
-    "Theoriginal Avalanche function was modified to take a path_dataset variable, which should be a child class of the Pytorch Dataset class. In this case we use the PathsDataset class defined in benchmarks/utils/datasets_from_filelists.py "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# In many continuous learning applications, a model should learn different 'tasks' (groups of classes).\n",
-    "# In our case this is not relavant, so all pieces of data belong to the same task: 0\n",
-    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
-    "\n",
-    "# generic_scenario = ava.benchmarks.scenarios.generic_benchmark_creation.create_generic_benchmark_from_paths(...\n",
-    "# calls the same funciton as:\n",
-    "scenario = ava.benchmarks.paths_benchmark(\n",
-    "   train_stream_without_seq,\n",
-    "   [test_stream_without_seq],  \n",
-    "   task_labels=task_labels,\n",
-    "   complete_test_set_only=True,\n",
-    "   train_transform=data_transforms['train'],\n",
-    "   eval_transform=data_transforms['val'],\n",
-    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
-    "   common_root = images_root\n",
-    ")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  0\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 0 (Task 0) from train stream --\n",
-      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:16<00:00, 16.33s/it]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|██████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:05<00:00,  1.37it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp000 = 0.5771\n",
-      "-- >> End of eval phase << --\n",
-      "\tTop1_Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  1\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 1 (Task 0) from train stream --\n",
-      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00,  3.01s/it]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      " 74%|█████████████████████████████████████████████████████████████████████▉                        | 67/90 [00:53<00:11,  2.07it/s]"
-     ]
-    }
-   ],
-   "source": [
-    "# Here the train function from the strategy is called for each sublist in the train_stream. \n",
-    "# Then the eval function is called on the test stream. This function could also be called on the train stream,\n",
-    "# which would evaluate all metrics on the entire train stream and might take a long time, since the train stream\n",
-    "# is three times as large as the test_stream. \n",
-    "\n",
-    "number_workers = 4\n",
-    "cl_strategy = naive_strategy # or cumulative_strategy\n",
-    "\n",
-    "for i, experience in enumerate(scenario.train_stream):\n",
-    "    cl_strategy.train(experience, num_workers=number_workers)\n",
-    "    print('Training completed')\n",
-    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
-    "    print('Computing accuracy on the whole test set')\n",
-    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Standard Joint Training As A Baseline\n",
-    "When we want to tain a simple model in the standard way, i.e, all data is available to the model at once, the Naive (Basestrategy) instance is used and the training loop is modified slightly.  Here it is important to instantiate the Naive Basestrategy with train_epochs=1 and define the number of epochs used in the loop seperately. Surely, there is a different way to do this too, but this works with the logging the way I want it to. Also, the train_stream is modified to be a list with a single list of data tuples."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "joint_train_stream_without_seq = [item for sublist in train_stream_without_seq for item in sublist ]\n",
-    "random.shuffle(joint_train_stream_without_seq)\n",
-    "\n",
-    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/joint_training/tb_data\", filename_suffix='test_run')\n",
-    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/joint_training/csvlogs')\n",
-    "interactive_logger = ava.logging.InteractiveLogger()\n",
-    "\n",
-    "# This following avalanche function returns a list of accuracy metrics \n",
-    "# that are evaluated at different invtervals during the training/evaluation loop\n",
-    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
-    "\n",
-    "combined_logger = ava.training.EvaluationPlugin(\n",
-    "    metrics,\n",
-    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
-    "    suppress_warnings=True)\n",
-    "\n",
-    "\n",
-    "scenario = ava.benchmarks.paths_benchmark(\n",
-    "   joint_train_stream_without_seq,\n",
-    "   [test_stream_without_seq],  \n",
-    "   task_labels=task_labels,\n",
-    "   complete_test_set_only=True,\n",
-    "   train_transform=data_transforms['train'],\n",
-    "   eval_transform=data_transforms['val'],\n",
-    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
-    "   common_root = images_root\n",
-    ")\n",
-    "\n",
-    "naive_strategy = ava.training.Naive(\n",
-    "    model, optimizer, criterion,  \n",
-    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
-    "\n",
-    "nr_of_epochs = 9\n",
-    "number_workers = 4\n",
-    "cl_strategy = naive_strategy # or cumulative_strategy\n",
-    "\n",
-    "for i in range(nr_of_epochs):\n",
-    "    cl_strategy.train(joint_train_stream_without_seq, num_workers=number_workers)\n",
-    "    print('Training completed')\n",
-    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
-    "    print('Computing accuracy on the whole test set')\n",
-    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Track Sequnece Information in Avalanche\n",
-    "In order to track sequence wise metrics, the information about the sequence affiliations of each image need to be available in the training/testing loop. This was implemented using the SeqPathsDataset class, which is an extension of the Python Dataset class that handles a third field (addditionally to image path and labell), the sequence id. Further, the SeqDataPlugin was added with a modified \\_unpack\\_minibatch callback function that is called during the training/evaluation loop to handle the third seq_id field that the Dataloader of the SeqPathsDataset returns. This plugin also keeps track  of class occurance statistics and saves the targets and predictions during evaluation.\n",
-    "This data is collected here to produce comprehensive visualisations of results in form of confusion matricies later on."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
-    "naive_strategy = ava.training.Naive(\n",
-    "    model, optimizer, criterion,  \n",
-    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
-    "\n",
-    "cumulative_strategy = ava.training.Cumulative(\n",
-    "    model, optimizer, criterion, train_mb_size=48, train_epochs=1, eval_mb_size=96, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
-    "\n",
-    "\n",
-    "\n",
-    "\n",
-    "scenario = ava.benchmarks.paths_benchmark(\n",
-    "   train_stream,\n",
-    "   [test_stream],  \n",
-    "   task_labels=task_labels,\n",
-    "   complete_test_set_only=True,\n",
-    "   train_transform=data_transforms['train'],\n",
-    "   eval_transform=data_transforms['val'],\n",
-    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
-    "   common_root = images_root\n",
-    ")\n",
-    "\n",
-    "\n",
-    "\n",
-    "number_workers = 4\n",
-    "cl_strategy = naive_strategy # or cumulative_strategy\n",
-    "\n",
-    "for i, experience in enumerate(scenario.train_stream):\n",
-    "    cl_strategy.train(experience, num_workers=number_workers)\n",
-    "    print('Training completed')\n",
-    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
-    "    print('Computing accuracy on the whole test set')\n",
-    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "ava-test5",
-   "language": "python",
-   "name": "ava-test5"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}