introductory jupyter notebook to avalanche and first extensions made for sequence id processing

avalanche/avalanche/__init__.py

@@ -1,4 +1,11 @@
 import os
+from .logging import *
+from .evaluation import *
+from .benchmarks import *
+from .models import *
+from .training import *
+
+
 __version__ = "0.0.1"
 
 _dataset_add = None

avalanche/avalanche/benchmarks/__init__.py

@@ -11,3 +11,4 @@ are made available.
 from .scenarios import *
 from .generators import *
 from .classic import *
+from .utils import *

avalanche/avalanche/benchmarks/scenarios/generic_benchmark_creation.py

@@ -465,7 +465,9 @@ def create_generic_benchmark_from_paths(
         train_transform=None, train_target_transform=None,
         eval_transform=None, eval_target_transform=None,
         other_streams_transforms: Dict[str, Tuple[Any, Any]] = None,
-        dataset_type: AvalancheDatasetType = AvalancheDatasetType.UNDEFINED) \
+        dataset_type: AvalancheDatasetType = AvalancheDatasetType.UNDEFINED,
+        path_dataset_class: None,
+        common_root=None ) \
         -> GenericCLScenario:
     """
     Creates a benchmark instance given a sequence of lists of files. A separate
@@ -554,11 +556,20 @@ def create_generic_benchmark_from_paths(
     for stream_name, lists_of_files in input_streams.items():
         stream_datasets = []
         for exp_id, list_of_files in enumerate(lists_of_files):
-            common_root, exp_paths_list = common_paths_root(list_of_files)
-            paths_dataset = PathsDataset(common_root, exp_paths_list)
+            #_, exp_paths_list = common_paths_root(list_of_files)
+            paths_dataset = path_dataset_class(root=common_root,files=list_of_files)
+            #print('paths.dataset has field paths: ')
+            #print(paths_dataset.paths)
+            #print(dir(paths_dataset))
+
             stream_datasets.append(AvalancheDataset(
                 paths_dataset,
-                task_labels=task_labels[exp_id]))
+                task_labels=task_labels[exp_id],
+                paths = paths_dataset.paths))
+            #print('in avalanche dataset wrap')
+            #print(stream_datasets[-1].paths)
+            #print(dir(stream_datasets[-1]._dataset))
+            
 
         stream_definitions[stream_name] = stream_datasets
 
@@ -574,6 +585,8 @@ def create_generic_benchmark_from_paths(
         dataset_type=dataset_type)
 
 
+
+
 def create_generic_benchmark_from_tensor_lists(
         train_tensors: Sequence[Sequence[Any]],
         test_tensors: Sequence[Sequence[Any]],

avalanche/avalanche/benchmarks/utils/avalanche_dataset.py

@@ -21,6 +21,7 @@ import warnings
 from collections import OrderedDict, defaultdict, deque
 from enum import Enum, auto
 
+import numpy as np
 import torch
 from torch.utils.data.dataloader import default_collate
 from torch.utils.data.dataset import Dataset, Subset, ConcatDataset
@@ -101,6 +102,7 @@ class AvalancheDataset(IDatasetWithTargets[T_co, TTargetType], Dataset[T_co]):
                  initial_transform_group: str = None,
                  task_labels: Union[int, Sequence[int]] = None,
                  targets: Sequence[TTargetType] = None,
+                 paths = None,
                  dataset_type: AvalancheDatasetType = None,
                  collate_fn: Callable[[List], Any] = None,
                  targets_adapter: Callable[[Any], TTargetType] = None):
@@ -201,6 +203,10 @@ class AvalancheDataset(IDatasetWithTargets[T_co, TTargetType], Dataset[T_co]):
         The original dataset.
         """
 
+
+        self.paths = paths
+
+
         self.dataset_type = dataset_type
         """
         The type of this dataset (UNDEFINED, CLASSIFICATION, ...).
@@ -319,6 +325,7 @@ class AvalancheDataset(IDatasetWithTargets[T_co, TTargetType], Dataset[T_co]):
     def __len__(self):
         return len(self._dataset)
 
+
     def train(self):
         """
         Returns a new dataset with the transformations of the 'train' group
@@ -1048,6 +1055,7 @@ class AvalancheSubset(AvalancheDataset[T_co, TTargetType]):
                  initial_transform_group: str = None,
                  task_labels: Union[int, Sequence[int]] = None,
                  targets: Sequence[TTargetType] = None,
+                 paths: Sequence[TTargetType] = None,
                  dataset_type: AvalancheDatasetType = None,
                  collate_fn: Callable[[List], Any] = None,
                  targets_adapter: Callable[[Any], TTargetType] = None):
@@ -1171,6 +1179,14 @@ class AvalancheSubset(AvalancheDataset[T_co, TTargetType]):
         # initialization procedures
         self._class_mapping = class_mapping
         self._indices = indices
+        self.paths = paths
+        if paths is not None:
+            self.paths = np.array(paths)
+            indices = np.array(indices)
+
+            self.paths = self.paths[indices]
+            self.paths = self.paths.tolist()
+      
 
         if initial_transform_group is None:
             if isinstance(dataset, AvalancheDataset):
@@ -1185,6 +1201,7 @@ class AvalancheSubset(AvalancheDataset[T_co, TTargetType]):
                          initial_transform_group=initial_transform_group,
                          task_labels=task_labels,
                          targets=targets,
+                         paths=self.paths,
                          dataset_type=dataset_type,
                          collate_fn=collate_fn,
                          targets_adapter=targets_adapter)
@@ -1517,6 +1534,8 @@ class AvalancheConcatDataset(AvalancheDataset[T_co, TTargetType]):
                                     Sequence[Sequence[int]]] = None,
                  targets: Union[Sequence[TTargetType],
                                 Sequence[Sequence[TTargetType]]] = None,
+                 paths: Union[Sequence[TTargetType],
+                                Sequence[Sequence[TTargetType]]] = None,
                  dataset_type: AvalancheDatasetType = None,
                  collate_fn: Callable[[List], Any] = None,
                  targets_adapter: Callable[[Any], TTargetType] = None):
@@ -1618,6 +1637,14 @@ class AvalancheConcatDataset(AvalancheDataset[T_co, TTargetType]):
         if targets is not None:
             targets = self._concat_targets(targets)
 
+
+        self.paths = None
+        if paths is not None:
+            paths = self.flatten_list(paths)
+            self.paths =paths
+
+
+
         self._adapt_concat_datasets()
 
         super().__init__(ClassificationDataset(),  # not used
@@ -1627,10 +1654,21 @@ class AvalancheConcatDataset(AvalancheDataset[T_co, TTargetType]):
                          initial_transform_group=initial_transform_group,
                          task_labels=task_labels,
                          targets=targets,
+                         paths=self.paths,
                          dataset_type=dataset_type,
                          collate_fn=collate_fn,
                          targets_adapter=targets_adapter)
 
+    def flatten_list(self, list_w_sublists):
+        flattened = []
+        for item in list_w_sublists:
+            if  isinstance(item, list):
+                for val in item:
+                    flattened.append(val)
+            else:
+                flattened.append(item)
+        return flattened
+
     def _get_dataset_type_collate_and_adapter(
             self, datasets, dataset_type, collate_fn, targets_adapter):
 
@@ -1944,6 +1982,17 @@ class AvalancheConcatDataset(AvalancheDataset[T_co, TTargetType]):
         result._frozen_transforms = f_groups
         return result
 
+def flattened(list_w_sublists):
+    flattened = []
+    for item in list_w_sublists:
+        if  isinstance(item, list):
+            for val in item:
+                flattened.append(val)
+        else:
+            flattened.append(item)
+    return flattened
+
+
 
 def concat_datasets_sequentially(
         train_dataset_list: Sequence[ISupportedClassificationDataset],
@@ -2256,3 +2305,4 @@ __all__ = [
     'as_undefined_dataset',
     'train_eval_avalanche_datasets'
 ]
+

avalanche/avalanche/benchmarks/utils/data_loader.py

@@ -39,6 +39,36 @@ def _default_collate_mbatches_fn(mbatches):
         batch.append(t)
     return batch
 
+def _seq_collate_mbatches_fn(mbatches):
+    """ Combines multiple mini-batches together.
+
+    Concatenates each tensor in the mini-batches along dimension 0 (usually this
+    is the batch size).
+
+    :param mbatches: sequence of mini-batches.
+    :return: a single mini-batch
+    """
+    paths = []
+    seq_codes = []
+
+    for i in range(len(mbatches)):
+        paths.append(mbatches[i][2])
+    for i in range(len(mbatches)):
+        seq_codes.append(mbatches[i][3])
+
+    batch = []  
+    for i in [0, 1,4]: #mbatches[0])):
+        test_list = []
+        for el in mbatches:
+            test_list.append(el[i])
+        if not torch.is_tensor(test_list[0]):
+            #print('ever in this case=')
+            test_list = [torch.tensor(test_list)]
+        t = torch.cat(test_list, dim=0)
+        batch.append(t)
+
+    return (batch, paths, seq_codes)
+
 
 class TaskBalancedDataLoader:
     def __init__(self, data: AvalancheDataset,

avalanche/avalanche/benchmarks/utils/datasets_from_filelists.py

@@ -86,6 +86,7 @@ class PathsDataset(data.Dataset):
         self.root: Optional[Path] = root
         self.imgs = files
         self.targets = [img_data[1] for img_data in self.imgs]
+        self.paths =  [img_data[0] for img_data in self.imgs]
         self.transform = transform
         self.target_transform = target_transform
         self.loader = loader
@@ -107,7 +108,16 @@ class PathsDataset(data.Dataset):
 
         if self.root is not None:
             impath = self.root / impath
-        img = self.loader(impath)
+        success = False
+        while not success:
+            try:
+                img = Image.open(impath).convert('RGB')
+                success = True
+            except:
+                print('image could not be loaded!')
+                print(impath)
+                continue
+                
 
         # If a bounding box is provided, crop the image before passing it to
         # any user-defined transformation.
@@ -133,6 +143,88 @@ class PathsDataset(data.Dataset):
         return len(self.imgs)
 
 
+
+class SeqPathsDataset(data.Dataset):
+    """
+    This class extends the basic Pytorch Dataset class to handle list of (path, target, seq_id) tupel
+    as the main data source.
+    """
+
+    def __init__(
+            self, root, files, transform=None, target_transform=None,
+            loader=default_image_loader):
+        """
+        Creates a File Dataset from a list of files and labels.
+
+        :param root: root path where the data to load are stored. May be None.
+        :param files: list of tuples. Each tuple must contain two elements: the
+            full path to the pattern and its class label. Optionally, the tuple
+            may contain a third element describing the bounding box to use for
+            cropping (top, left, height, width).
+        :param transform: eventual transformation to add to the input data (x)
+        :param target_transform: eventual transformation to add to the targets
+            (y)
+        :param loader: loader function to use (for the real data) given path.
+        """
+
+        if root is not None:
+            root = Path(root)
+        self.root= root
+        self.imgs = files
+        self.targets = [img_data[1] for img_data in self.imgs]
+        self.paths =  [img_data[0] for img_data in self.imgs]
+        self.transform = transform
+        self.target_transform = target_transform
+        self.loader = loader
+
+    def __getitem__(self, index):
+        """
+        Returns next element in the dataset given the current index.
+
+        :param index: index of the data to get.
+        :return: loaded item.
+        """
+ 
+        img_description = self.imgs[index]
+
+        impath = img_description[0]
+        target = img_description[1]
+        seq_code = img_description[2]
+
+        if self.root is not None:
+
+            impath = self.root / impath
+
+        success = False
+        while not success:
+            try:
+                img = Image.open(impath).convert('RGB')
+                success = True
+            except:
+                print('image could not be loaded!')
+                print(impath)
+                continue
+                
+
+        if self.transform is not None:
+            img = self.transform(img)
+        if self.target_transform is not None:
+            target = self.target_transform(target)
+     
+        return img, target, str(impath), seq_code
+
+    def __len__(self):
+        """
+        Returns the total number of elements in the dataset.
+
+        :return: Total number of dataset items.
+        """
+
+        return len(self.imgs)
+
+
+
+
 class FilelistDataset(PathsDataset):
     """
     This class extends the basic Pytorch Dataset class to handle filelists as
@@ -432,6 +524,7 @@ __all__ = [
     'default_image_loader',
     'default_flist_reader',
     'PathsDataset',
+    'SeqPathsDataset',
     'FilelistDataset',
     'datasets_from_filelists',
     'datasets_from_paths',

avalanche/avalanche/models/__init__.py

@@ -9,6 +9,7 @@ the near future.
 
 from .simple_cnn import *
 from .simple_mlp import *
+from .pytorchcv_wrapper import *
 from .mlp_tiny_imagenet import SimpleMLP_TinyImageNet
 from .mobilenetv1 import MobilenetV1
 from .dynamic_modules import *

avalanche/avalanche/training/__init__.py

@@ -6,3 +6,4 @@ CL strategies. These are provided either as standalone strategies in
 can be easily combined with your own strategy.
 """
 from .strategies import *
+from .plugins import *

avalanche/avalanche/training/plugins/__init__.py

@@ -10,3 +10,4 @@ from .replay import ReplayPlugin, StoragePolicy, ClassBalancedStoragePolicy, \
 from .strategy_plugin import StrategyPlugin
 from .synaptic_intelligence import SynapticIntelligencePlugin
 from .cope import CoPEPlugin, PPPloss
+from .sequence_data import SeqDataPlugin

avalanche/avalanche/training/plugins/sequence_data.py

@@ -0,0 +1,128 @@
+import random
+from abc import ABC, abstractmethod
+from typing import Dict, List, Optional, TYPE_CHECKING
+
+import torch
+from numpy import inf
+import numpy as np
+from torch import Tensor, cat
+from torch.nn import Module
+from torch.utils.data import random_split, DataLoader
+
+from avalanche.benchmarks.utils import AvalancheConcatDataset, \
+    AvalancheDataset, AvalancheSubset
+from avalanche.benchmarks.utils.data_loader import \
+    ReplayDataLoader
+from avalanche.models import FeatureExtractorBackbone
+from avalanche.training.plugins.strategy_plugin import StrategyPlugin
+
+if TYPE_CHECKING:
+    from avalanche.training.strategies import BaseStrategy
+
+
+from avalanche.benchmarks.utils.data_loader import TaskBalancedDataLoader, _seq_collate_mbatches_fn
+import matplotlib.pyplot as plt
+
+import pickle as pkl
+
+
+class SeqDataPlugin(StrategyPlugin):
+    """Plugin that handles SeqPathDataset where sequence information is relevant. 
+    the class ant seq wise accuracies are logged using the funcitons implemented in this function. 
+    """
+
+    def __init__(self):
+        super().__init__()
+        
+
+    def before_train_dataset_adaptation(self, strategy: "BaseStrategy"):
+        # counting unique class occurances in train stream to produce help evaluate the results 
+        unique, counts = torch.unique(torch.as_tensor(strategy.experience.dataset.targets), return_counts=True)
+
+        for i, cls_i in enumerate(unique):
+            strategy.unique_train_cls_dict[cls_i.item()] += counts[i]
+
+    def before_eval_dataset_adaptation(self, strategy: "BaseStrategy"):
+        # counting class occurances in validation and test data to help evaluate the results 
+        if strategy.last_eval:
+
+            if strategy.experience.origin_stream.name =='test':
+                unique, counts = torch.unique(torch.as_tensor(strategy.experience.dataset.targets), return_counts=True)
+                if strategy.current_train_exp_seen == 0: 
+                    for i, cls_i in enumerate(unique):
+                        strategy.total_test_cls_dict[cls_i.item()] +=counts[i]
+
+            if strategy.experience.origin_stream.name =='validation':
+                unique, counts = torch.unique(torch.as_tensor(strategy.experience.dataset.targets), return_counts=True)
+                if strategy.current_train_exp_seen == 0: 
+                    for i, cls_i in enumerate(unique):
+                        strategy.total_validation_cls_dict[cls_i.item()] +=counts[i]
+
+    def after_train_dataset_adaptation(self, strategy: "BaseStrategy"):
+        # counting class occurances in train data after the rehearsal set has been selected to help evaluate the results. 
+        unique, counts =torch.unique(torch.as_tensor(strategy.adapted_dataset.targets), return_counts=True)
+
+        for i, cls_i in enumerate(unique):
+            strategy.total_train_cls_dict[cls_i.item()] +=counts[i]
+
+
+    def make_train_dataloader(self, strategy: "BaseStrategy", num_workers=0, shuffle=True,
+                              pin_memory=False, **kwargs):
+        kwargs['oversample_small_groups'] =True
+        strategy.dataloader = TaskBalancedDataLoader(
+                strategy.adapted_dataset,
+                collate_mbatches=_seq_collate_mbatches_fn,
+                num_workers=num_workers,
+                batch_size=strategy.train_mb_size,
+                shuffle=shuffle,
+                pin_memory=pin_memory, **kwargs)
+
+    def before_eval(self, strategy: "BaseStrategy"):
+
+        if strategy.last_eval:
+            strategy.all_preds = torch.tensor([], dtype=int, device=strategy.device)
+            strategy.all_targets = torch.tensor([], dtype=int, device=strategy.device)
+
+
+
+    def make_eval_dataloader(self, strategy: "BaseStrategy", num_workers=0, shuffle=True,
+                              pin_memory=False, **kwargs):
+        kwargs['oversample_small_groups'] =True
+        strategy.dataloader = TaskBalancedDataLoader(
+                strategy.adapted_dataset,
+                collate_mbatches=_seq_collate_mbatches_fn,
+                num_workers=num_workers,
+                batch_size=strategy.train_mb_size,
+                shuffle=shuffle,
+                pin_memory=pin_memory, **kwargs)
+
+    def _unpack_minibatch(self, strategy: "BaseStrategy",  **kwargs):
+
+        # Callback function to handle the third seq_id field that the Dataloader of the SeqPathsDataset returns
+
+        strategy.paths = strategy.mbatch[1][0]
+        strategy.seq_codes = torch.tensor(strategy.mbatch[2][0]).to(strategy.device)
+        strategy.mbatch = strategy.mbatch[0]
+
+        assert len(strategy.mbatch) >= 3
+        for i in range(len(strategy.mbatch)):
+
+            strategy.mbatch[i] = strategy.mbatch[i].to(strategy.device)
+
+
+
+    def after_forward(self, strategy:"BaseStrategy", **kwargs):
+        _, strategy.preds = torch.max(strategy.mb_output, dim=1)
+
+
+    def after_eval_forward(self, strategy:"BaseStrategy", **kwargs):
+        # keeps track of predictions and targets during evaluation phase to produce comprehensive confusion matricies after evaluation.
+        _, strategy.preds = torch.max(strategy.mb_output, dim=1)
+
+        if strategy.last_eval:
+            strategy.all_targets = torch.cat([strategy.all_targets, strategy.mb_y])
+            strategy.all_preds = torch.cat([strategy.all_preds, strategy.preds])
+
+    
+
+

avalanche/avalanche/training/plugins/strategy_plugin.py

@@ -27,6 +27,11 @@ class StrategyPlugin(StrategyCallbacks[Any]):
                                         **kwargs):
         pass
 
+
+    def train_dataset_adaptation(self, strategy: 'BaseStrategy',
+                                        **kwargs):
+        pass
+
     def after_train_dataset_adaptation(self, strategy: 'BaseStrategy',
                                        **kwargs):
         pass

avalanche/avalanche/training/strategies/base_strategy.py

@@ -18,7 +18,7 @@ from torch.nn import Module, CrossEntropyLoss
 from torch.optim import Optimizer
 
 from avalanche.benchmarks.scenarios import Experience
-from avalanche.benchmarks.utils.data_loader import TaskBalancedDataLoader
+from avalanche.benchmarks.utils.data_loader import TaskBalancedDataLoader, _seq_collate_mbatches_fn
 from avalanche.models import DynamicModule
 from avalanche.models.dynamic_optimizers import reset_optimizer
 from avalanche.models.utils import avalanche_forward
@@ -30,7 +30,7 @@ from avalanche.training.plugins import EvaluationPlugin
 if TYPE_CHECKING:
     from avalanche.core import StrategyCallbacks
     from avalanche.training.plugins import StrategyPlugin
-
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +43,7 @@ class BaseStrategy:
                  train_mb_size: int = 1, train_epochs: int = 1,
                  eval_mb_size: int = 1, device='cpu',
                  plugins: Optional[Sequence['StrategyPlugin']] = None,
-                 evaluator=default_logger, eval_every=-1):
+                 evaluator=default_logger, eval_every=-1, label_dict=None):
         """
         BaseStrategy is the super class of all task-based continual learning
         strategies. It implements a basic training loop and callback system
@@ -131,15 +131,29 @@ class BaseStrategy:
         if evaluator is None:
             evaluator = EvaluationPlugin()
         self.plugins.append(evaluator)
+
         self.evaluator = evaluator
         """ EvaluationPlugin used for logging and metric computations. """
 
+        self.log_dir = ''
+        for p in self.evaluator.loggers:
+            if getattr(p,'tb_log_dir', None):
+                self.log_dir = p.tb_log_dir
+            if getattr(p, 'log_folder', None):
+                self.log_dir =p.log_folder
+        self.log_dir = str(Path(self.log_dir).parent)+'/'
+
         self.eval_every = eval_every
         """ Frequency of the evaluation during training. """
 
+ 
+
         ###################################################################
         # State variables. These are updated during the train/eval loops. #
         ###################################################################
+
+       
+
         self.training_exp_counter = 0
         """ Counts the number of training steps. +1 at the end of each 
         experience. """
@@ -186,6 +200,27 @@ class BaseStrategy:
         self._warn_for_disabled_plugins_callbacks()
         self._warn_for_disabled_metrics_callbacks()
 
+        self.cls_acc_dict = None
+        #class-acc_dict where current data accuracies are stored class wise
+
+        self.unique_train_cls_dict = {}
+        self.total_train_cls_dict = {}
+        self.total_test_cls_dict = {}
+        self.total_validation_cls_dict = {}
+        for i in range(self.model.num_classes):
+
+            self.unique_train_cls_dict[i] = 0
+            self.total_train_cls_dict[i] = 0
+            self.total_test_cls_dict[i] = 0
+            self.total_validation_cls_dict[i] = 0
+
+        # variables to track class wise data occurances
+
+
+        self.label_dict = label_dict 
+        #""" Dictionary with int-labels as keys and class names as values """
+
+
     @property
     def is_eval(self):
         """ True if the strategy is in evaluation mode. """
@@ -228,6 +263,12 @@ class BaseStrategy:
         :return: dictionary containing last recorded value for
             each metric name.
         """
+    
+        self.num_workers = 0
+        if 'num_workers' in kwargs:
+            self.num_workers = kwargs['num_workers']
+            del kwargs['num_workers']
+
         self.is_training = True
         self._stop_training = False
 
@@ -242,9 +283,11 @@ class BaseStrategy:
 
         self.before_training(**kwargs)
 
-        self._periodic_eval(eval_streams, do_final=False, do_initial=True)
+        #self._periodic_eval(eval_streams, do_final=False, do_initial=True)
 
         for self.experience in experiences:
+            print("Start of experience: ", self.experience.current_experience)
+            print("Current Classes: ", self.experience.classes_in_this_experience)
             self.train_exp(self.experience, eval_streams, **kwargs)
         self.after_training(**kwargs)
 
@@ -282,7 +325,7 @@ class BaseStrategy:
 
         self.before_training_exp(**kwargs)
         
-        do_final = True
+        do_final = False
         if self.eval_every > 0 and \
                 (self.train_epochs - 1) % self.eval_every == 0:
             do_final = False
@@ -297,7 +340,7 @@ class BaseStrategy:
 
             self.training_epoch(**kwargs)
             self.after_training_epoch(**kwargs)
-            self._periodic_eval(eval_streams, do_final=False)
+            #self._periodic_eval(eval_streams, do_final=False)
 
         # Final evaluation
         self._periodic_eval(eval_streams, do_final=do_final)
@@ -308,6 +351,7 @@ class BaseStrategy:
         # Since we are switching from train to eval model inside the training
         # loop, we need to save the training state, and restore it after the
         # eval is done.
+
         _prev_state = (
             self.epoch,
             self.experience,
@@ -320,6 +364,8 @@ class BaseStrategy:
                 (self.eval_every > 0 and self.epoch % self.eval_every == 0):
             # in the first case we are outside epoch loop
             # in the second case we are within epoch loop
+
+            #print('starting eval() from here')
             for exp in eval_streams:
                 self.eval(exp)
 
@@ -336,6 +382,8 @@ class BaseStrategy:
     def train_dataset_adaptation(self, **kwargs):
         """ Initialize `self.adapted_dataset`. """
         self.adapted_dataset = self.experience.dataset
+        for p in self.plugins:
+            p.train_dataset_adaptation(self, **kwargs)
         self.adapted_dataset = self.adapted_dataset.train()
 
     @torch.no_grad()
@@ -359,9 +407,21 @@ class BaseStrategy:
             exp_list = [exp_list]
         self.current_eval_stream = exp_list
 
+        
+        if 'last_eval' in kwargs:
+
+            self.last_eval = kwargs['last_eval']
+            del kwargs['last_eval']
+        else: 
+            self.last_eval = False
+        if 'num_workers' in kwargs:
+            self.num_workers = kwargs['num_workers']
+            del kwargs['num_workers']
+
         self.before_eval(**kwargs)
         for self.experience in exp_list:
             # Data Adaptation
+
             self.before_eval_dataset_adaptation(**kwargs)
             self.eval_dataset_adaptation(**kwargs)
             self.after_eval_dataset_adaptation(**kwargs)
@@ -380,6 +440,12 @@ class BaseStrategy:
 
         return res
 
+  
+    def get_int_value(self, value):
+        if torch.is_tensor(value):
+            return value.item()
+        return int(value)
+
     def before_training_exp(self, **kwargs):
         """
         Called  after the dataset and data loader creation and
@@ -397,15 +463,24 @@ class BaseStrategy:
         :param pin_memory: If True, the data loader will copy Tensors into CUDA
             pinned memory before returning them. Defaults to True.
         """
-        self.dataloader = TaskBalancedDataLoader(
-            self.adapted_dataset,
-            oversample_small_groups=True,
-            num_workers=num_workers,
-            batch_size=self.train_mb_size,
-            shuffle=shuffle,
-            pin_memory=pin_memory)
+        plugin_dataloader_found = False
+        for p in self.plugins:
 
-    def make_eval_dataloader(self, num_workers=0, pin_memory=True,
+            make_train_dataloader =  getattr(p, "make_train_dataloader", None)
+            if callable(make_train_dataloader):
+                plugin_dataloader_found=True
+                p.make_train_dataloader(self, **kwargs)
+                
+        if not plugin_dataloader_found: 
+            self.dataloader = TaskBalancedDataLoader(
+                self.adapted_dataset,
+                oversample_small_groups=True,
+                num_workers=self.num_workers,
+                batch_size=self.train_mb_size,
+                shuffle=shuffle,
+                pin_memory=pin_memory)
+
+    def make_eval_dataloader(self, num_workers=0, shuffle=True, pin_memory=True,
                              **kwargs):
         """
         Initializes the eval data loader.
@@ -417,11 +492,20 @@ class BaseStrategy:
         :param kwargs:
         :return:
         """
-        self.dataloader = DataLoader(
-            self.adapted_dataset,
-            num_workers=num_workers,
-            batch_size=self.eval_mb_size,
-            pin_memory=pin_memory)
+
+        plugin_dataloader_found = False
+        for p in self.plugins:
+            make_eval_dataloader =  getattr(p, "make_eval_dataloader", None)
+            if callable(make_eval_dataloader):
+                plugin_dataloader_found=True
+                p.make_eval_dataloader(self, **kwargs)
+
+        if not plugin_dataloader_found: 
+            self.dataloader = DataLoader(
+                self.adapted_dataset,
+                num_workers=self.num_workers,
+                batch_size=self.eval_mb_size,
+                pin_memory=pin_memory)
 
     def after_train_dataset_adaptation(self, **kwargs):
         """
@@ -448,11 +532,15 @@ class BaseStrategy:
         :param kwargs:
         :return:
         """
+   
+
         for self.mb_it, self.mbatch in enumerate(self.dataloader):
+
             if self._stop_training:
                 break
+            self._unpack_minibatch(**kwargs)
+
 
-            self._unpack_minibatch()
             self.before_training_iteration(**kwargs)
 
             self.optimizer.zero_grad()
@@ -477,18 +565,28 @@ class BaseStrategy:
 
             self.after_training_iteration(**kwargs)
 
-    def _unpack_minibatch(self):
+    def _unpack_minibatch(self, **kwargs):
         """ We assume mini-batches have the form <x, y, ..., t>.
         This allows for arbitrary tensors between y and t.
         Keep in mind that in the most general case mb_task_id is a tensor
         which may contain different labels for each sample.
         """
-        assert len(self.mbatch) >= 3
-        for i in range(len(self.mbatch)):
-            self.mbatch[i] = self.mbatch[i].to(self.device)
+
+        plugin_unpack_found = False
+        for p in self.plugins:
+            _unpack_minibatch =  getattr(p, "_unpack_minibatch", None)
+            if callable(_unpack_minibatch):
+                plugin_unpack_found=True
+                p._unpack_minibatch(self, **kwargs)
+
+        if not plugin_unpack_found: 
+            assert len(self.mbatch) >= 3
+            for i in range(len(self.mbatch)):
+                self.mbatch[i] = self.mbatch[i].to(self.device)
 
     def before_training(self, **kwargs):
         for p in self.plugins:
+
             p.before_training(self, **kwargs)
 
     def after_training(self, **kwargs):
@@ -560,11 +658,13 @@ class BaseStrategy:
     def eval_epoch(self, **kwargs):
         for self.mb_it, self.mbatch in \
                 enumerate(self.dataloader):
-            self._unpack_minibatch()
+
+            self._unpack_minibatch(**kwargs)
             self.before_eval_iteration(**kwargs)
 
             self.before_eval_forward(**kwargs)
             self.mb_output = self.forward()
+
             self.after_eval_forward(**kwargs)
             self.loss = self.criterion()
 
@@ -576,6 +676,7 @@ class BaseStrategy:
 
     def after_eval(self, **kwargs):
         for p in self.plugins:
+
             p.after_eval(self, **kwargs)
 
     def before_eval_iteration(self, **kwargs):
@@ -645,3 +746,4 @@ class BaseStrategy:
 
 
 __all__ = ['BaseStrategy']
+

scripts/jupyter_notbooks/introduction_to_avlanche.ipynb

@@ -0,0 +1,429 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/boehlke/programs/anaconda3/envs/ava-test5/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import pickle as pkl\n",
+    "import numpy as np\n",
+    "import random\n",
+    "import torch\n",
+    "import torchvision\n",
+    "import avalanche as ava"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Introduction to Avalance\n",
+    "This Notebook provides some example code which should act as a start off point to understand the inner workings of the avalnche framework and extentsions made. The way in which thecode in this notebook is written should make it easy to follow where the different Classes are impoted from and thereby understand the code and its application. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Checking if GPU is recognized: "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "device(type='cuda', index=0)"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "use_cuda = torch.cuda.is_available()\n",
+    "detected_device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
+    "detected_device"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Importing Data:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "images_root = '/home/AMMOD_data/camera_traps/BayerWald/G-Fallen/MDcrops/'\n",
+    "data_dir_path = os.path.join(os.getcwd(), '../data/')\n",
+    "train_stream_file = data_dir_path+'data_stream_files/cv0_expsize128_crop_train_stream.pkl'\n",
+    "test_stream_file = data_dir_path+'data_stream_files/cv0_expsize128_crop_test_stream.pkl'\n",
+    "label_dict_file = data_dir_path+'label_dictionaries/BIRDS_11_Species.pkl'\n",
+    "with open(train_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
+    "    train_stream = pkl.load(handle)\n",
+    "with open(test_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
+    "    test_stream = pkl.load(handle)\n",
+    "with open(label_dict_file, 'rb') as handle:\n",
+    "    label_dict = pkl.load(handle)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Defining Model Parameters and Data Augmentations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "resize = int(224/0.875)\n",
+    "data_transforms = {\n",
+    "    'train': torchvision.transforms.Compose([\n",
+    "        torchvision.transforms.Resize((resize,resize)),\n",
+    "        torchvision.transforms.RandomCrop(size=(224,224)),\n",
+    "        torchvision.transforms.RandomHorizontalFlip(), \n",
+    "        torchvision.transforms.ToTensor(),\n",
+    "        torchvision.transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1), \n",
+    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
+    "    ]),\n",
+    "    'val': torchvision.transforms.Compose([\n",
+    "        torchvision.transforms.Resize((224,224)),\n",
+    "        torchvision.transforms.ToTensor(),\n",
+    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
+    "    ]),\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "model = ava.models.pytorchcv_wrapper.resnet('imagenet', depth=18, pretrained=True)\n",
+    "model.num_classes = 11\n",
+    "num_input_ftrs = model.output.in_features\n",
+    "model.output = torch.nn.Linear(num_input_ftrs , model.num_classes)\n",
+    "criterion = torch.nn.CrossEntropyLoss()\n",
+    "optimizer = torch.optim.AdamW(model.parameters(), eps=1e-4, weight_decay=75e-3)\n",
+    "batchsize = 128"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "###  Instantiating Avalanche's Basic Metrics and Logging Classes \n",
+    "#### (combined in the Evaluation Plugin)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/without_seq_id/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/without_seq_id/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "\n",
+    "# This following avalanche function returns a list of accuracy metrics \n",
+    "# that are evaluated at different invtervals during the training/evaluation loop\n",
+    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
+    "\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Datastream Without Sequence Information\n",
+    "\n",
+    "The data loaded from the data_stream_files folder is camerat trap data, where each piece of data comes with the path, label and sequence identifier. In order to simplify things at the start and demonstrate the usage in cases without sequence ids, we are removing the sequence_id from the stream data. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_stream_without_seq = [[(tupel[0], tupel[1]) for tupel in sublist] for sublist in train_stream]\n",
+    "test_stream_without_seq = [(tupel[0], tupel[1]) for tupel in test_stream]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Defining the Strategy\n",
+    "Next, we are instanciating the Basestrategy class (Naive) or the child of the Basestrategy class (Cumulative) in case we want to cumulatively train the Model. The Basestrategy class defines a training loop with several callback functions at different stages in the training/evaluation cycle that are called by 'Plugins'. At this stage, the plugins list is empty, but the evaluator is basically a plugin, that calls the different callback functions, and checks whether any of the metrics have to be updated at each point in the loop."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
+    "\n",
+    "cumulative_strategy = ava.training.Cumulative(\n",
+    "    model, optimizer, criterion, train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Avalanche Scenario\n",
+    "Next, the so-called scenario instance is created, by which avalanche combines the data and augmentations. The paths_benchmark function is actually called create_generic_benchmark_from_paths (defined in benchmarks/scenarios/generic_benchmark_creation.py). It is renamed in benchmarks/generators/benchmark_generators.py for some reason...\n",
+    "\n",
+    "Theoriginal Avalanche function was modified to take a path_dataset variable, which should be a child class of the Pytorch Dataset class. In this case we use the PathsDataset class defined in benchmarks/utils/datasets_from_filelists.py "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# In many continuous learning applications, a model should learn different 'tasks' (groups of classes).\n",
+    "# In our case this is not relavant, so all pieces of data belong to the same task: 0\n",
+    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
+    "\n",
+    "# generic_scenario = ava.benchmarks.scenarios.generic_benchmark_creation.create_generic_benchmark_from_paths(...\n",
+    "# calls the same funciton as:\n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream_without_seq,\n",
+    "   [test_stream_without_seq],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  0\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 0 (Task 0) from train stream --\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:16<00:00, 16.33s/it]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:05<00:00,  1.37it/s]\n",
+      "> Eval on experience 0 (Task 0) from test stream ended.\n",
+      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp000 = 0.5771\n",
+      "-- >> End of eval phase << --\n",
+      "\tTop1_Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
+      "-- >> Start of training phase << --\n",
+      "Start of experience:  1\n",
+      "Current Classes:  [10, 3, 4]\n",
+      "-- Starting training on experience 1 (Task 0) from train stream --\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00,  3.01s/it]\n",
+      "Epoch 0 ended.\n",
+      "-- >> End of training phase << --\n",
+      "Training completed\n",
+      "Computing accuracy on the whole test set\n",
+      "-- >> Start of eval phase << --\n",
+      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
+      " 74%|█████████████████████████████████████████████████████████████████████▉                        | 67/90 [00:53<00:11,  2.07it/s]"
+     ]
+    }
+   ],
+   "source": [
+    "# Here the train function from the strategy is called for each sublist in the train_stream. \n",
+    "# Then the eval function is called on the test stream. This function could also be called on the train stream,\n",
+    "# which would evaluate all metrics on the entire train stream and might take a long time, since the train stream\n",
+    "# is three times as large as the test_stream. \n",
+    "\n",
+    "number_workers = 4\n",
+    "cl_strategy = naive_strategy # or cumulative_strategy\n",
+    "\n",
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Standard Joint Training As A Baseline\n",
+    "When we want to tain a simple model in the standard way, i.e, all data is available to the model at once, the Naive (Basestrategy) instance is used and the training loop is modified slightly.  Here it is important to instantiate the Naive Basestrategy with train_epochs=1 and define the number of epochs used in the loop seperately. Surely, there is a different way to do this too, but this works with the logging the way I want it to. Also, the train_stream is modified to be a list with a single list of data tuples."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "joint_train_stream_without_seq = [item for sublist in train_stream_without_seq for item in sublist ]\n",
+    "random.shuffle(joint_train_stream_without_seq)\n",
+    "\n",
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/joint_training/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/joint_training/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "\n",
+    "# This following avalanche function returns a list of accuracy metrics \n",
+    "# that are evaluated at different invtervals during the training/evaluation loop\n",
+    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
+    "\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "\n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   joint_train_stream_without_seq,\n",
+    "   [test_stream_without_seq],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n",
+    "\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
+    "\n",
+    "nr_of_epochs = 9\n",
+    "number_workers = 4\n",
+    "cl_strategy = naive_strategy # or cumulative_strategy\n",
+    "\n",
+    "for i in range(nr_of_epochs):\n",
+    "    cl_strategy.train(joint_train_stream_without_seq, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Track Sequnece Information in Avalanche\n",
+    "In order to track sequence wise metrics, the information about the sequence affiliations of each image need to be available in the training/testing loop. This was implemented using the SeqPathsDataset class, which is an extension of the Python Dataset class that handles a third field (addditionally to image path and labell), the sequence id. Further, the SeqDataPlugin was added with a modified \\_unpack\\_minibatch callback function that is called during the training/evaluation loop to handle the third seq_id field that the Dataloader of the SeqPathsDataset returns. This plugin also keeps track  of class occurance statistics and saves the targets and predictions during evaluation.\n",
+    "This data is collected here to produce comprehensive visualisations of results in form of confusion matricies later on."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
+    "\n",
+    "cumulative_strategy = ava.training.Cumulative(\n",
+    "    model, optimizer, criterion, train_mb_size=48, train_epochs=1, eval_mb_size=96, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream,\n",
+    "   [test_stream],  \n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n",
+    "\n",
+    "\n",
+    "\n",
+    "number_workers = 4\n",
+    "cl_strategy = naive_strategy # or cumulative_strategy\n",
+    "\n",
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ava-test5",
+   "language": "python",
+   "name": "ava-test5"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}