forgot update to introductory jupyter notebook

scripts/jupyter_notbooks/introduction_to_avalanche.ipynb

@@ -147,7 +147,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -333,7 +333,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -404,14 +404,24 @@
     "\n",
     "The GenericPluginMetric was written to handle the newly added accuracy metrics. It is unfortunately not as generic as the name makes it out to be. It could definately be improved upon. Also the implementation of the metrics could be improved to avoid for example the looping over sequence ids or computing the same updates more than once for similar metrics like class-wise accuracies for each class and class avg accuracy. The metrics that are evaluated for every epoch, experience or iteration are quite slow. Doing this after each epoch/experience/iteration would mean that the whole training-evaluation process would take a very long time. \n",
     "\n",
-    "\n",
-    "\n",
+    "when calling the eval function of the naive strategy (base\\_strategy) i have added the optional boolean variable 'last_eval', which, if true means a confusion matrix and dictionary is pickled that contains all sorts of infomration on the current metrics, predictions, targets and so on. This implemented to allow quick overviews of the performance of sdifferent methods. The data for this confusion matrix is collected at different points in the evaluation/training cycle and saved as variables of the strategy. Currently, it is required that some specific metrics are tracked when last_eval is true so that these variables are set. A work aroud could be easily implemented. Also, a label_dict file has to be passed to the strategy.  The actual confusion plot is created in the after\\_eval() callback funciton of the Evaluation Plugin in evaluation\\_plugin()\n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "label_dict_path = os.path.join(os.getcwd(), '../../data/label_dictionaries/BIRDS_11_Species.pkl')\n",
+    "with open(label_dict_path, 'rb') as handle:\n",
+    "     label_dict = pkl.load(handle)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -442,7 +452,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -472,7 +482,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -487,8 +497,12 @@
     "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
     "naive_strategy = ava.training.Naive(\n",
     "    model, optimizer, criterion,  \n",
-    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
+    "    train_mb_size=32, train_epochs=1, \n",
+    "    eval_mb_size=48, device=detected_device, \n",
+    "    plugins=[seq_plugin], evaluator=combined_logger,\n",
+    "    label_dict=label_dict)\n",
     "\n",
+    "number_workers = 8\n",
     "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
     "scenario = ava.benchmarks.paths_benchmark(\n",
     "   train_stream,\n",
@@ -499,7 +513,9 @@
     "   eval_transform=data_transforms['val'],\n",
     "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
     "   common_root = images_root\n",
-    ")"
+    ")\n",
+    "\n",
+    "last_eval = True"
    ]
   },
   {
@@ -515,472 +531,29 @@
       "Start of experience:  0\n",
       "Current Classes:  [10, 3, 4]\n",
       "-- Starting training on experience 0 (Task 0) from train stream --\n",
-      "5it [52:28, 629.60s/it]                                                         \n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:05<00:00,  1.17it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1927\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7653\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  1\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 1 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.39it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:11<00:00,  1.15it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1963\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7742\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  2\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 2 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.46it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:09<00:00,  1.16it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1900\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7751\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  3\n",
-      "Current Classes:  [10, 4]\n",
-      "-- Starting training on experience 3 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.63it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:15<00:00,  1.14it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1926\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7650\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = nan\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  4\n",
-      "Current Classes:  [10, 4]\n",
-      "-- Starting training on experience 4 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:01<00:00,  2.34it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:10<00:00,  1.15it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1928\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.7087\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  5\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 5 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.66it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:19<00:00,  1.12it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n"
+      "0it [00:00, ?it/s]"
      ]
     },
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1878\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.6948\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  6\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 6 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.72it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:18<00:00,  1.13it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1818\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 1.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5530\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6020\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6288\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5771\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.6661\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0909\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4769\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4890\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  7\n",
-      "Current Classes:  [3, 10, 2, 4]\n",
-      "-- Starting training on experience 7 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.92it/s]\n",
-      "Epoch 0 ended.\n",
-      "-- >> End of training phase << --\n",
-      "Training completed\n",
-      "Computing accuracy on the whole test set\n",
-      "-- >> Start of eval phase << --\n",
-      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      "100%|█████████████████████████████████████████| 358/358 [05:28<00:00,  1.09it/s]\n",
-      "> Eval on experience 0 (Task 0) from test stream ended.\n",
-      "found validation stream name\n",
-      "-- >> End of eval phase << --\n",
-      "\t Top1AccTransfer_Stream/eval_phase/test_stream/Task000 = None\n",
-      "\tClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0935\n",
-      "\tClassTop_2AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.1818\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 0.9792\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0492\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqAnyAcc_Stream/eval_phase/test_stream/Task000 = 0.5699\n",
-      "\tSeqClassAvgMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.0934\n",
-      "\tSeqClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0919\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/0 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/1 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/10 = 0.9935\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/2 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/3 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/4 = 0.0169\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/5 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/6 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/7 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/8 = 0.0000\n",
-      "\tSeqClasswiseTop_1Acc_Stream/eval_phase/test_stream/9 = 0.0000\n",
-      "\tSeqMAXAcc_Stream/eval_phase/test_stream/Task000 = 0.5506\n",
-      "\tSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5512\n",
-      "\tSummerClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0936\n",
-      "\tSummerSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0920\n"
+      "/home/boehlke/AMMOD/avalanche_extension/avalanche/avalanche/training/plugins/sequence_data.py:104: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+      "  strategy.seq_codes = torch.tensor(strategy.mbatch[2][0]).to(strategy.device)\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\tSummerSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6000\n",
-      "\tSummerTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.6206\n",
-      "\tTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.5695\n",
-      "\tTop_2Acc_Stream/eval_phase/test_stream/Task000 = 0.6661\n",
-      "\tWinterClassTop_1AvgAcc_Stream/eval_phase/test_stream/Task000 = 0.0921\n",
-      "\tWinterSeqClassTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.0906\n",
-      "\tWinterSeqTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4754\n",
-      "\tWinterTop_1Acc_Stream/eval_phase/test_stream/Task000 = 0.4824\n",
-      "-- >> Start of training phase << --\n",
-      "Start of experience:  8\n",
-      "Current Classes:  [10, 3, 4]\n",
-      "-- Starting training on experience 8 (Task 0) from train stream --\n",
-      "100%|█████████████████████████████████████████████| 4/4 [00:02<00:00,  1.89it/s]\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:15<00:00,  3.96s/it]\n",
       "Epoch 0 ended.\n",
       "-- >> End of training phase << --\n",
       "Training completed\n",
       "Computing accuracy on the whole test set\n",
       "-- >> Start of eval phase << --\n",
       "-- Starting eval on experience 0 (Task 0) from test stream --\n",
-      " 89%|████████████████████████████████████▋    | 320/358 [04:33<00:30,  1.24it/s]"
+      " 34%|██████████████████████████████▊                                                             | 120/358 [01:56<03:41,  1.08it/s]"
      ]
     }
    ],
@@ -991,8 +564,177 @@
     "    print('Training completed')\n",
     "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
     "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers, last_eval= last_eval)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Rehearsal Learning \n",
+    "\n",
+    "In the examples so far, the strategy for continual learning meant either a naive approach (finetuning with latest new data) or a cumulative approach (finetuning with all data seen so far). A bunch of methods were implemented to use the rehearsal approach to continual learning to ensure the model does not overwrite previously learned knowledge when finetuning with the new incoming data. The idea is to select images from the memory strategically and use them together with the new data to fine tune the model. \n",
+    "\n",
+    "The main functionalities of this approach where implemented in the ClassImbalanceRehersalPlugin class in /avalanche/training/plugins/imbalance_focus_replay.py. Each instance of this class or child class needs to pass the 'buffer\\_data\\_ratio' variable which defines how many images from the memory are selcted for each images in the current experience. The callback function train_dataset_adaptation() is called to create a AvalancheSubset of the memory data by using the indicies for the memory data returned by the \\_get\\_indices() function, which is implemented by the child class. If the memory is not yet large enough, the entire memory is used for rehearsal. After the finetuning is completed, the after\\_training\\_exp() callback function is called to add the current experience to the memory. \n",
+    "\n",
+    "A full list of all the child classes implementing different rehearsal selection methods:\n",
+    "MaximallyInterferedRetrievalRehersalPlugin, \n",
+    "RandomRehersal, \n",
+    "ClassErrorRehersalPlugin, \n",
+    "ClassErrorRehersalTemperaturePlugin, \n",
+    "ClassFrequencyRehearsalPlugin, \n",
+    "WeightedeMovingClassErrorAverageRehersalPlugin,\n",
+    "ClassErrorFrequencyAvgRehearsalPlugin,\n",
+    "FillExpBasedRehearsalPlugin,\n",
+    "FillExpOversampleBasedRehearsalPlugin\n",
+    "\n",
+    "A more detailed description of what they do and how they are implemented can be found in the comments under each class definition in imbalance_focus_replay.py\n",
+    "\n",
+    "To use these rehearsal strategies, the correct plugin just has to be added to the list of plugins for the Naive strategy class.  \n",
+    "In the following the use of the class error based rehearsal method is demonstrated. In order to use this rehearsal strategy, validation data is necessary to estimate the clas error rates. Further it is required to use the ClasswiseTop_nAcc with n=1 as a metric, where the validation data's class accuracies are stored in a stratedy variable. The validation data needs to be used for evaluation at least once, before the plugin aims to select images from the memory. \n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "val_stream_file = data_dir_path+'data_stream_files/BW_stream_files/cv0_expsize128_crop_val_stream.pkl'\n",
+    "label_dict_file = data_dir_path+'label_dictionaries/BIRDS_11_Species.pkl'\n",
+    "with open(val_stream_file, 'rb') as handle: \n",
+    "    val_stream = pkl.load(handle)\n",
+    "# The val_stream is passed to the pahts_benchmark function below, that creats the 'scenario'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/rehearsal_learning/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.GenericCSVLogger(log_folder='./logging_example/rehearsal_learning/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "number_workers = 8\n",
+    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream,\n",
+    "   [test_stream],  \n",
+    "   other_streams_lists_of_files={'validation': [val_stream]},\n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n",
+    "\n",
+    "\n",
+    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
+    "class_error_method = ava.training.plugins.ClassErrorRehersalPlugin(buffer_data_ratio=1)\n",
+    "plugins_list = [seq_plugin, class_error_method]\n",
+    "\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=plugins_list, evaluator=combined_logger)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cl_strategy = naive_strategy\n",
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    cl_strategy.eval(scenario.validation_stream, num_workers=number_workers, last_eval=True)\n",
+    "    print('Computing accuracy on the whole test set')\n",
+    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers, last_eval=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Rehearsal Learning with Limited Memory \n",
+    "\n",
+    "In the real world, there will never be a unlimited memory. Two memory filling strategies were implemented in /avalanche/training/plugins/class_balancing_replay.py. These are the baseline stadnard reservoir sampling used to draw uniformly from a stream and a strategy specifically designed for class imbalace called class balancing reservoir sampling ('Online Continual Learning from Imbalanced Data' by Chrysakis et al.)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/rehearsal_learning_limited_memory/tb_data\", filename_suffix='test_run')\n",
+    "csv_logger = ava.logging.GenericCSVLogger(log_folder='./logging_example/rehearsal_learning_limited_memory/csvlogs')\n",
+    "interactive_logger = ava.logging.InteractiveLogger()\n",
+    "combined_logger = ava.training.EvaluationPlugin(\n",
+    "    metrics,\n",
+    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
+    "    suppress_warnings=True)\n",
+    "\n",
+    "number_workers = 8\n",
+    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
+    "scenario = ava.benchmarks.paths_benchmark(\n",
+    "   train_stream,\n",
+    "   [test_stream],  \n",
+    "   other_streams_lists_of_files={'validation': [val_stream]},\n",
+    "   task_labels=task_labels,\n",
+    "   complete_test_set_only=True,\n",
+    "   train_transform=data_transforms['train'],\n",
+    "   eval_transform=data_transforms['val'],\n",
+    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
+    "   common_root = images_root\n",
+    ")\n",
+    "\n",
+    "\n",
+    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
+    "plugin_cbrs = ava.training.plugins.ClassBalancingReservoirMemoryRehersalPlugin(buffer_data_ratio=1, memory_size=500, nr_classes=11, \n",
+    "                                                          rehearsal_selection_strategy='ce', temperature=0.2)\n",
+    "plugins_list = [seq_plugin, plugin_cbrs]\n",
+    "\n",
+    "naive_strategy = ava.training.Naive(\n",
+    "    model, optimizer, criterion,  \n",
+    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=plugins_list, evaluator=combined_logger)\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cl_strategy = naive_strategy\n",
+    "for i, experience in enumerate(scenario.train_stream):\n",
+    "    cl_strategy.train(experience, num_workers=number_workers)\n",
+    "    print('Training completed')\n",
+    "    cl_strategy.eval(scenario.validation_stream, num_workers=number_workers)\n",
+    "    print('Computing accuracy on the whole test set')\n",
     "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {