{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle as pkl\n",
    "import numpy as np\n",
    "import random\n",
    "import torch\n",
    "import torchvision\n",
    "import avalanche as ava"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introduction to Avalance\n",
    "This Notebook provides some example code which should act as a start off point to understand the inner workings of the avalnche framework and extentsions made. The way in which thecode in this notebook is written should make it easy to follow where the different Classes are impoted from and thereby understand the code and its application. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking if GPU is recognized: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "detected_device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
    "detected_device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "images_root = '/home/AMMOD_data/camera_traps/BayerWald/G-Fallen/MDcrops/'\n",
    "data_dir_path = os.path.join(os.getcwd(), '../../data/')\n",
    "train_stream_file = data_dir_path+'data_stream_files/BW_stream_files/cv0_expsize128_crop_train_stream.pkl'\n",
    "test_stream_file = data_dir_path+'data_stream_files/BW_stream_files/cv0_expsize128_crop_test_stream.pkl'\n",
    "label_dict_file = data_dir_path+'label_dictionaries/BIRDS_11_Species.pkl'\n",
    "with open(train_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
    "    train_stream = pkl.load(handle)\n",
    "with open(test_stream_file, 'rb') as handle: # even for training with stream of exp size384 128 needs to be loaded first to match the winter stream files\n",
    "    test_stream = pkl.load(handle)\n",
    "with open(label_dict_file, 'rb') as handle:\n",
    "    label_dict = pkl.load(handle)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Model Parameters and Data Augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "resize = int(224/0.875)\n",
    "data_transforms = {\n",
    "    'train': torchvision.transforms.Compose([\n",
    "        torchvision.transforms.Resize((resize,resize)),\n",
    "        torchvision.transforms.RandomCrop(size=(224,224)),\n",
    "        torchvision.transforms.RandomHorizontalFlip(), \n",
    "        torchvision.transforms.ToTensor(),\n",
    "        torchvision.transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1), \n",
    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': torchvision.transforms.Compose([\n",
    "        torchvision.transforms.Resize((224,224)),\n",
    "        torchvision.transforms.ToTensor(),\n",
    "        torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    }\n",
    "\n",
    "\n",
    "model = ava.models.pytorchcv_wrapper.resnet('imagenet', depth=18, pretrained=True)\n",
    "model.num_classes = 11\n",
    "num_input_ftrs = model.output.in_features\n",
    "model.output = torch.nn.Linear(num_input_ftrs , model.num_classes)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(model.parameters(), eps=1e-4, weight_decay=75e-3)\n",
    "batchsize = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Instantiating Avalanche's Basic Metrics and Logging Classes \n",
    "#### (combined in the Evaluation Plugin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/without_seq_id/tb_data\", filename_suffix='test_run')\n",
    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/without_seq_id/csvlogs')\n",
    "interactive_logger = ava.logging.InteractiveLogger()\n",
    "\n",
    "# This following avalanche function returns a list of accuracy metrics \n",
    "# that are evaluated at different invtervals during the training/evaluation loop\n",
    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
    "\n",
    "combined_logger = ava.training.EvaluationPlugin(\n",
    "    metrics,\n",
    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
    "    suppress_warnings=True)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datastream Without Sequence Information\n",
    "\n",
    "The data loaded from the data_stream_files folder is camerat trap data, where each piece of data comes with the path, label and sequence identifier. In order to simplify things at the start and demonstrate the usage in cases without sequence ids, we are removing the sequence_id from the stream data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stream_without_seq = [[(tupel[0], tupel[1]) for tupel in sublist] for sublist in train_stream]\n",
    "test_stream_without_seq = [(tupel[0], tupel[1]) for tupel in test_stream]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the Strategy\n",
    "Next, we are instanciating the Basestrategy class (Naive) or the child of the Basestrategy class (Cumulative) in case we want to cumulatively train the Model. The Basestrategy class defines a training loop with several callback functions at different stages in the training/evaluation cycle that are called by 'Plugins'. At this stage, the plugins list is empty, but the evaluator is basically a plugin, that calls the different callback functions, and checks whether any of the metrics have to be updated at each point in the loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_strategy = ava.training.Naive(\n",
    "    model, optimizer, criterion,  \n",
    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
    "\n",
    "cumulative_strategy = ava.training.Cumulative(\n",
    "    model, optimizer, criterion, train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Avalanche Scenario\n",
    "Next, the so-called scenario instance is created, by which avalanche combines the data and augmentations. The paths_benchmark function is actually called create_generic_benchmark_from_paths (defined in benchmarks/scenarios/generic_benchmark_creation.py). It is renamed in benchmarks/generators/benchmark_generators.py for some reason...\n",
    "\n",
    "Theoriginal Avalanche function was modified to take a path_dataset variable, which should be a child class of the Pytorch Dataset class. In this case we use the PathsDataset class defined in benchmarks/utils/datasets_from_filelists.py "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'np' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# In many continuous learning applications, a model should learn different 'tasks' (groups of classes).\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m# In our case this is not relavant, so all pieces of data belong to the same task: 0\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m task_labels \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241m.\u001b[39mzeros((\u001b[38;5;241m1\u001b[39m,\u001b[38;5;28mlen\u001b[39m(train_stream)))\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mint\u001b[39m)\u001b[38;5;241m.\u001b[39mtolist()[\u001b[38;5;241m0\u001b[39m] \n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# generic_scenario = ava.benchmarks.scenarios.generic_benchmark_creation.create_generic_benchmark_from_paths(...\u001b[39;00m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# calls the same funciton as:\u001b[39;00m\n\u001b[1;32m      7\u001b[0m scenario \u001b[38;5;241m=\u001b[39m ava\u001b[38;5;241m.\u001b[39mbenchmarks\u001b[38;5;241m.\u001b[39mpaths_benchmark(\n\u001b[1;32m      8\u001b[0m    train_stream_without_seq,\n\u001b[1;32m      9\u001b[0m    [test_stream_without_seq],  \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     15\u001b[0m    common_root \u001b[38;5;241m=\u001b[39m images_root\n\u001b[1;32m     16\u001b[0m )\n",
      "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
     ]
    }
   ],
   "source": [
    "# In many continuous learning applications, a model should learn different 'tasks' (groups of classes).\n",
    "# In our case this is not relavant, so all pieces of data belong to the same task: 0\n",
    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
    "\n",
    "# generic_scenario = ava.benchmarks.scenarios.generic_benchmark_creation.create_generic_benchmark_from_paths(...\n",
    "# calls the same funciton as:\n",
    "scenario = ava.benchmarks.paths_benchmark(\n",
    "   train_stream_without_seq,\n",
    "   [test_stream_without_seq],  \n",
    "   task_labels=task_labels,\n",
    "   complete_test_set_only=True,\n",
    "   train_transform=data_transforms['train'],\n",
    "   eval_transform=data_transforms['val'],\n",
    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
    "   common_root = images_root\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here the train function from the strategy is called for each sublist in the train_stream. \n",
    "# Then the eval function is called on the test stream. This function could also be called on the train stream,\n",
    "# which would evaluate all metrics on the entire train stream and might take a long time, since the train stream\n",
    "# is three times as large as the test_stream. \n",
    "# This will take some time to run through, it does not need to be completed for further use.\n",
    "# This is just for demonstartion purposes.\n",
    "\n",
    "number_workers = 4\n",
    "cl_strategy = naive_strategy # or cumulative_strategy\n",
    "\n",
    "for i, experience in enumerate(scenario.train_stream):\n",
    "    cl_strategy.train(experience, num_workers=number_workers)\n",
    "    print('Training completed')\n",
    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
    "    print('Computing accuracy on the whole test set')\n",
    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standard Joint Training As A Baseline\n",
    "When we want to tain a simple model in the standard way, i.e, all data is available to the model at once, the Naive (Basestrategy) instance is used and the training loop is modified slightly.  Here it is important to instantiate the Naive Basestrategy with train_epochs=1 and define the number of epochs used in the loop seperately. Surely, there is a different way to do this too, but this works with the logging the way I want it to. Also, the train_stream is modified to be a list with a single list of data tuples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joint_train_stream_without_seq = [item for sublist in train_stream_without_seq for item in sublist ]\n",
    "random.shuffle(joint_train_stream_without_seq)\n",
    "\n",
    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/joint_training/tb_data\", filename_suffix='test_run')\n",
    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/joint_training/csvlogs')\n",
    "interactive_logger = ava.logging.InteractiveLogger()\n",
    "\n",
    "# This following avalanche function returns a list of accuracy metrics \n",
    "# that are evaluated at different invtervals during the training/evaluation loop\n",
    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
    "\n",
    "combined_logger = ava.training.EvaluationPlugin(\n",
    "    metrics,\n",
    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
    "    suppress_warnings=True)\n",
    "\n",
    "\n",
    "scenario = ava.benchmarks.paths_benchmark(\n",
    "   joint_train_stream_without_seq,\n",
    "   [test_stream_without_seq],  \n",
    "   task_labels=task_labels,\n",
    "   complete_test_set_only=True,\n",
    "   train_transform=data_transforms['train'],\n",
    "   eval_transform=data_transforms['val'],\n",
    "   path_dataset_class=ava.benchmarks.PathsDataset,\n",
    "   common_root = images_root\n",
    ")\n",
    "\n",
    "naive_strategy = ava.training.Naive(\n",
    "    model, optimizer, criterion,  \n",
    "    train_mb_size=batchsize, train_epochs=1, eval_mb_size=batchsize, device=detected_device, plugins=[], evaluator=combined_logger)\n",
    "\n",
    "nr_of_epochs = 9\n",
    "number_workers = 4\n",
    "cl_strategy = naive_strategy # or cumulative_strategy\n",
    "\n",
    "for i in range(nr_of_epochs):\n",
    "    cl_strategy.train(joint_train_stream_without_seq, num_workers=number_workers)\n",
    "    print('Training completed')\n",
    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
    "    print('Computing accuracy on the whole test set')\n",
    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Track Sequnece Information in Avalanche\n",
    "In order to track sequence wise metrics, the information about the sequence affiliations of each image need to be available in the training/testing loop. This was implemented using the SeqPathsDataset class, which is an extension of the Python Dataset class that handles a third field (addditionally to image path and labell), the sequence id. Further, the SeqDataPlugin was added with a modified \\_unpack\\_minibatch callback function that is called during the training/evaluation loop to handle the third seq_id field that the Dataloader of the SeqPathsDataset returns. This plugin also keeps track  of class occurance statistics and saves the targets and predictions during evaluation.\n",
    "This data is collected here to produce comprehensive visualisations of results in form of confusion matricies later on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/seq_id_data/tb_data\", filename_suffix='test_run')\n",
    "csv_logger = ava.logging.CSVLogger(log_folder='./logging_example/seq_id_data/csvlogs')\n",
    "interactive_logger = ava.logging.InteractiveLogger()\n",
    "\n",
    "# This following avalanche function returns a list of accuracy metrics \n",
    "# that are evaluated at different invtervals during the training/evaluation loop\n",
    "metrics = ava.evaluation.accuracy_metrics(stream=True, experience=True)\n",
    "\n",
    "combined_logger = ava.training.EvaluationPlugin(\n",
    "    metrics,\n",
    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
    "    suppress_warnings=True)\n",
    "\n",
    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
    "naive_strategy = ava.training.Naive(\n",
    "    model, optimizer, criterion,  \n",
    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
    "\n",
    "cumulative_strategy = ava.training.Cumulative(\n",
    "    model, optimizer, criterion, train_mb_size=48, train_epochs=1, eval_mb_size=96, device=detected_device, plugins=[seq_plugin], evaluator=combined_logger)\n",
    "\n",
    "\n",
    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
    "scenario = ava.benchmarks.paths_benchmark(\n",
    "   train_stream,\n",
    "   [test_stream],  \n",
    "   task_labels=task_labels,\n",
    "   complete_test_set_only=True,\n",
    "   train_transform=data_transforms['train'],\n",
    "   eval_transform=data_transforms['val'],\n",
    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
    "   common_root = images_root\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "number_workers = 4\n",
    "cl_strategy = naive_strategy # or cumulative_strategy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, experience in enumerate(scenario.train_stream):\n",
    "    cl_strategy.train(experience, num_workers=number_workers)\n",
    "    print('Training completed')\n",
    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
    "    print('Computing accuracy on the whole test set')\n",
    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comprehensive Metrics and Generic Logger\n",
    "\n",
    "In order to get a better understanding of the performance of different continual leaning / rehearsal strategies, several different metrics were implemented. This includes sequence-wise metrics, class-wise metrics, season-based evaluations and several more. See the /evaluation/metrics/accuracy.py implementation comments for more details.\n",
    "\n",
    "Each metric inherits from the GenericPluginMetric class defined in /metrics/metric\\_definitions.py which coordinates the update()-/reset()-/result()- function call of the metrics at different stages during the training, depending on whether the metric is calculated over an iteration, experience, epoch or stream. More precisely, the EvaluationPlugin has a funciton called \\_update\\_metrics() which calls the callback function of the GenericPluginMetric which again calls the appropriate update()-/reset()-/result()- function for each metric and collects the results in a list called metric_values at the appropriate stage. Then the evalutaion plugin also calls all loggers and passes the metric_values list as an argument. \n",
    "\n",
    "The GenericPluginMetric was written to handle the newly added accuracy metrics. It is unfortunately not as generic as the name makes it out to be. It could definately be improved upon. Also the implementation of the metrics could be improved to avoid for example the looping over sequence ids or computing the same updates more than once for similar metrics like class-wise accuracies for each class and class avg accuracy. The metrics that are evaluated for every epoch, experience or iteration are quite slow. Doing this after each epoch/experience/iteration would mean that the whole training-evaluation process would take a very long time. \n",
    "\n",
    "when calling the eval function of the naive strategy (base\\_strategy) i have added the optional boolean variable 'last_eval', which, if true means a confusion matrix and dictionary is pickled that contains all sorts of infomration on the current metrics, predictions, targets and so on. This implemented to allow quick overviews of the performance of sdifferent methods. The data for this confusion matrix is collected at different points in the evaluation/training cycle and saved as variables of the strategy. Currently, it is required that some specific metrics are tracked when last_eval is true so that these variables are set. A work aroud could be easily implemented. Also, a label_dict file has to be passed to the strategy.  The actual confusion plot is created in the after\\_eval() callback funciton of the Evaluation Plugin in evaluation\\_plugin()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_dict_path = os.path.join(os.getcwd(), '../../data/label_dictionaries/BIRDS_11_Species.pkl')\n",
    "with open(label_dict_path, 'rb') as handle:\n",
    "     label_dict = pkl.load(handle)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'cv0_expsize128_crop'\n",
    "data_streams_path = os.path.join(os.getcwd(), '../../data/data_stream_files/BW_stream_files/')\n",
    "with open(data_streams_path+file.replace('384','128')+'_winter_val_stream.pkl', 'rb') as handle:\n",
    "     validation_stream_winter = pkl.load(handle)\n",
    "\n",
    "with open(data_streams_path+file.replace('384','128')+'_winter_test_stream.pkl', 'rb') as handle:\n",
    "    test_stream_winter = pkl.load(handle)\n",
    "\n",
    "with open(data_streams_path+file.replace('_crop','').replace('384','128')+'_exp_season_split_dict.pkl', 'rb') as handle:\n",
    "    exp_season_split_dict = pkl.load(handle)\n",
    "    \n",
    "validation_stream_winter_files = np.array(validation_stream_winter)[:,0]\n",
    "validation_stream_winter_files = [images_root+s[0:] for s in validation_stream_winter_files ]\n",
    "test_stream_winter_files = np.array(test_stream_winter)[:,0]\n",
    "test_stream_winter_files = [images_root+s[0:] for s in test_stream_winter_files ]\n",
    "\n",
    "winter_exp = exp_season_split_dict['winter']\n",
    "summer_exp = exp_season_split_dict['summer']\n",
    "winter_train_exp = [train_stream[i] for i in winter_exp]\n",
    "all_winter_train_data = [i for sublist in winter_train_exp for i in sublist]\n",
    "all_winter_train_files = np.array(all_winter_train_data)[:,0]\n",
    "all_winter_train_files = [images_root+s[0:] for s in all_winter_train_files ]\n",
    "all_winter_files = all_winter_train_files +test_stream_winter_files+validation_stream_winter_files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "nc =11 #number of classes\n",
    "\n",
    "metrics = [ava.metrics.accuracy.Top1AccTransfer(nr_train_exp=len(train_stream), reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.ClassTop_nAvgAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.ClassTop_nAvgAcc(nr_classes=nc, n=2, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.ClasswiseTop_nAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.Top_nAcc(n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonTop_nAcc(n=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonTop_nAcc(n=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.Top_nAcc(n=2, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeqTop_nAcc(n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeqMaxAcc( reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonSeqTop_nAcc(num=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonSeqTop_nAcc(num=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeqClasswiseTop_nAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeqClassTop_nAvgAcc(nr_classes=nc, n=1, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeqClassAvgMaxAcc(nr_classes=nc, reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonSeqClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='summer', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeasonSeqClassTop_nAcc(nr_classes=nc, n=1, winter_files_list=all_winter_files, season='winter', reset_at='stream', emit_at='stream', mode='eval'),\n",
    "           ava.metrics.accuracy.SeqAnyAcc(reset_at='stream', emit_at='stream', mode='eval')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/comprehnesive_metrics/tb_data\", filename_suffix='test_run')\n",
    "csv_logger = ava.logging.GenericCSVLogger(log_folder='./logging_example/comprehnesive_metrics/csvlogs')\n",
    "interactive_logger = ava.logging.InteractiveLogger()\n",
    "combined_logger = ava.training.EvaluationPlugin(\n",
    "    metrics,\n",
    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
    "    suppress_warnings=True)\n",
    "\n",
    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
    "naive_strategy = ava.training.Naive(\n",
    "    model, optimizer, criterion,  \n",
    "    train_mb_size=32, train_epochs=1, \n",
    "    eval_mb_size=48, device=detected_device, \n",
    "    plugins=[seq_plugin], evaluator=combined_logger,\n",
    "    label_dict=label_dict)\n",
    "\n",
    "number_workers = 8\n",
    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
    "scenario = ava.benchmarks.paths_benchmark(\n",
    "   train_stream,\n",
    "   [test_stream],  \n",
    "   task_labels=task_labels,\n",
    "   complete_test_set_only=True,\n",
    "   train_transform=data_transforms['train'],\n",
    "   eval_transform=data_transforms['val'],\n",
    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
    "   common_root = images_root\n",
    ")\n",
    "\n",
    "last_eval = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- >> Start of training phase << --\n",
      "Start of experience:  0\n",
      "Current Classes:  [10, 3, 4]\n",
      "-- Starting training on experience 0 (Task 0) from train stream --\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/boehlke/AMMOD/avalanche_extension/avalanche/avalanche/training/plugins/sequence_data.py:104: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  strategy.seq_codes = torch.tensor(strategy.mbatch[2][0]).to(strategy.device)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:15<00:00,  3.96s/it]\n",
      "Epoch 0 ended.\n",
      "-- >> End of training phase << --\n",
      "Training completed\n",
      "Computing accuracy on the whole test set\n",
      "-- >> Start of eval phase << --\n",
      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
      " 34%|██████████████████████████████▊                                                             | 120/358 [01:56<03:41,  1.08it/s]"
     ]
    }
   ],
   "source": [
    "cl_strategy = naive_strategy\n",
    "for i, experience in enumerate(scenario.train_stream):\n",
    "    cl_strategy.train(experience, num_workers=number_workers)\n",
    "    print('Training completed')\n",
    "    #cl_strategy.eval(scenario.train_stream, num_workers=number_workers)\n",
    "    \n",
    "    print('Computing accuracy on the whole test set')\n",
    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers, last_eval= last_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rehearsal Learning \n",
    "\n",
    "In the examples so far, the strategy for continual learning meant either a naive approach (finetuning with latest new data) or a cumulative approach (finetuning with all data seen so far). A bunch of methods were implemented to use the rehearsal approach to continual learning to ensure the model does not overwrite previously learned knowledge when finetuning with the new incoming data. The idea is to select images from the memory strategically and use them together with the new data to fine tune the model. \n",
    "\n",
    "The main functionalities of this approach where implemented in the ClassImbalanceRehersalPlugin class in /avalanche/training/plugins/imbalance_focus_replay.py. Each instance of this class or child class needs to pass the 'buffer\\_data\\_ratio' variable which defines how many images from the memory are selcted for each images in the current experience. The callback function train_dataset_adaptation() is called to create a AvalancheSubset of the memory data by using the indicies for the memory data returned by the \\_get\\_indices() function, which is implemented by the child class. If the memory is not yet large enough, the entire memory is used for rehearsal. After the finetuning is completed, the after\\_training\\_exp() callback function is called to add the current experience to the memory. \n",
    "\n",
    "A full list of all the child classes implementing different rehearsal selection methods:\n",
    "MaximallyInterferedRetrievalRehersalPlugin, \n",
    "RandomRehersal, \n",
    "ClassErrorRehersalPlugin, \n",
    "ClassErrorRehersalTemperaturePlugin, \n",
    "ClassFrequencyRehearsalPlugin, \n",
    "WeightedeMovingClassErrorAverageRehersalPlugin,\n",
    "ClassErrorFrequencyAvgRehearsalPlugin,\n",
    "FillExpBasedRehearsalPlugin,\n",
    "FillExpOversampleBasedRehearsalPlugin\n",
    "\n",
    "A more detailed description of what they do and how they are implemented can be found in the comments under each class definition in imbalance_focus_replay.py\n",
    "\n",
    "To use these rehearsal strategies, the correct plugin just has to be added to the list of plugins for the Naive strategy class.  \n",
    "In the following the use of the class error based rehearsal method is demonstrated. In order to use this rehearsal strategy, validation data is necessary to estimate the clas error rates. Further it is required to use the ClasswiseTop_nAcc with n=1 as a metric, where the validation data's class accuracies are stored in a stratedy variable. The validation data needs to be used for evaluation at least once, before the plugin aims to select images from the memory. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_stream_file = data_dir_path+'data_stream_files/BW_stream_files/cv0_expsize128_crop_val_stream.pkl'\n",
    "with open(val_stream_file, 'rb') as handle: \n",
    "    val_stream = pkl.load(handle)\n",
    "# The val_stream is passed to the pahts_benchmark function below, that creats the 'scenario'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/rehearsal_learning/tb_data\", filename_suffix='test_run')\n",
    "csv_logger = ava.logging.GenericCSVLogger(log_folder='./logging_example/rehearsal_learning/csvlogs')\n",
    "interactive_logger = ava.logging.InteractiveLogger()\n",
    "combined_logger = ava.training.EvaluationPlugin(\n",
    "    metrics,\n",
    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
    "    suppress_warnings=True)\n",
    "\n",
    "number_workers = 8\n",
    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
    "scenario = ava.benchmarks.paths_benchmark(\n",
    "   train_stream,\n",
    "   [test_stream],  \n",
    "   other_streams_lists_of_files={'validation': [val_stream]},\n",
    "   task_labels=task_labels,\n",
    "   complete_test_set_only=True,\n",
    "   train_transform=data_transforms['train'],\n",
    "   eval_transform=data_transforms['val'],\n",
    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
    "   common_root = images_root\n",
    ")\n",
    "\n",
    "\n",
    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
    "class_error_method = ava.training.plugins.ClassErrorRehersalPlugin(buffer_data_ratio=1)\n",
    "plugins_list = [seq_plugin, class_error_method]\n",
    "\n",
    "naive_strategy = ava.training.Naive(\n",
    "    model, optimizer, criterion,  \n",
    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=plugins_list, evaluator=combined_logger)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_strategy = naive_strategy\n",
    "for i, experience in enumerate(scenario.train_stream):\n",
    "    cl_strategy.train(experience, num_workers=number_workers)\n",
    "    print('Training completed')\n",
    "    cl_strategy.eval(scenario.validation_stream, num_workers=number_workers, last_eval=True)\n",
    "    print('Computing accuracy on the whole test set')\n",
    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers, last_eval=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rehearsal Learning with Limited Memory \n",
    "\n",
    "In the real world, there will never be a unlimited memory. Two memory filling strategies were implemented in /avalanche/training/plugins/class_balancing_replay.py. These are the baseline stadnard reservoir sampling used to draw uniformly from a stream and a strategy specifically designed for class imbalace called class balancing reservoir sampling ('Online Continual Learning from Imbalanced Data' by Chrysakis et al.)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb_logger = ava.logging.TensorboardLogger(tb_log_dir=\"./logging_example/rehearsal_learning_limited_memory/tb_data\", filename_suffix='test_run')\n",
    "csv_logger = ava.logging.GenericCSVLogger(log_folder='./logging_example/rehearsal_learning_limited_memory/csvlogs')\n",
    "interactive_logger = ava.logging.InteractiveLogger()\n",
    "combined_logger = ava.training.EvaluationPlugin(\n",
    "    metrics,\n",
    "    loggers=[tb_logger, csv_logger, interactive_logger],\n",
    "    suppress_warnings=True)\n",
    "\n",
    "number_workers = 8\n",
    "task_labels = np.zeros((1,len(train_stream))).astype(int).tolist()[0] \n",
    "scenario = ava.benchmarks.paths_benchmark(\n",
    "   train_stream,\n",
    "   [test_stream],  \n",
    "   other_streams_lists_of_files={'validation': [val_stream]},\n",
    "   task_labels=task_labels,\n",
    "   complete_test_set_only=True,\n",
    "   train_transform=data_transforms['train'],\n",
    "   eval_transform=data_transforms['val'],\n",
    "   path_dataset_class=ava.benchmarks.SeqPathsDataset,\n",
    "   common_root = images_root\n",
    ")\n",
    "\n",
    "\n",
    "seq_plugin = ava.training.plugins.SeqDataPlugin()\n",
    "plugin_cbrs = ava.training.plugins.ClassBalancingReservoirMemoryRehersalPlugin(buffer_data_ratio=1, memory_size=500, nr_classes=11, \n",
    "                                                          rehearsal_selection_strategy='ce', temperature=0.2)\n",
    "plugins_list = [seq_plugin, plugin_cbrs]\n",
    "\n",
    "naive_strategy = ava.training.Naive(\n",
    "    model, optimizer, criterion,  \n",
    "    train_mb_size=32, train_epochs=1, eval_mb_size=48, device=detected_device, plugins=plugins_list, evaluator=combined_logger)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_strategy = naive_strategy\n",
    "for i, experience in enumerate(scenario.train_stream):\n",
    "    cl_strategy.train(experience, num_workers=number_workers)\n",
    "    print('Training completed')\n",
    "    cl_strategy.eval(scenario.validation_stream, num_workers=number_workers)\n",
    "    print('Computing accuracy on the whole test set')\n",
    "    cl_strategy.eval(scenario.test_stream, num_workers=number_workers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bringing it all together for cross-validated Experiments\n",
    "\n",
    "In order to perform experiments on all cross-validation datasets a script was used which reads arguments from the command line withwhich different parameters and options can be set for all the continual learning strategies implemented. The script is found in scripts/cont_ava.py. The scripts/example_script shows how this script was used to run experiments with different settings on multiple cross validation scripts. The names of the arguments and a brief statement on what they are can be seen with python cont_ava.py --help or in the file itself. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ava-test5",
   "language": "python",
   "name": "ava-test5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
