{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import cv2 as cv\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "from py.PlotUtils import get_percentiles\n",
    "from py.Dataset import Dataset\n",
    "from py.ImageUtils import display_images\n",
    "from py.Labels import LABELS\n",
    "\n",
    "DIR = '/home/kleinsteuber/vscode/ResizedSessions_NoBackup' # dataset directory\n",
    "CONFIGS_BEAVER_01 = [\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 1024,\n",
    "        \"step\": 20,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 2048,\n",
    "        \"step\": 20,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 4096,\n",
    "        \"step\": 20,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 512,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 1024,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 2048,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 4096,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 8192,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 1024,\n",
    "        \"step\": 40,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 2048,\n",
    "        \"step\": 40,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 4096,\n",
    "        \"step\": 40,\n",
    "        \"random\": True,\n",
    "    },\n",
    "]\n",
    "\n",
    "CONFIGS_COMPARE_SESSIONS = [\n",
    "    {\n",
    "        \"session\": \"beaver_01\",\n",
    "        \"clusters\": 4096,\n",
    "        \"step\": 20,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"marten_01\",\n",
    "        \"clusters\": 4096,\n",
    "        \"step\": 20,\n",
    "        \"random\": True,\n",
    "    },\n",
    "    {\n",
    "        \"session\": \"gfox_03\",\n",
    "        \"clusters\": 4096,\n",
    "        \"step\": 20,\n",
    "        \"random\": True,\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 3 sessions\n"
     ]
    }
   ],
   "source": [
    "ds = Dataset(DIR)\n",
    "\n",
    "def read_results(clusters: int, session: str, step: int = 30, size: int = None, random: bool = False, motion: bool = False):\n",
    "    # size = step by default\n",
    "    if size is None:\n",
    "        size = step\n",
    "    \n",
    "    # Get filename suffix\n",
    "    suffix = \"\"\n",
    "    if random:\n",
    "        suffix += \"_random\"\n",
    "    if motion:\n",
    "        suffix += \"_motion\"\n",
    "\n",
    "    # correct name (e.g. from beaver_01 to Beaver_01)\n",
    "    session = ds.create_session(session).name\n",
    "\n",
    "    # Read CSV\n",
    "    test_labels = []\n",
    "    test_df = []\n",
    "    with open(f\"./bow_train_NoBackup/{session}/bow_eval_{step}_{size}_{clusters}{suffix}.csv\", \"r\") as f:\n",
    "        for line in f:\n",
    "            entries = line.split(\",\")\n",
    "            # Get label\n",
    "            filename = entries[0]\n",
    "            img_number = int(filename[-9:-4])\n",
    "            if img_number > LABELS[session][\"max\"] or img_number in LABELS[session][\"not_annotated\"]:\n",
    "                continue\n",
    "            is_normal = (img_number in LABELS[session][\"normal\"])\n",
    "            test_labels.append(1 if is_normal else -1)\n",
    "            # Get decision function values\n",
    "            test_df.append([float(df) for df in entries[1:]])\n",
    "    test_labels = np.array(test_labels)\n",
    "    test_df = np.array(test_df)\n",
    "    print(f\"{len(test_df)} test results with {len(test_labels)} labels\")\n",
    "    return test_labels, test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_aucs(test_labels, test_dfs):\n",
    "    aucs = []\n",
    "    for i in range(test_dfs.shape[1]):\n",
    "        fpr, tpr, thresholds = roc_curve(test_labels, test_dfs[:,i])\n",
    "        aucs.append(auc(fpr, tpr))\n",
    "    return aucs\n",
    "\n",
    "def plot_scores(configs, scores, savefile=None, label_below=[], color=\"darkgoldenrod\", figsize=(15, 10)):\n",
    "    plt.rcParams['text.usetex'] = True\n",
    "    plt.rcParams.update({\"font.size\": 18})\n",
    "    plt.figure(figsize=figsize)\n",
    "    bp_dict = plt.boxplot(scores, medianprops={\"linewidth\": 1.5, \"color\": color}, widths=0.68)\n",
    "\n",
    "    for i, line in enumerate(bp_dict['medians']):\n",
    "        # get position data for median line\n",
    "        x, y = line.get_xydata()[1] # top of median line\n",
    "        # overlay median value\n",
    "        if i in label_below:\n",
    "            plt.text(x - 0.335, y - 0.0007, f\"{y:.4f}\", verticalalignment=\"top\", horizontalalignment=\"center\", color=color) # draw below, centered\n",
    "        else:\n",
    "            plt.text(x - 0.335, y + 0.0005, f\"{y:.4f}\", verticalalignment=\"bottom\", horizontalalignment=\"center\", color=color) # draw above, centered\n",
    "\n",
    "    plt.xticks(np.arange(1, len(configs) + 1), [f\"k={config['clusters']}\\ns={config['step'] if 'step' in config else 30}\" for config in configs])\n",
    "    plt.grid(True, ls=\"dotted\", lw=0.5)\n",
    "    # plt.ylim((0.7, 0.9))\n",
    "    # plt.legend()\n",
    "    if savefile is not None:\n",
    "        plt.savefig(savefile, bbox_inches=\"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare configurations on Beaver_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "beaver_01_scores = [get_aucs(*read_results(**config)) for config in CONFIGS_BEAVER_01]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 43996 characters]",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scores(CONFIGS_BEAVER_01, beaver_01_scores, savefile=\"plots/approach3/boxplot_random.pdf\", label_below=[2,8,9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n"
     ]
    }
   ],
   "source": [
    "def get_elimination_rate_vals(test_labels, test_dfs, percentile=0.95):\n",
    "    percentiles = []\n",
    "    for i in range(test_dfs.shape[1]):\n",
    "        fpr, tpr, thresholds = roc_curve(test_labels, test_dfs[:,i])\n",
    "        percentiles.append(get_percentiles(fpr, tpr, thresholds, percentiles=[percentile], verbose = False)[0])\n",
    "    return percentiles\n",
    "\n",
    "beaver_01_tnr_scores = [get_elimination_rate_vals(*read_results(**config)) for config in CONFIGS_BEAVER_01]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38948 characters]",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scores(CONFIGS_BEAVER_01, beaver_01_tnr_scores, savefile=\"plots/approach3/boxplot_random_tnr95.pdf\", label_below=[2], color=\"red\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Marten_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Marten_01\n",
      "Loaded scans.\n",
      "3105 test results with 3105 labels\n",
      "Session 'GFox_03' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session03-VIELAAS_GFox_03\n",
      "Loaded scans.\n",
      "2738 test results with 2738 labels\n"
     ]
    }
   ],
   "source": [
    "session_scores = [get_aucs(*read_results(**config)) for config in CONFIGS_COMPARE_SESSIONS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 15876 characters]",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scores(CONFIGS_COMPARE_SESSIONS, session_scores, savefile=\"plots/approach3/boxplot_random_sessions.pdf\", figsize=(8, 6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Session 'Beaver_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Beaver_01\n",
      "Loaded scans.\n",
      "695 test results with 695 labels\n",
      "Session 'Marten_01' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session01-VIELAAS_Marten_01\n",
      "Loaded scans.\n",
      "3105 test results with 3105 labels\n",
      "Session 'GFox_03' at folder: /home/kleinsteuber/vscode/ResizedSessions_NoBackup/VIELAAS_Spring_Session03-VIELAAS_GFox_03\n",
      "Loaded scans.\n",
      "2738 test results with 2738 labels\n"
     ]
    }
   ],
   "source": [
    "session_tnr_scores = [get_elimination_rate_vals(*read_results(**config)) for config in CONFIGS_COMPARE_SESSIONS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 17992 characters]",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scores(CONFIGS_COMPARE_SESSIONS, session_tnr_scores, savefile=\"plots/approach3/boxplot_random_sessions_tnr95.pdf\", figsize=(8, 6), color=\"red\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
