{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Approach 2: Background estimation\n",
    "For this approach, we consider a time series of motion (and lapse) images, i.e. all images in the serious are close together in time (here +- 10 min).\n",
    "\n",
    "We extract the median image by taking the median of every pixel over all images in the series. Then, for every image the squared pixel-wise difference to the median image is computed. As in the previous approach, we assume that anomalous images highly differ from the median image.\n",
    "\n",
    "The median image is used as a (possibly more accurate) approximation of the background instead of the closest lapse image.\n",
    "\n",
    "### Advantages compared to approach 1a\n",
    "- more resistent to noise\n",
    "- more resistent to small object movements\n",
    "- only sensitive to camera movement within the time series\n",
    "\n",
    "### Problems\n",
    "- still no color information\n",
    "- the lighting difference within the series are still a problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "Found 3 sessions\n",
      "Session 'GFox_03' at folder: ./ResizedSessions_NoBackup/VIELAAS_Spring_Session03-VIELAAS_GFox_03\n",
      "Loaded scans.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import warnings\n",
    "from scipy import ndimage\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from py.PlotUtils import plot_roc_curve, get_percentiles\n",
    "from py.Dataset import Dataset\n",
    "from py.ImageUtils import display_images\n",
    "from py.Labels import LABELS\n",
    "\n",
    "DIR = './ResizedSessions_NoBackup' # dataset directory\n",
    "\n",
    "ds = Dataset(DIR)\n",
    "session = ds.create_session(\"gfox_03\")\n",
    "append_lapse_image = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating motion map...\n",
      "45 images in set\n"
     ]
    }
   ],
   "source": [
    "img_set = session.get_random_motion_image_set(day_only=True)\n",
    "if append_lapse_image:\n",
    "    before, after, rel = img_set[0].get_closest_lapse_images()\n",
    "    if rel < 0.15 and before is not None:\n",
    "        img_set.append(before)\n",
    "    if rel > 0.85 and after is not None:\n",
    "        img_set.append(after)\n",
    "print(f\"{len(img_set)} images in set\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f93046643394902aafb1fa2673f4ba4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/45 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Read all images\n",
    "imgs = []\n",
    "for img in tqdm(img_set):\n",
    "    arr = img.read(gray=True)\n",
    "    # normalize\n",
    "    # arr = (arr - np.mean(arr)) / np.std(arr)\n",
    "    imgs.append(arr)\n",
    "imgs = np.array(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 480016 characters]",
      "text/plain": [
       "<Figure size 1080x432 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_images(imgs[:6], list(range(len(imgs)))[:12], row_size=3, size=(5, 3), cmap=\"gray\", vmin=0, vmax=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f13e2cc6be0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 240156 characters]",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Find median image\n",
    "median = np.median(imgs, axis=0)\n",
    "plt.figure(figsize=(10, 8))\n",
    "plt.imshow(median, cmap=\"gray\", vmin=0, vmax=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "diffs = abs(imgs - median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 490148 characters]",
      "text/plain": [
       "<Figure size 1440x648 with 12 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_images(diffs[:12], list(range(len(diffs)))[:12], row_size=4, size=(5, 3), cmap=\"gray\", vmin=0, vmax=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Complete test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "\n",
    "with tqdm(total=len(session.motion_dates)) as pbar:\n",
    "    for img_set in session.generate_motion_image_sets():\n",
    "        # Optionally append a close lapse image\n",
    "        if append_lapse_image:\n",
    "            before, after, rel = img_set[0].get_closest_lapse_images()\n",
    "            if rel < 0.15 and before is not None:\n",
    "                img_set.append(before)\n",
    "            if rel > 0.85 and after is not None:\n",
    "                img_set.append(after)\n",
    "            print(f\"{len(img_set)} images in set\")\n",
    "        \n",
    "        # Read all images\n",
    "        imgs = []\n",
    "        for img in img_set:\n",
    "            arr = img.read(gray=True)\n",
    "            # normalize\n",
    "            # arr = (arr - np.mean(arr)) / np.std(arr)\n",
    "            imgs.append(arr)\n",
    "            pbar.update(1)\n",
    "        imgs = np.array(imgs)\n",
    "\n",
    "        # Find median image, calculate diffs\n",
    "        median = np.median(imgs, axis=0)\n",
    "        diffs = (imgs - median) ** 2\n",
    "        means = np.mean(diffs, axis=(1,2))\n",
    "        vars = np.var(diffs, axis=(1,2))\n",
    "        for img, mean, var in zip(img_set, means, vars):\n",
    "            results.append([img.filename, mean, var])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save and load results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f\"Cache_NoBackup/approach2_{session.name}_results.npy\", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Beaver_01_00001.jpg', '0.03157315058171535',\n",
       "        '0.008842501879227922'],\n",
       "       ['Beaver_01_00002.jpg', '0.01590260488964755',\n",
       "        '0.004257656931850018'],\n",
       "       ['Beaver_01_00003.jpg', '0.005390408061453107',\n",
       "        '0.00032305310691840193'],\n",
       "       ...,\n",
       "       ['Beaver_01_00693.jpg', '0.06754646257597216',\n",
       "        '0.012471854739153881'],\n",
       "       ['Beaver_01_00694.jpg', '0.055833027874570236',\n",
       "        '0.008597549614760561'],\n",
       "       ['Beaver_01_00695.jpg', '0.07819557309669879',\n",
       "        '0.009699179824057685']], dtype='<U22')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = np.load(f\"Cache_NoBackup/approach2_{session.name}_results.npy\")\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize datapoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 95612 characters]",
      "text/plain": [
       "<Figure size 1152x1152 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "normal = []\n",
    "anomalous = []\n",
    "motion_only_session = (session.name[0] == \"G\")\n",
    "for filename, mean, var in results:\n",
    "    vals = [float(mean), float(var)]\n",
    "    if not filename.startswith(\"20\"):\n",
    "        # motion img, check label\n",
    "        img_nr = int(filename[-9:-4])\n",
    "        if img_nr in LABELS[session.name][\"normal\"]:\n",
    "            normal.append(vals)\n",
    "        elif img_nr not in LABELS[session.name][\"not_annotated\"]:\n",
    "            anomalous.append(vals)\n",
    "    else:\n",
    "        # lapse image, always normal\n",
    "        assert not motion_only_session\n",
    "        normal.append(vals)\n",
    "normal = np.array(normal)\n",
    "anomalous = np.array(anomalous)\n",
    "\n",
    "plt.figure(figsize=(16,16))\n",
    "plt.scatter(anomalous[:,0], anomalous[:,1], label=\"Anomalous\", color=\"red\", marker=\"x\")\n",
    "plt.scatter(normal[:,0], normal[:,1], label=\"Normal\", color=\"blue\", marker=\"x\")\n",
    "plt.xscale(\"log\")\n",
    "plt.yscale(\"log\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Thresholding & Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3105, 2)\n",
      "(3105,)\n"
     ]
    }
   ],
   "source": [
    "y_anom = anomalous\n",
    "assert y_anom.shape[1] == 2\n",
    "\n",
    "y_norm = normal\n",
    "assert y_norm.shape[1] == 2\n",
    "\n",
    "X = np.concatenate([y_norm, y_anom])\n",
    "print(X.shape)\n",
    "y = np.concatenate([-np.ones((y_norm.shape[0])), np.ones((y_anom.shape[0]))])\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sqmean auc = 0.6018951247987953\n",
      "0.9 percentile : TPR = 0.9057, FPR = 0.9684 <-> TNR = 0.0316 @ thresh 0.0003758795016096645\n",
      "0.95 percentile : TPR = 0.9727, FPR = 0.9764 <-> TNR = 0.0236 @ thresh 0.0002671284369219634\n",
      "0.98 percentile : TPR = 0.9850, FPR = 0.9772 <-> TNR = 0.0228 @ thresh 0.0002464688812231967\n",
      "0.99 percentile : TPR = 0.9904, FPR = 0.9819 <-> TNR = 0.0181 @ thresh 0.0002225624735737834\n",
      "sqvar auc = 0.5664796239110761\n",
      "0.9 percentile : TPR = 0.9003, FPR = 0.9633 <-> TNR = 0.0367 @ thresh 9.54648154742993e-06\n",
      "0.95 percentile : TPR = 0.9508, FPR = 0.9684 <-> TNR = 0.0316 @ thresh 3.238179320252646e-06\n",
      "0.98 percentile : TPR = 0.9809, FPR = 0.9815 <-> TNR = 0.0185 @ thresh 9.98492551834797e-07\n",
      "0.99 percentile : TPR = 0.9904, FPR = 0.9857 <-> TNR = 0.0143 @ thresh 5.41580910027951e-07\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19840 characters]",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 20084 characters]",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "vals = [\"sqmean\", \"sqvar\"]\n",
    "for i in range(2):\n",
    "    fpr, tpr, thresholds, auc_score = plot_roc_curve(y, X[:,i], f\"Background estimation, {vals[i]}\", savefile=f\"plots/approach2/roc_curves/{session.name}_{vals[i]}\", show=False, figsize=(5,5))\n",
    "    print(f\"{vals[i]} auc = {auc_score}\")\n",
    "    get_percentiles(fpr, tpr, thresholds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 2: Add Gaussian filter\n",
    "Two orders possible:\n",
    "- 1. Gaussian filter images, 2. Calculate median image\n",
    "- 1. Calculate median image, 2. Gaussian filter images + median image\n",
    "\n",
    "Experiments show that order 1 yields better results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "sigma = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07b5ec5af7be40d3a39919d338409cf9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4285 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = []\n",
    "\n",
    "with tqdm(total=len(session.motion_dates)) as pbar:\n",
    "    for img_set in session.generate_motion_image_sets():\n",
    "        # Optionally append a close lapse image\n",
    "        if append_lapse_image:\n",
    "            before, after, rel = img_set[0].get_closest_lapse_images()\n",
    "            if rel < 0.15 and before is not None:\n",
    "                img_set.append(before)\n",
    "            if rel > 0.85 and after is not None:\n",
    "                img_set.append(after)\n",
    "            print(f\"{len(img_set)} images in set\")\n",
    "        \n",
    "        # Read all images\n",
    "        imgs = []\n",
    "        for img in img_set:\n",
    "            arr = img.read(gray=True)\n",
    "            # normalize\n",
    "            # arr = (arr - np.mean(arr)) / np.std(arr)\n",
    "            imgs.append(arr)\n",
    "            pbar.update(1)\n",
    "        imgs = np.array(imgs)\n",
    "\n",
    "        # filter images\n",
    "        for i in range(len(imgs)):\n",
    "            imgs[i] = ndimage.gaussian_filter(imgs[i], sigma=sigma)\n",
    "\n",
    "        # Find median image\n",
    "        median = np.median(imgs, axis=0)\n",
    "\n",
    "        # calculate diffs\n",
    "        diffs = (imgs - median) ** 2\n",
    "        means = np.mean(diffs, axis=(1,2))\n",
    "        vars = np.var(diffs, axis=(1,2))\n",
    "        for img, mean, var in zip(img_set, means, vars):\n",
    "            results.append([img.filename, mean, var])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save and load results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f\"Cache_NoBackup/approach2_{session.name}_sigma{sigma}_results.npy\", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Beaver_01_00001.jpg', '0.01584533247488625',\n",
       "        '0.0026823466841092818'],\n",
       "       ['Beaver_01_00002.jpg', '0.008824170959448192',\n",
       "        '0.0010263360919268011'],\n",
       "       ['Beaver_01_00003.jpg', '0.0013506161474556024',\n",
       "        '5.737223796775002e-05'],\n",
       "       ...,\n",
       "       ['Beaver_01_00693.jpg', '0.04884697176797362',\n",
       "        '0.007907754301799172'],\n",
       "       ['Beaver_01_00694.jpg', '0.037500496727618925',\n",
       "        '0.004932861765486779'],\n",
       "       ['Beaver_01_00695.jpg', '0.06166672730106219',\n",
       "        '0.004550548160890873']], dtype='<U22')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = np.load(f\"Cache_NoBackup/approach2_{session.name}_sigma{sigma}_results.npy\")\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize datapoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 111672 characters]",
      "text/plain": [
       "<Figure size 1152x1152 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "normal = []\n",
    "anomalous = []\n",
    "motion_only_session = (session.name[0] == \"G\")\n",
    "for filename, mean, var in results:\n",
    "    vals = [float(mean), float(var)]\n",
    "    if not filename.startswith(\"20\"):\n",
    "        # motion img, check label\n",
    "        img_nr = int(filename[-9:-4])\n",
    "        if img_nr in LABELS[session.name][\"normal\"]:\n",
    "            normal.append(vals)\n",
    "        elif img_nr not in LABELS[session.name][\"not_annotated\"]:\n",
    "            anomalous.append(vals)\n",
    "    else:\n",
    "        # lapse image, always normal\n",
    "        assert not motion_only_session\n",
    "        normal.append(vals)\n",
    "normal = np.array(normal)\n",
    "anomalous = np.array(anomalous)\n",
    "\n",
    "plt.figure(figsize=(16,16))\n",
    "plt.scatter(anomalous[:,0], anomalous[:,1], label=\"Anomalous\", color=\"red\", marker=\"x\")\n",
    "plt.scatter(normal[:,0], normal[:,1], label=\"Normal\", color=\"blue\", marker=\"x\")\n",
    "plt.xscale(\"log\")\n",
    "plt.yscale(\"log\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Thresholding & Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2738, 2)\n",
      "(2738,)\n"
     ]
    }
   ],
   "source": [
    "y_anom = anomalous\n",
    "assert y_anom.shape[1] == 2\n",
    "\n",
    "y_norm = normal\n",
    "assert y_norm.shape[1] == 2\n",
    "\n",
    "X = np.concatenate([y_norm, y_anom])\n",
    "print(X.shape)\n",
    "y = np.concatenate([-np.ones((y_norm.shape[0])), np.ones((y_anom.shape[0]))])\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sqmean auc = 0.9660719179422934\n",
      "0.9 percentile : TPR = 0.9024, FPR = 0.0630 <-> TNR = 0.9370 @ thresh 0.0019918023103186093\n",
      "0.95 percentile : TPR = 0.9512, FPR = 0.0770 <-> TNR = 0.9230 @ thresh 0.0015200018108547971\n",
      "0.98 percentile : TPR = 0.9837, FPR = 0.1236 <-> TNR = 0.8764 @ thresh 0.0009584579791728374\n",
      "0.99 percentile : TPR = 0.9919, FPR = 0.1693 <-> TNR = 0.8307 @ thresh 0.0006298934033365306\n",
      "sqvar auc = 0.9812065275548422\n",
      "0.9 percentile : TPR = 0.9024, FPR = 0.0445 <-> TNR = 0.9555 @ thresh 3.187573137314759e-05\n",
      "0.95 percentile : TPR = 0.9512, FPR = 0.0490 <-> TNR = 0.9510 @ thresh 2.6650384273320288e-05\n",
      "0.98 percentile : TPR = 0.9837, FPR = 0.0783 <-> TNR = 0.9217 @ thresh 1.4823147816135393e-05\n",
      "0.99 percentile : TPR = 0.9919, FPR = 0.0827 <-> TNR = 0.9173 @ thresh 1.3226769228072742e-05\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 16316 characters]",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 16056 characters]",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "vals = [\"sqmean\", \"sqvar\"]\n",
    "for i in range(2):\n",
    "    fpr, tpr, thresholds, auc_score = plot_roc_curve(y, X[:,i], f\"Background estimation, {vals[i]}, σ={sigma}\", savefile=f\"plots/approach2/roc_curves/{session.name}_{vals[i]}_sigma{sigma}\", show=False, figsize=(5,5))\n",
    "    print(f\"{vals[i]} auc = {auc_score}\")\n",
    "    get_percentiles(fpr, tpr, thresholds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
