plots

day_vs_night.ipynb

@@ -0,0 +1,77 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from skimage import io\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "img = io.imread(\"sample.jpg\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "99.88014299849979"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.mean([abs(img[:,:,0] - img[:,:,1]), abs(img[:,:,1] - img[:,:,2]), abs(img[:,:,2] - img[:,:,0])])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.10.4 ('pytorch-gpu')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "17cd5c528a3345b75540c61f907eece919c031d57a2ca1e5653325af249173c9"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

eval_bow.py

@@ -1,3 +1,8 @@
+# Approach 3: Local features
+# This script is used for calculating BOW features of Motion images
+# using a BOW vocabulary.
+# See train_bow.py for training.
+
 import argparse
 import os
 import numpy as np
@@ -12,6 +17,7 @@ def main():
     parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
     parser.add_argument("--clusters", type=int, help="Number of clusters / BOW vocabulary size", default=1024)
     parser.add_argument("--step_size", type=int, help="DSIFT keypoint step size. Smaller step size = more keypoints.", default=30)
+    parser.add_argument("--keypoint_size", type=int, help="DSIFT keypoint size. Should be >= step_size.", default=60)
 
     args = parser.parse_args()
 
@@ -21,9 +27,9 @@ def main():
 
     # Lapse DSIFT descriptors
 
-    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.clusters}.npy")
-    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.clusters}.npy")
-    eval_file = os.path.join(save_dir, f"bow_eval_{args.step_size}_{args.clusters}.csv")
+    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
+    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
+    eval_file = os.path.join(save_dir, f"bow_eval_{args.step_size}_{args.keypoint_size}_{args.clusters}.csv")
 
     if not os.path.isfile(dictionary_file):
         print(f"ERROR: BOW dictionary missing! ({dictionary_file})")
@@ -42,7 +48,7 @@ def main():
 
         print("Evaluating...")
         with open(eval_file, "a+") as f:
-            for filename, feat in generate_bow_features(list(session.generate_motion_images()), dictionary, kp_step=args.step_size):
+            for filename, feat in generate_bow_features(list(session.generate_motion_images()), dictionary, kp_step=args.step_size, kp_size=args.keypoint_size):
                 y = clf.decision_function(feat)[0]
                 f.write(f"{filename},{y}\n")
                 f.flush()

py/ImageUtils.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from PIL import Image
+import numpy as np
 import matplotlib.pyplot as plt
 
 def get_image_date(img_path: str) -> datetime:
@@ -37,4 +38,5 @@ def display_images(images: list, titles: list, colorbar=False, size=(8, 5), row_
     plt.tight_layout()
     plt.show()
 
-
+def is_daytime(img, threshold=50) -> bool:
+    return np.mean([abs(img[:,:,0] - img[:,:,1]), abs(img[:,:,1] - img[:,:,2]), abs(img[:,:,2] - img[:,:,0])]) > threshold

py/LocalFeatures.py

@@ -7,13 +7,14 @@ from tqdm import tqdm
 
 from py.Session import SessionImage
 
-def dense_keypoints(img, step=30):
+def dense_keypoints(img, step=30, size=60):
     """Generates a list of densely sampled keypoints on img. The keypoints are arranged tightly
     next to each other without spacing. The group of all keypoints is centered in the image.
 
     Args:
         img (_type_): Image to sample from. (only the shape is relevant)
-        step (int, optional): Vertical and horizontal step size between and size of keypoints. Defaults to 30.
+        step (int, optional): Vertical and horizontal step size between keypoints. Defaults to 30.
+        size (int, optional): Size of keypoints. Defaults to 60.
 
     Returns:
         list[cv.KeyPoint]: List of keypoints
@@ -21,16 +22,17 @@ def dense_keypoints(img, step=30):
     # calculate offset to center keypoints
     off = ((img.shape[0] % step) // 2, (img.shape[1] % step) // 2)
     border_dist = (step + 1) // 2
-    return [cv.KeyPoint(x, y, step) for y in range(border_dist + off[0], img.shape[0] - border_dist, step) 
+    return [cv.KeyPoint(x, y, size) for y in range(border_dist + off[0], img.shape[0] - border_dist, step) 
                                     for x in range(border_dist + off[1], img.shape[1] - border_dist, step)]
 
 
-def extract_descriptors(images: list[SessionImage], kp_step: int = 30):
+def extract_descriptors(images: list[SessionImage], kp_step: int = 30, kp_size: int = 60):
     """Extracts DSIFT descriptors from the provided images and returns them in a single array.
 
     Args:
         images (list[SessionImage]): List of images to read and compute descriptors from.
         kp_step (int, optional): Keypoint step size, see dense_keypoints. Defaults to 30.
+        kp_size (int, optional): Keypoint size, see dense_keypoints. Defaults to 60.
 
     Returns:
         np.array, shape=(len(images)*keypoints_per_image, 128): DSIFT descriptors.
@@ -40,7 +42,7 @@ def extract_descriptors(images: list[SessionImage], kp_step: int = 30):
     output_kp = False
     for image in tqdm(images):
         img = image.read_opencv(gray=True)
-        kp = dense_keypoints(img, kp_step)
+        kp = dense_keypoints(img, kp_step, kp_size)
         # output number of keypoints once
         if not output_kp:
             print(f"{len(kp)} keypoints per image.")
@@ -66,7 +68,7 @@ def generate_dictionary_from_descriptors(dscs, dictionary_size: int):
     dictionary = BOW.cluster()
     return dictionary
 
-def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int = 30):
+def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int = 30, kp_size: int = 60):
     """Calculates the BOW features for the provided images using dictionary.
     Yields a feature vector for every image.
 
@@ -74,6 +76,7 @@ def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int =
         images (list[SessionImage]): List of images to read and compute feature vectors from.
         dictionary (np.array, shape=(-1, 128)): BOW dictionary.
         kp_step (int, optional): Keypoint step size, see dense_keypoints. Must be identical to the step size used for vocabulary generation. Defaults to 30.
+        kp_size (int, optional): Keypoint size, see dense_keypoints. Must be identical to the size used for vocabulary generation. Defaults to 60.
 
     Yields:
         (str, np.array of shape=(dictionary.shape[0])): (filename, feature vector)
@@ -85,6 +88,6 @@ def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int =
     
     for image in tqdm(images):
         img = image.read_opencv(gray=True)
-        kp = dense_keypoints(img, kp_step)
+        kp = dense_keypoints(img, kp_step, kp_size)
         feat = bow_extractor.compute(img, kp)
         yield image.filename, feat

py/PlotUtils.py

@@ -1,7 +1,7 @@
 import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
 
-def plot_roc_curve(test_labels: list, test_df: list, title: str, figsize=(8, 8), savefile = None):
+def plot_roc_curve(test_labels: list, test_df: list, title: str, figsize=(8, 8), savefile = None, show: bool = True):
     fpr, tpr, thresholds = roc_curve(test_labels, test_df)
     auc_score = auc(fpr, tpr)
 
@@ -17,5 +17,13 @@ def plot_roc_curve(test_labels: list, test_df: list, title: str, figsize=(8, 8),
     if savefile is not None:
         plt.savefig(f"{savefile}.png", bbox_inches="tight")
         plt.savefig(f"{savefile}.pdf", bbox_inches="tight")
-    plt.show()
-    return fpr, tpr, thresholds, auc_score
+    if show:
+        plt.show()
+    return fpr, tpr, thresholds, auc_score
+
+def get_percentiles(fpr, tpr, thresholds, percentiles=[0.9, 0.95, 0.98, 0.99]):
+    for percentile in percentiles:
+        for i, tp in enumerate(tpr):
+            if tp >= percentile:
+                print(f"{percentile} percentile : TPR = {tp:.4f}, FPR = {fpr[i]:.4f} <-> TNR = {(1 - fpr[i]):.4f} @ thresh {thresholds[i]}")
+                break

results.ipynb

@@ -13,14 +13,17 @@
    "source": [
     "## Beaver_01\n",
     "\n",
-    "| Approach | Configuration | Best AUC |\n",
-    "| --- | --- | ---: |\n",
-    "| 1a - Basic Frame Differencing | - | 0.7335 |\n",
-    "| | $\\sigma=2$ | 0.8658 |\n",
-    "| | $\\sigma=4$ | 0.8747 |\n",
-    "| 2 - Background Estimation | No Lapse | 0.7524 |\n",
-    "| 3 - BOW | $k=2048, kp=30$ | 0.7741 |\n",
-    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 0.9209 |"
+    "| Approach | Configuration | Best AUC | TNR @TPR>0.9 | TNR @TPR>0.99 |\n",
+    "| --- | --- | ---: | ---: | ---: |\n",
+    "| 1a - Basic Frame Differencing | abs var | 0.7415 | | |\n",
+    "| | $\\sigma=2$, sq var | 0.8986 | | |\n",
+    "| | $\\sigma=4$, sq var | 0.9156 | | |\n",
+    "| 1b - Histogram Comparison | p-mean | 0.6707 | | |\n",
+    "| 2 - Background Estimation | no lapse, sq var | 0.7897 | | |\n",
+    "| | $\\sigma=2$, no lapse, sq var | 0.8735 | | |\n",
+    "| | $\\sigma=4$, no lapse, sq var | 0.8776 | | |\n",
+    "| 3 - BOW | $k=2048, kp=30$ | 0.7741 | 0.4976 | 0.0564 |\n",
+    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 0.9209 | | |"
    ]
   },
   {
@@ -29,12 +32,16 @@
    "source": [
     "\n",
     "## Marten_01\n",
+    "teilannotiert!\n",
     "\n",
-    "| Approach | Configuration | Best AUC |\n",
-    "| --- | --- | ---: |\n",
-    "| 2 - Background Estimation | No Lapse | 0.5832 |\n",
-    "| 3 - BOW | $k=1024, kp=30$ | 0.7099 |\n",
-    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 1.0000 |"
+    "| Approach | Configuration | Best AUC | TNR @TPR>0.9 | TNR @TPR>0.99 |\n",
+    "| --- | --- | ---: | ---: | ---: |\n",
+    "| 1a - Basic Frame Differencing | sq var | 0.9854 | | |\n",
+    "| | $\\sigma = 2$, sq var | 1.0000 | 1.0000 | 1.0000 |\n",
+    "| | $\\sigma = 4$, sq var | 1.0000 | 1.0000 | 1.0000 |\n",
+    "| 2 - Background Estimation | No Lapse | 0.5832 | | |\n",
+    "| 3 - BOW | $k=1024, kp=30$ | 0.7099 | | |\n",
+    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 1.0000 | | |"
    ]
   },
   {

train_bow.py

@@ -16,6 +16,7 @@ def main():
     parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
     parser.add_argument("--clusters", type=int, help="Number of clusters / BOW vocabulary size", default=1024)
     parser.add_argument("--step_size", type=int, help="DSIFT keypoint step size. Smaller step size = more keypoints.", default=30)
+    parser.add_argument("--keypoint_size", type=int, help="DSIFT keypoint size. Should be >= step_size.", default=60)
 
     args = parser.parse_args()
 
@@ -25,9 +26,9 @@ def main():
 
     # Lapse DSIFT descriptors
 
-    lapse_dscs_file = os.path.join(save_dir, f"lapse_dscs_{args.step_size}.npy")
-    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.clusters}.npy")
-    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.clusters}.npy")
+    lapse_dscs_file = os.path.join(save_dir, f"lapse_dscs_{args.step_size}_{args.keypoint_size}.npy")
+    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
+    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
 
     if os.path.isfile(lapse_dscs_file):
         if os.path.isfile(dictionary_file):
@@ -39,7 +40,7 @@ def main():
     else:
         # Step 1 - extract dense SIFT descriptors
         print("Extracting lapse descriptors...")
-        lapse_dscs = extract_descriptors(list(session.generate_lapse_images()), kp_step=args.step_size)
+        lapse_dscs = extract_descriptors(list(session.generate_lapse_images()), kp_step=args.step_size, kp_size=args.keypoint_size)
         os.makedirs(save_dir, exist_ok=True)
         np.save(lapse_dscs_file, lapse_dscs)
 
@@ -61,7 +62,7 @@ def main():
     else:
         # Step 3 - calculate training data (BOW features of Lapse images)
         print(f"Extracting BOW features from Lapse images...")
-        features = [feat for _, feat in generate_bow_features(list(session.generate_lapse_images()), dictionary, kp_step=args.step_size)]
+        features = [feat for _, feat in generate_bow_features(list(session.generate_lapse_images()), dictionary, kp_step=args.step_size, kp_size=args.keypoint_size)]
         np.save(train_feat_file, features)
     
     print("Complete!")