Results + ROC Curves

eval_bow.py

@@ -11,6 +11,7 @@ def main():
     parser.add_argument("dataset_dir", type=str, help="Directory of the dataset containing all session folders")
     parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
     parser.add_argument("--clusters", type=int, help="Number of clusters / BOW vocabulary size", default=1024)
+    parser.add_argument("--step_size", type=int, help="DSIFT keypoint step size. Smaller step size = more keypoints.", default=30)
 
     args = parser.parse_args()
 
@@ -20,9 +21,9 @@ def main():
 
     # Lapse DSIFT descriptors
 
-    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.clusters}.npy")
-    train_feat_file = os.path.join(save_dir, f"bow_train_{args.clusters}.npy")
-    eval_file = os.path.join(save_dir, f"bow_eval_{args.clusters}.csv")
+    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.clusters}.npy")
+    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.clusters}.npy")
+    eval_file = os.path.join(save_dir, f"bow_eval_{args.step_size}_{args.clusters}.csv")
 
     if not os.path.isfile(dictionary_file):
         print(f"ERROR: BOW dictionary missing! ({dictionary_file})")
@@ -41,7 +42,7 @@ def main():
 
         print("Evaluating...")
         with open(eval_file, "a+") as f:
-            for filename, feat in generate_bow_features(list(session.generate_motion_images()), dictionary):
+            for filename, feat in generate_bow_features(list(session.generate_motion_images()), dictionary, kp_step=args.step_size):
                 y = clf.decision_function(feat)[0]
                 f.write(f"{filename},{y}\n")
                 f.flush()

py/LocalFeatures.py

@@ -7,36 +7,44 @@ from tqdm import tqdm
 
 from py.Session import SessionImage
 
-def dense_keypoints(img, step=30, off=(15, 12)):
-    """Generates a list of densely sampled keypoints on img.
+def dense_keypoints(img, step=30):
+    """Generates a list of densely sampled keypoints on img. The keypoints are arranged tightly
+    next to each other without spacing. The group of all keypoints is centered in the image.
 
     Args:
         img (_type_): Image to sample from. (only the shape is relevant)
         step (int, optional): Vertical and horizontal step size between and size of keypoints. Defaults to 30.
-        off (tuple, optional): y and x offset of the first keypoint in the grid. Defaults to (15, 12).
 
     Returns:
         list[cv.KeyPoint]: List of keypoints
     """
+    # calculate offset to center keypoints
+    off = ((img.shape[0] % step) // 2, (img.shape[1] % step) // 2)
     border_dist = (step + 1) // 2
     return [cv.KeyPoint(x, y, step) for y in range(border_dist + off[0], img.shape[0] - border_dist, step) 
                                     for x in range(border_dist + off[1], img.shape[1] - border_dist, step)]
 
 
-def extract_descriptors(images: list[SessionImage]):
+def extract_descriptors(images: list[SessionImage], kp_step: int = 30):
     """Extracts DSIFT descriptors from the provided images and returns them in a single array.
 
     Args:
         images (list[SessionImage]): List of images to read and compute descriptors from.
+        kp_step (int, optional): Keypoint step size, see dense_keypoints. Defaults to 30.
 
     Returns:
         np.array, shape=(len(images)*keypoints_per_image, 128): DSIFT descriptors.
     """
     sift = cv.SIFT_create()
     dscs = []
+    output_kp = False
     for image in tqdm(images):
         img = image.read_opencv(gray=True)
-        kp = dense_keypoints(img)
+        kp = dense_keypoints(img, kp_step)
+        # output number of keypoints once
+        if not output_kp:
+            print(f"{len(kp)} keypoints per image.")
+            output_kp = True
         kp, des = sift.compute(img, kp)
         dscs.append(des)
     return np.array(dscs)
@@ -58,13 +66,14 @@ def generate_dictionary_from_descriptors(dscs, dictionary_size: int):
     dictionary = BOW.cluster()
     return dictionary
 
-def generate_bow_features(images: list[SessionImage], dictionary):
+def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int = 30):
     """Calculates the BOW features for the provided images using dictionary.
     Yields a feature vector for every image.
 
     Args:
         images (list[SessionImage]): List of images to read and compute feature vectors from.
         dictionary (np.array, shape=(-1, 128)): BOW dictionary.
+        kp_step (int, optional): Keypoint step size, see dense_keypoints. Must be identical to the step size used for vocabulary generation. Defaults to 30.
 
     Yields:
         (str, np.array of shape=(dictionary.shape[0])): (filename, feature vector)
@@ -76,6 +85,6 @@ def generate_bow_features(images: list[SessionImage], dictionary):
     
     for image in tqdm(images):
         img = image.read_opencv(gray=True)
-        kp = dense_keypoints(img)
+        kp = dense_keypoints(img, kp_step)
         feat = bow_extractor.compute(img, kp)
         yield image.filename, feat

py/PlotUtils.py

@@ -0,0 +1,21 @@
+import matplotlib.pyplot as plt
+from sklearn.metrics import roc_curve, auc
+
+def plot_roc_curve(test_labels: list, test_df: list, title: str, figsize=(8, 8), savefile = None):
+    fpr, tpr, thresholds = roc_curve(test_labels, test_df)
+    auc_score = auc(fpr, tpr)
+
+    plt.figure(figsize=figsize)
+    plt.plot(fpr, tpr, lw=1, label="ROC Curve")
+    plt.plot([0, 1], [0, 1], color="lime", linestyle="--")
+    plt.xlim([0.0, 1.0])
+    plt.ylim([0.0, 1.05])
+    plt.xlabel("FPR")
+    plt.ylabel("TPR")
+    plt.title(f"{title} (AUC = {auc_score})")
+    plt.legend(loc="lower right")
+    if savefile is not None:
+        plt.savefig(f"{savefile}.png", bbox_inches="tight")
+        plt.savefig(f"{savefile}.pdf", bbox_inches="tight")
+    plt.show()
+    return fpr, tpr, thresholds, auc_score

py/Session.py

@@ -197,9 +197,9 @@ class Session:
             date = random.choice(list(self.motion_map.keys()))
             filenames = self.motion_map.get(date, [])
             imgs = [MotionImage(self, filename, date) for filename in filenames]
-        # include all images within +- 10 min
+        # include all images within +- 5 min
         for other_date in self.motion_map.keys():
-            if date != other_date and abs((date - other_date).total_seconds()) <= 60 * 10:
+            if date != other_date and abs((date - other_date).total_seconds()) <= 60 * 5:
                 filenames = self.motion_map.get(other_date, [])
                 imgs += [MotionImage(self, filename, other_date) for filename in filenames]
         return imgs

results.ipynb

@@ -0,0 +1,54 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Results"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Beaver_01\n",
+    "\n",
+    "| Approach | Configuration | Best AUC |\n",
+    "| --- | --- | ---: |\n",
+    "| 1a - Basic Frame Differencing | - | 0.7335 |\n",
+    "| | $\\sigma=2$ | 0.8658 |\n",
+    "| | $\\sigma=4$ | 0.8747 |\n",
+    "| 2 - Background Estimation | No Lapse | 0.7524 |\n",
+    "| 3 - BOW | $k=2048, kp=30$ | 0.7741 |\n",
+    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 0.9209 |"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "## Marten_01\n",
+    "\n",
+    "| Approach | Configuration | Best AUC |\n",
+    "| --- | --- | ---: |\n",
+    "| 2 - Background Estimation | No Lapse | 0.5832 |\n",
+    "| 3 - BOW | $k=1024, kp=30$ | 0.7099 |\n",
+    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 1.0000 |"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

train_bow.py

@@ -15,6 +15,7 @@ def main():
     parser.add_argument("dataset_dir", type=str, help="Directory of the dataset containing all session folders")
     parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
     parser.add_argument("--clusters", type=int, help="Number of clusters / BOW vocabulary size", default=1024)
+    parser.add_argument("--step_size", type=int, help="DSIFT keypoint step size. Smaller step size = more keypoints.", default=30)
 
     args = parser.parse_args()
 
@@ -24,9 +25,9 @@ def main():
 
     # Lapse DSIFT descriptors
 
-    lapse_dscs_file = os.path.join(save_dir, "lapse_dscs.npy")
-    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.clusters}.npy")
-    train_feat_file = os.path.join(save_dir, f"bow_train_{args.clusters}.npy")
+    lapse_dscs_file = os.path.join(save_dir, f"lapse_dscs_{args.step_size}.npy")
+    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.clusters}.npy")
+    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.clusters}.npy")
 
     if os.path.isfile(lapse_dscs_file):
         if os.path.isfile(dictionary_file):
@@ -38,7 +39,7 @@ def main():
     else:
         # Step 1 - extract dense SIFT descriptors
         print("Extracting lapse descriptors...")
-        lapse_dscs = extract_descriptors(list(session.generate_lapse_images()))
+        lapse_dscs = extract_descriptors(list(session.generate_lapse_images()), kp_step=args.step_size)
         os.makedirs(save_dir, exist_ok=True)
         np.save(lapse_dscs_file, lapse_dscs)
 
@@ -60,7 +61,7 @@ def main():
     else:
         # Step 3 - calculate training data (BOW features of Lapse images)
         print(f"Extracting BOW features from Lapse images...")
-        features = list(generate_bow_features(list(session.generate_lapse_images()), dictionary))
+        features = [feat for _, feat in generate_bow_features(list(session.generate_lapse_images()), dictionary, kp_step=args.step_size)]
         np.save(train_feat_file, features)
     
     print("Complete!")