quick labeling, more plots

eval_bow.py

@@ -17,19 +17,22 @@ def main():
     parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
     parser.add_argument("--clusters", type=int, help="Number of clusters / BOW vocabulary size", default=1024)
     parser.add_argument("--step_size", type=int, help="DSIFT keypoint step size. Smaller step size = more keypoints.", default=30)
-    parser.add_argument("--keypoint_size", type=int, help="DSIFT keypoint size. Should be >= step_size.", default=60)
+    parser.add_argument("--keypoint_size", type=int, help="DSIFT keypoint size. Defaults to step_size.", default=-1)
+    parser.add_argument("--include_motion", action="store_true", help="Include motion images for training.")
 
     args = parser.parse_args()
+    if args.keypoint_size <= 0:
+        args.keypoint_size = args.step_size
+    print(f"Using keypoint size {args.keypoint_size} with step size {args.step_size}.")
 
     ds = Dataset(args.dataset_dir)
     session = ds.create_session(args.session_name)
     save_dir = f"./bow_train_NoBackup/{session.name}"
 
-    # Lapse DSIFT descriptors
-
-    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
-    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
-    eval_file = os.path.join(save_dir, f"bow_eval_{args.step_size}_{args.keypoint_size}_{args.clusters}.csv")
+    suffix = "_motion" if args.include_motion else ""
+    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.keypoint_size}_{args.clusters}{suffix}.npy")
+    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.keypoint_size}_{args.clusters}{suffix}.npy")
+    eval_file = os.path.join(save_dir, f"bow_eval_{args.step_size}_{args.keypoint_size}_{args.clusters}{suffix}.csv")
 
     if not os.path.isfile(dictionary_file):
         print(f"ERROR: BOW dictionary missing! ({dictionary_file})")

py/FileUtils.py

@@ -24,6 +24,7 @@ def list_jpegs_recursive(path: str) -> list:
     Returns:
         list: list of all jpeg files
     """
+    print(os.path.join(path, "**/*.jpg"))
     return [name for name in glob(os.path.join(path, "**/*.jpg"), recursive=True) if os.path.isfile(os.path.join(path, name))]
 
 def verify_expected_subfolders(session_path: str):

py/LocalFeatures.py

@@ -48,8 +48,8 @@ def extract_descriptors(images: list[SessionImage], kp_step: int = 30, kp_size:
             print(f"{len(kp)} keypoints per image.")
             output_kp = True
         kp, des = sift.compute(img, kp)
-        dscs.append(des)
-    return np.array(dscs)
+        dscs.extend(des)
+    return np.array(dscs).reshape(-1, 128)
 
 def generate_dictionary_from_descriptors(dscs, dictionary_size: int):
     """Clusters the given (D)SIFT descriptors using k-means.
@@ -62,10 +62,14 @@ def generate_dictionary_from_descriptors(dscs, dictionary_size: int):
     Returns:
         np.array, shape=(dictionary_size, 128): BOW dictionary.
     """
+    assert len(dscs.shape) == 2 and dscs.shape[1] == 128
+    assert dictionary_size > 0 and dictionary_size <= dscs.shape[0]
+
     BOW = cv.BOWKMeansTrainer(dictionary_size)
     for dsc in dscs:
         BOW.add(dsc)
     dictionary = BOW.cluster()
+    assert dictionary.shape == (dictionary_size, 128)
     return dictionary
 
 def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int = 30, kp_size: int = 60):
@@ -81,6 +85,9 @@ def generate_bow_features(images: list[SessionImage], dictionary, kp_step: int =
     Yields:
         (str, np.array of shape=(dictionary.shape[0])): (filename, feature vector)
     """
+    assert len(dictionary.shape) == 2 and dictionary.shape[1] == 128
+    assert kp_size > 0 and kp_step > 0
+
     flann = cv.FlannBasedMatcher({"algorithm": 0, "trees": 5}, {"checks": 50})
     sift = cv.SIFT_create()
     bow_extractor = cv.BOWImgDescriptorExtractor(sift, flann) # or cv.BFMatcher(cv.NORM_L2)

quick_label.py

@@ -0,0 +1,60 @@
+import cv2
+import argparse
+import os
+
+from py.Dataset import Dataset
+from py.FileUtils import list_jpegs_recursive
+
+def main():
+    parser = argparse.ArgumentParser(description="BOW train script")
+    parser.add_argument("dataset_dir", type=str, help="Directory of the dataset containing all session folders")
+    parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
+    parser.add_argument("--skip", type=int, help="Skip first n images", default=0)
+
+    args = parser.parse_args()
+
+    ds = Dataset(args.dataset_dir)
+    session = ds.create_session(args.session_name)
+    
+    skip = args.skip
+    if skip > 0:
+        print(f"Skipping the first {skip} images...")
+    normal = []
+    anomalous = []
+    motion_folder = session.get_motion_folder()
+    quit = False
+    print(list_jpegs_recursive(motion_folder), motion_folder)
+    for img_file in sorted(list_jpegs_recursive(motion_folder)):
+        if skip > 0:
+            skip -= 1
+            print(skip)
+            continue
+        img_nr = int(img_file[-9:-4])
+
+        print(f"Labeling img #{img_nr}... ", end="")
+        image = cv2.imread(os.path.join(motion_folder, img_file))
+        cv2.imshow("labeler", image)
+        while True:
+            key = cv2.waitKey(0)
+            if key == ord("1"):
+                print("normal")
+                normal.append(img_nr)
+            elif key == ord("2"):
+                print("anomalous")
+                anomalous.append(img_nr)
+            elif key == ord("x"):
+                quit = True
+            else:
+                continue
+            print(f"normal = {normal}")
+            print(f"anomalous = {anomalous}")
+            break
+        if quit:
+            break
+    cv2.destroyAllWindows()
+    print("Done.")
+    print(f"normal = {normal}")
+    print(f"anomalous = {anomalous}")
+
+if __name__ == "__main__":
+    main()

results.ipynb

@@ -13,17 +13,17 @@
    "source": [
     "## Beaver_01\n",
     "\n",
-    "| Approach | Configuration | Best AUC | TNR @TPR>0.9 | TNR @TPR>0.99 |\n",
+    "| Approach | Configuration | Best AUC | TNR @TPR $\\geq$ 0.9 | TNR @TPR $\\geq$ 0.99 |\n",
     "| --- | --- | ---: | ---: | ---: |\n",
-    "| 1a - Basic Frame Differencing | abs var | 0.7415 | | |\n",
-    "| | $\\sigma=2$, sq var | 0.8986 | | |\n",
-    "| | $\\sigma=4$, sq var | 0.9156 | | |\n",
+    "| 1a - Basic Frame Differencing | abs var | 0.7415 | 0.4865 | 0.2432 |\n",
+    "| | $\\sigma=2$, sq var | 0.8986 | 0.7162 | 0.5270 |\n",
+    "| | $\\sigma=4$, sq var | 0.9156 | 0.7973 | 0.5676 |\n",
     "| 1b - Histogram Comparison | p-mean | 0.6707 | | |\n",
-    "| 2 - Background Estimation | no lapse, sq var | 0.7897 | | |\n",
-    "| | $\\sigma=2$, no lapse, sq var | 0.8735 | | |\n",
-    "| | $\\sigma=4$, no lapse, sq var | 0.8776 | | |\n",
+    "| 2 - Background Estimation | no lapse, sq var | 0.7897 | 0.6622 | 0.2703 |\n",
+    "| | $\\sigma=2$, no lapse, sq var | 0.8735 | 0.7973 | 0.4865 |\n",
+    "| | $\\sigma=4$, no lapse, sq var | 0.8776 | 0.7838 | 0.4459 |\n",
     "| 3 - BOW | $k=2048, kp=30$ | 0.7741 | 0.4976 | 0.0564 |\n",
-    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 0.9209 | | |"
+    "| 4 - Autoencoder | Deep +Noise +Sparse KDE | 0.9209 | 0.8514 | 0.1216 |"
    ]
   },
   {
@@ -48,13 +48,39 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.10.4 ('pytorch-gpu')",
+   "language": "python",
+   "name": "python3"
+  },
   "language_info": {
-   "name": "python"
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
   },
-  "orig_nbformat": 4
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "17cd5c528a3345b75540c61f907eece919c031d57a2ca1e5653325af249173c9"
+   }
+  }
  },
  "nbformat": 4,
  "nbformat_minor": 2

train_bow.py

@@ -6,6 +6,8 @@
 import argparse
 import os
 import numpy as np
+from timeit import default_timer as timer
+from datetime import timedelta
 
 from py.Dataset import Dataset
 from py.LocalFeatures import extract_descriptors, generate_dictionary_from_descriptors, generate_bow_features
@@ -16,33 +18,63 @@ def main():
     parser.add_argument("session_name", type=str, help="Name of the session to use for Lapse images (e.g. marten_01)")
     parser.add_argument("--clusters", type=int, help="Number of clusters / BOW vocabulary size", default=1024)
     parser.add_argument("--step_size", type=int, help="DSIFT keypoint step size. Smaller step size = more keypoints.", default=30)
-    parser.add_argument("--keypoint_size", type=int, help="DSIFT keypoint size. Should be >= step_size.", default=60)
+    parser.add_argument("--keypoint_size", type=int, help="DSIFT keypoint size. Defaults to step_size.", default=-1)
+    parser.add_argument("--include_motion", action="store_true", help="Include motion images for training.")
 
     args = parser.parse_args()
+    if args.keypoint_size <= 0:
+        args.keypoint_size = args.step_size
+    print(f"Using keypoint size {args.keypoint_size} with step size {args.step_size}.")
 
     ds = Dataset(args.dataset_dir)
     session = ds.create_session(args.session_name)
     save_dir = f"./bow_train_NoBackup/{session.name}"
 
-    # Lapse DSIFT descriptors
-
+    suffix = "_motion" if args.include_motion else ""
     lapse_dscs_file = os.path.join(save_dir, f"lapse_dscs_{args.step_size}_{args.keypoint_size}.npy")
-    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
-    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.keypoint_size}_{args.clusters}.npy")
+    motion_dscs_file = os.path.join(save_dir, f"motion_dscs_{args.step_size}_{args.keypoint_size}.npy")
+    dictionary_file = os.path.join(save_dir, f"bow_dict_{args.step_size}_{args.keypoint_size}_{args.clusters}{suffix}.npy")
+    train_feat_file = os.path.join(save_dir, f"bow_train_{args.step_size}_{args.keypoint_size}_{args.clusters}{suffix}.npy")
+
+    # Lapse DSIFT descriptors
 
     if os.path.isfile(lapse_dscs_file):
         if os.path.isfile(dictionary_file):
             # if dictionary file already exists, we don't need the lapse descriptors
-            print(f"{lapse_dscs_file} already exists, skipping lapse descriptor extraction...")
+            print(f"{dictionary_file} already exists, skipping lapse descriptor extraction...")
         else:
-            print(f"{lapse_dscs_file} already exists, loading lapse descriptor from file...")
+            print(f"{lapse_dscs_file} already exists, loading lapse descriptors from file... ", end="")
             lapse_dscs = np.load(lapse_dscs_file)
+            assert lapse_dscs.shape[-1] == 128
+            lapse_dscs = lapse_dscs.reshape(-1, 128)
+            print(f"Loaded {len(lapse_dscs)} lapse descriptors!")
     else:
         # Step 1 - extract dense SIFT descriptors
         print("Extracting lapse descriptors...")
         lapse_dscs = extract_descriptors(list(session.generate_lapse_images()), kp_step=args.step_size, kp_size=args.keypoint_size)
         os.makedirs(save_dir, exist_ok=True)
         np.save(lapse_dscs_file, lapse_dscs)
+    
+    # Motion DSIFT descriptors
+    if args.include_motion:
+        if os.path.isfile(motion_dscs_file):
+            if os.path.isfile(dictionary_file):
+                # if dictionary file already exists, we don't need the descriptors
+                print(f"{dictionary_file} already exists, skipping motion descriptor extraction...")
+            else:
+                print(f"{motion_dscs_file} already exists, loading motion descriptors from file...", end="")
+                motion_dscs = np.load(motion_dscs_file)
+                assert motion_dscs.shape[-1] == 128
+                motion_dscs = motion_dscs.reshape(-1, 128)
+                print(f"Loaded {len(motion_dscs)} motion descriptors!")
+                lapse_dscs = np.concatenate([lapse_dscs, motion_dscs])
+        else:
+            # Step 1b - extract dense SIFT descriptors from motion images
+            print("Extracting motion descriptors...")
+            motion_dscs = extract_descriptors(list(session.generate_motion_images()), kp_step=args.step_size, kp_size=args.keypoint_size)
+            os.makedirs(save_dir, exist_ok=True)
+            np.save(motion_dscs_file, motion_dscs)
+            lapse_dscs = np.concatenate([lapse_dscs, motion_dscs])
 
     # BOW dictionary
 
@@ -51,8 +83,12 @@ def main():
         dictionary = np.load(dictionary_file)
     else:
         # Step 2 - create BOW dictionary from Lapse SIFT descriptors
-        print(f"Creating BOW vocabulary with {args.clusters} clusters...")
+        print(f"Creating BOW vocabulary with {args.clusters} clusters from {len(lapse_dscs)} descriptors...")
+        start_time = timer()
         dictionary = generate_dictionary_from_descriptors(lapse_dscs, args.clusters)
+        end_time = timer()
+        delta_time = timedelta(seconds=end_time-start_time)
+        print(f"Clustering took {delta_time}.")
         np.save(dictionary_file, dictionary)
     
     # Extract Lapse BOW features using vocabulary (train data)