session scan + results

analyze_dataset.ipynb

@@ -16,33 +16,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2\n",
-    "\n",
-    "from tqdm.notebook import tqdm\n",
-    "import os\n",
-    "\n",
-    "from py.Dataset import Dataset\n",
-    "from py.DatasetStatistics import DatasetStatistics\n",
-    "from py.FileUtils import list_folders, list_jpegs_recursive"
+    "stats.view()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -61,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -111,14 +94,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Saved to dataset_stats.npy.\n"
+      "Loaded from dataset_stats.npy.\n"
      ]
     }
    ],
@@ -137,7 +120,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -440,7 +423,7 @@
        "Z_Total      31579  167918  4390  203887"
       ]
      },
-     "execution_count": 32,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -449,6 +432,30 @@
     "stats.view()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Total     1.000000\n",
+       "Full      0.021532\n",
+       "Lapse     0.154885\n",
+       "Motion    0.823584\n",
+       "Name: Z_Total, dtype: float64"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats.df.iloc[-1] / 203887"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

py/Session.py

@@ -1,4 +1,6 @@
+from datetime import datetime
 import pickle
+import subprocess
 from warnings import warn
 import os
 from tqdm import tqdm
@@ -36,9 +38,9 @@ class Session:
             with open(lapse_dates_file, "rb") as handle:
                 self.lapse_dates = pickle.load(handle)
             with open(motion_dates_file, "rb") as handle:
-                self.motion_dates_file = pickle.load(handle)
+                self.motion_dates = pickle.load(handle)
             with open(lapse_map_file, "rb") as handle:
-                self.lapse_map_file = pickle.load(handle)
+                self.lapse_map = pickle.load(handle)
             self.scanned = True
             print("Loaded scans.")
         else:
@@ -47,6 +49,7 @@ class Session:
             self.scanned = False
     
     def save_scans(self):
+        os.makedirs(os.path.join("session_scans", self.name), exist_ok=True)
         lapse_dates_file = os.path.join("session_scans", self.name, "lapse_dates.pickle")
         motion_dates_file = os.path.join("session_scans", self.name, "motion_dates.pickle")
         lapse_map_file = os.path.join("session_scans", self.name, "lapse_map.pickle")
@@ -66,13 +69,15 @@ class Session:
         # Scan motion dates
         print("Scanning motion dates...")
         self.motion_dates = {}
-        for motionFile in tqdm(list_jpegs_recursive(os.path.join(self.folder, "Motion"))):
-            self.motion_dates[os.path.basename(motionFile)] = get_image_date(motionFile)
+        motion_folder = os.path.join(self.folder, "Motion")
+        for file in tqdm(list_jpegs_recursive(motion_folder)):
+            self.motion_dates[os.path.relpath(file, motion_folder)] = get_image_date(file)
         # Scan lapse dates
         print("Scanning lapse dates...")
         self.lapse_dates = {}
-        for motionFile in tqdm(list_jpegs_recursive(os.path.join(self.folder, "Motion"))):
-            self.motion_dates[os.path.basename(motionFile)] = get_image_date(motionFile)
+        lapse_folder = os.path.join(self.folder, "Lapse")
+        for file in tqdm(list_jpegs_recursive(lapse_folder)):
+            self.lapse_dates[os.path.relpath(file, lapse_folder)] = get_image_date(file)
         # Create lapse map
         print("Creating lapse map...")
         self.lapse_map = {}
@@ -81,7 +86,43 @@ class Session:
                 self.lapse_map[date].append(file)
             else:
                 self.lapse_map[date] = [file]
+        self.scanned = True
         # Auto save
         if auto_save:
             print("Saving...")
-            self.save_scans()
+            self.save_scans()
+    
+    def check_lapse_duplicates(self) -> bool:
+        total = 0
+        total_duplicates = 0
+        total_multiples = 0
+        deviant_duplicates = []
+        for date, files in tqdm(self.lapse_map.items()):
+            total += 1
+            if len(files) > 1:
+                total_duplicates += 1
+                file_size = -1
+                for f in files:
+                    f_size = os.path.getsize(os.path.join(self.folder, "Lapse", f))
+                    if file_size == -1:
+                        file_size = f_size
+                    elif f_size != file_size:
+                        deviant_duplicates.append(date)
+                        break
+                if len(files) > 2:
+                    total_multiples += 1
+        deviant_duplicates.sort()
+        print(f"* {total} lapse dates")
+        print(f"* {total_duplicates} duplicates")
+        print(f"* {total_multiples} multiples (more than two files per date)")
+        print(f"* {len(deviant_duplicates)} deviant duplicates: {deviant_duplicates}")
+        return total, total_duplicates, total_multiples, deviant_duplicates
+    
+    def open_images_for_date(self, date: datetime):
+        img_names = self.lapse_map.get(date, [])
+        if len(img_names) == 0:
+            warn("No images for this date!")
+        for i, img_name in enumerate(img_names):
+            full_path = os.path.join(self.folder, "Lapse", img_name)
+            print(f"#{i+1} {full_path}")
+            subprocess.call(("xdg-open", full_path))

scan_sessions.txt

@@ -0,0 +1,32 @@
+| Beaver_01 | 1734 | 0 | 0 | 0 |
+| Marten_01 | 1733 | 729 | 0 | 108 |
+| Raccoon_01 | 850 | 0 | 0 | 0 |
+| Reddeer_01 | 1628 | 0 | 0 | 0 |
+| Roedeer_01 | 1353 | 27 | 0 | 0 |
+| Wildboar_01 | 1732 | 0 | 0 | 0 |
+| Badger_02 | 1728 | 0 | 0 | 0 |
+| Beaver_02 | 1727 | 0 | 0 | 0 |
+| Fox_02 | 957 | 0 | 0 | 0 |
+| Marten_02 | 1726 | 0 | 0 | 0 |
+| Raccoon_02 | 1639 | 0 | 0 | 0 |
+| Rat_02 | 578 | 48 | 0 | 0 |
+| Reddeer_02 | 1635 | 0 | 0 | 0 |
+| Roedeer_02 | 888 | 0 | 0 | 0 |
+| Badger_03 | 46 | 0 | 0 | 0 |
+| Fox_03 | 34 | 1 | 1 | 1 |
+| Raccoon_03 | 35 | 0 | 0 | 0 |
+| Reddeer_03 | 1027 | 0 | 0 | 0 |
+| Wildboar_03 | 46 | 0 | 0 | 0 |
+| Badger_04 | 56 | 0 | 0 | 0 |
+| Rat_04 | 39 | 0 | 0 | 0 |
+| Reddeer_04 | 46 | 0 | 0 | 0 |
+| Wildboar_04 | 39 | 0 | 0 | 0 |
+| Badger_05 | 1105 | 69 | 0 | 1 |
+| Beaver_05 | 1105 | 216 | 0 | 0 |
+| Ermine_05 | 867 | 0 | 0 | 0 |
+| Fox_05 | 1083 | 0 | 0 | 0 |
+| Marten_05 | 888 | 2 | 0 | 2 |
+| Raccoon_05 | 1088 | 1 | 0 | 1 |
+| Reddeer_05 | 1078 | 0 | 0 | 0 |
+| Roedeer_05 | 1090 | 0 | 0 | 0 |
+| Wildboar_05 | 902 | 1 | 0 | 1 |