moved old annotations. updated bbox and parts mixins

cvdatasets/annotation/base.py

@@ -7,6 +7,8 @@ from collections import defaultdict
 from pathlib import Path
 from typing import Tuple
 
+from cvdatasets.annotation import mixins
+from cvdatasets.annotation.files import AnnotationFiles
 from cvdatasets.dataset import Dataset
 from cvdatasets.utils import feature_file_name
 from cvdatasets.utils import pretty_print_dict
@@ -16,10 +18,11 @@ from cvdatasets.utils.decorators import only_with_info
 
 class BaseAnnotations(abc.ABC):
 
-	def __init__(self, *, root_or_infofile, dataset_key=None, images_folder="images", **kwargs):
+	def __init__(self, *, root_or_infofile, dataset_key=None, images_folder="images", load_strict=True, **kwargs):
 
 		self.dataset_key = dataset_key
 		self.images_folder = images_folder
+		self.load_strict = load_strict
 
 		root_or_infofile = Path(root_or_infofile)
 		if root_or_infofile.is_dir():
@@ -75,9 +78,6 @@ class BaseAnnotations(abc.ABC):
 	def label(self, uuid) -> int:
 		return self.labels[self.uuid_to_idx[uuid]].copy()
 
-	def parts(self, uuid) -> object:
-		return None
-
 	def bounding_box(self, uuid) -> object:
 		return None
 
@@ -142,21 +142,24 @@ class BaseAnnotations(abc.ABC):
 		return new_kwargs
 
 	@abc.abstractmethod
-	def read_annotation_files(self):
-		raise NotImplementedError
+	def read_annotation_files(self) -> AnnotationFiles:
+		return AnnotationFiles(root=self.root, load_strict=self.load_strict)
 
 	@abc.abstractmethod
 	def _parse_uuids(self):
-		raise NotImplementedError
+		pass
 
 	@abc.abstractmethod
 	def _parse_labels(self):
-		raise NotImplementedError
+		pass
 
 	@abc.abstractmethod
 	def _parse_split(self):
-		raise NotImplementedError
+		pass
+
 
+class Annotations(mixins.BBoxMixin, mixins.PartsMixin, mixins.FeaturesMixin, BaseAnnotations):
+	pass
 
 class _BaseAnnotations(abc.ABC):
 

cvdatasets/annotation/files.py

@@ -21,13 +21,15 @@ class AnnotationFiles(object):
 
 		return fpath, opts
 
-
 	def __init__(self, *files, root=".", load_strict=False, **named_files):
 		super(AnnotationFiles, self).__init__()
 		self.load_strict = load_strict
 		self.root = Path(root)
 		self._files = []
 
+		self.load_files(*files, **named_files)
+
+	def load_files(self, *files, **named_files):
 		for fpath in files:
 			fpath, opts = self._parse_opts(fpath)
 			self.add_file_content(fpath, *opts)

cvdatasets/annotation/mixins/bbox_mixin.py

@@ -4,31 +4,46 @@ import numpy as np
 
 class BBoxMixin(abc.ABC):
 
+	dtype = np.dtype([
+		("x", np.int32),
+		("y", np.int32),
+		("w", np.int32),
+		("h", np.int32),
+	])
+
+	def read_annotation_files(self):
+		files = super(BBoxMixin, self).read_annotation_files()
+		files.load_files(
+			bounding_boxes=("bounding_boxes.txt", True),
+		)
+		return files
+
 	@property
 	def has_bounding_boxes(self):
-		return hasattr(self, "_bounding_boxes") and self._bounding_boxes is not None
+		return self.files.bounding_boxes is not None
 
-	def load(self):
-		super(BBoxMixin, self).load()
+	def parse_annotations(self):
+		super(BBoxMixin, self).parse_annotations()
 
 		if self.has_bounding_boxes:
-			self._load_bounding_boxes()
+			self._parse_bounding_boxes()
 
-	def _load_bounding_boxes(self):
-		logging.debug("Loading bounding box annotations")
-		assert self._bounding_boxes is not None, "Bouding boxes were not loaded!"
+	def _parse_bounding_boxes(self):
+		logging.debug("Parsing bounding box annotations")
+		assert self.has_bounding_boxes, \
+			"Bouding boxes were not loaded!"
 
 		uuid_to_bbox = {}
-		for content in [i.split() for i in self._bounding_boxes]:
+		for content in [i.split() for i in self.files.bounding_boxes]:
 			uuid, bbox = content[0], content[1:]
 			uuid_to_bbox[uuid] = [float(i) for i in bbox]
 
 		self.bounding_boxes = np.array(
 			[tuple(uuid_to_bbox[uuid]) for uuid in self.uuids],
-			dtype=self.meta.bounding_box_dtype)
+			dtype=self.dtype)
 
 	def bounding_box(self, uuid):
 		if self.has_bounding_boxes:
 			return self.bounding_boxes[self.uuid_to_idx[uuid]].copy()
 
-		return np.array((0,0, 1,1), dtype=self.meta.bounding_box_dtype)
+		return np.array((0,0, 1,1), dtype=self.dtype)

cvdatasets/annotation/mixins/features_mixin.py

@@ -1,3 +1,4 @@
+import abc
 
-class FeaturesMixin(object):
+class FeaturesMixin(abc.ABC):
 	pass

cvdatasets/annotation/mixins/parts_mixin.py

@@ -15,6 +15,15 @@ class PartsMixin(abc.ABC):
 
 		super(PartsMixin, self).__init__(**kwargs)
 
+	def read_annotation_files(self):
+
+		files = super(PartsMixin, self).read_annotation_files()
+		files.load_files(
+			part_locs=("parts/part_locs.txt", True),
+			part_names=("parts/parts.txt", True),
+		)
+		return files
+
 	@property
 	@only_with_info
 	def dataset_info(self):
@@ -38,24 +47,25 @@ class PartsMixin(abc.ABC):
 
 	@property
 	def has_parts(self):
-		return hasattr(self, "_part_locs") and self._part_locs is not None
+		return self.files.part_locs is not None
 
 	@property
 	def has_part_names(self):
-		return hasattr(self, "_part_names") and self._part_names is not None
+		return self.files.part_names is not None
 
-	def load(self):
-		super(PartsMixin, self).load()
+	def parse_annotations(self):
+		super(PartsMixin, self).parse_annotations()
 
 		if self.has_parts:
-			self._load_parts()
+			self._parse_parts()
 
-	def _load_parts(self):
-		logging.debug("Loading part annotations")
-		assert self.has_parts, "Part locations were not loaded!"
+	def _parse_parts(self):
+		logging.debug("Parsing part annotations")
+		assert self.has_parts, \
+			"Part locations were not loaded!"
 		# this part is quite slow... TODO: some runtime improvements?
 		uuid_to_parts = defaultdict(list)
-		for content in [i.split() for i in self._part_locs]:
+		for content in [i.split() for i in self.files.part_locs]:
 			uuid = content[0]
 			# assert uuid in self.uuids, \
 			# 	"Could not find UUID \"\" from part annotations in image annotations!".format(uuid)
@@ -66,13 +76,13 @@ class PartsMixin(abc.ABC):
 			uuid_to_parts[uuid] for uuid in self.uuids]).astype(int)
 
 		if self.has_part_names:
-			self._load_part_names()
+			self._parse_part_names()
 
-	def _load_part_names(self):
+	def _parse_part_names(self):
 		self.part_names.clear()
 		self.part_name_list.clear()
 
-		for line in self._part_names:
+		for line in self.files.part_names:
 			part_idx, _, name = line.partition(" ")
 			self.part_names[int(part_idx)] = name
 			self.part_name_list.append(name)

cvdatasets/annotation/types/__init__.py

@@ -6,6 +6,7 @@ from cvargparse.utils import BaseChoiceType
 from cvargparse.utils.enumerations import MetaBaseType
 
 class AnnotationMetaType(MetaBaseType):
+
 	def __getitem__(cls, key):
 		res = super(AnnotationMetaType, cls).__getitem__(key)
 		res.value.name = key
@@ -29,10 +30,10 @@ class AnnotationType(BaseChoiceType, metaclass=AnnotationMetaType):
 			],
 
 			cls.FILE_LIST : [
-				"CUB200", "CUB200_2FOLD", "CUB200_GOOGLE", "CUB200_GOOGLE_SEM"
+				"CUB200", "CUB200_2FOLD", "CUB200_GOOGLE", "CUB200_GOOGLE_SEM",
 				"NAB", "BIRDSNAP",
-				"CARS", "DOGS", "FLOWERS"
-				"HED", "TIGERS", "TIGERS_TEST"
+				"CARS", "DOGS", "FLOWERS",
+				"HED", "TIGERS", "TIGERS_TEST",
 
 			],
 
@@ -63,4 +64,4 @@ class AnnotationType(BaseChoiceType, metaclass=AnnotationMetaType):
 		return choices
 
 if __name__ == '__main__':
-	print(AnnotationType.as_choices)
+	print(AnnotationType.as_choices().keys())

cvdatasets/annotation/types/file_list.py

@@ -1,16 +1,14 @@
 import numpy as np
 
-from cvdatasets.annotation.base import BaseAnnotations
+from cvdatasets.annotation.base import Annotations
 from cvdatasets.annotation.files import AnnotationFiles
 
-class FileListAnnotations(BaseAnnotations):
+class FileListAnnotations(Annotations):
 
 	def read_annotation_files(self) -> AnnotationFiles:
-		return AnnotationFiles(
-			"images.txt", "labels.txt", "tr_ID.txt",
-			root=self.root,
-			load_strict=True,
-		)
+		files = super(FileListAnnotations, self).read_annotation_files()
+		files.load_files("images.txt", "labels.txt", "tr_ID.txt")
+		return files
 
 	def _parse_uuids(self) -> None:
 		assert self.files.images is not None, \

cvdatasets/annotation/types/folder_annotations.py

@@ -1,9 +1,9 @@
 import numpy as np
 
-from cvdatasets.annotation.base import BaseAnnotations
+from cvdatasets.annotation.base import Annotations
 from cvdatasets.annotation.files import AnnotationFiles
 
-class FolderAnnotations(BaseAnnotations):
+class FolderAnnotations(Annotations):
 
 	def read_annotation_files(self) -> AnnotationFiles:
 		return AnnotationFiles(

cvdatasets/annotation/types/json_annotations.py

@@ -3,7 +3,7 @@ import hashlib
 import logging
 import numpy as np
 
-from cvdatasets.annotation.base import BaseAnnotations
+from cvdatasets.annotation.base import Annotations
 from cvdatasets.annotation.files import AnnotationFiles
 
 def _uuid_check(uuids):
@@ -12,7 +12,7 @@ def _uuid_check(uuids):
 def _uuid_entry(im_info):
 	return hashlib.md5(im_info["file_name"].encode()).hexdigest()
 
-class JSONAnnotations(BaseAnnotations):
+class JSONAnnotations(Annotations):
 
 	def read_annotation_files(self) -> AnnotationFiles:
 		return AnnotationFiles(

scripts/info_files/info.yml

@@ -53,63 +53,69 @@ DATASETS:
   IMAGENET:         &inet
     folder: ImageNet
     annotations: "BJOERN"
+    annotation_type: FOLDER
     n_classes: 1000
 
   IMAGENET_TOP_INAT20: &inet_top_inat20
     <<: *inet
     annotations: "TOP_INAT20"
+    annotation_type: FOLDER
     n_classes: 44
 
   CUB200:         &cub200
     folder: birds/cub200
     annotations: "ORIGINAL"
+    annotation_type: FILE_LIST
     n_classes: 200
 
   CUB200_2FOLD:   &cub200_2fold
+    <<: *cub200
     folder: birds/cub200_2fold
-    annotations: "ORIGINAL"
-    n_classes: 200
 
   CUB200_GOOGLE: &cub200_google
+    <<: *cub200
     folder: birds/cub200_google_images
-    annotations: "ORIGINAL"
-    n_classes: 200
 
   CUB200_GOOGLE_SEM: &cub200_google_sem
+    <<: *cub200
     folder: birds/cub200_google_images
     annotations: "sem_selected"
-    n_classes: 200
-
+    annotation_type: FILE_LIST
 
   NAB:         &nabirds
     folder: birds/nabirds
     annotations: "ORIGINAL"
+    annotation_type: FILE_LIST
     n_classes: 555
 
   BIRDSNAP:         &birdsnap
     folder: birds/birdsnap
     annotations: "RESIZED"
+    annotation_type: FILE_LIST
     n_classes: 500
 
-
   CARS:         &cars
     folder: cars
     annotations: "ORIGINAL"
+    annotation_type: FILE_LIST
     n_classes: 196
 
   DOGS:         &dogs
     folder: dogs
     annotations: "ORIGINAL"
+    annotation_type: FILE_LIST
     n_classes: 120
 
   FLOWERS:         &flowers
     folder: flowers
     annotations: "flowers102"
+    annotation_type: FILE_LIST
     n_classes: 102
 
   INAT20:         &inat20
     folder: inat
     annotations: "2020/PLAIN"
+    annotation_type: JSON
     n_classes: 200
 
   INAT20_IN_CLASS:         &inat20_in_class
@@ -139,13 +145,9 @@ DATASETS:
   INAT19:         &inat19
     folder: inat
     annotations: "2019"
+    annotation_type: JSON
     n_classes: 1010
 
-  INAT18:         &inat18
-    folder: inat
-    annotations: "2018"
-    n_classes: 8142
-
   INAT19_MINI:    &inat19_mini
     <<: *inat19
     annotations: "2019_small"
@@ -154,20 +156,28 @@ DATASETS:
     <<: *inat19
     annotations: "2019_test"
 
+  INAT18:         &inat18
+    folder: inat
+    annotations: "2018"
+    annotation_type: JSON
+    n_classes: 8142
 
   HED:         &hed
     folder: medical/HED
     annotations: "patches224x224"
+    annotation_type: FILE_LIST
     n_classes: 2
 
   TIGERS:         &tigers
     folder: tigers
     annotations: "reid/train"
+    annotation_type: FILE_LIST
     n_classes: 107
 
   TIGERS_TEST:         &tigers_test
     folder: tigers
     annotations: "reid/test"
+    annotation_type: FILE_LIST
     n_classes: 107
 
 ############ Existing Part Annotations and Part Features
@@ -205,24 +215,28 @@ PART_TYPES:
       - 0.45
 
   L1_pred:        &parts_l1p
+    annotations: L1_pred
     feature_suffix: _5parts_L1_pred
     rescale_size: !!int 299
     scales:
       - 0.31
 
   L1_full:        &parts_l1f
+    annotations: L1_full
     feature_suffix: _5parts_L1_full
     rescale_size: !!int 299
     scales:
       - 0.31
 
   NTS:        &parts_nts
+    annotations: NTS
     feature_suffix: _7parts_nts
     rescale_size: !!int 448
     scales:
       - 0.31
 
   NTS2:        &parts_nts2
+    annotations: NTS2
     feature_suffix: _5parts_nts
     rescale_size: !!int 448
     scales:
@@ -347,13 +361,11 @@ PARTS:
   CUB200_2FOLD_L1_pred:
     <<: *cub200_2fold
     <<: *parts_l1p
-    annotations: L1_pred
     rescale_size: !!int 427
 
   CUB200_2FOLD_L1_full:
     <<: *cub200_2fold
     <<: *parts_l1f
-    annotations: L1_full
     rescale_size: !!int 427
 
   ####################################
@@ -381,12 +393,10 @@ PARTS:
   CUB200_NTS:
     <<: *cub200
     <<: *parts_nts
-    annotations: NTS
 
   CUB200_NTS2:
     <<: *cub200
     <<: *parts_nts2
-    annotations: NTS2
 
   CUB200_L1_pred:
     <<: *cub200
@@ -440,13 +450,11 @@ PARTS:
   NAB_L1_pred:
     <<: *nabirds
     <<: *parts_l1p
-    annotations: L1_pred
     rescale_size: !!int 427
 
   NAB_L1_full:
     <<: *nabirds
     <<: *parts_l1f
-    annotations: L1_full
     rescale_size: !!int 427
 
 
@@ -457,13 +465,11 @@ PARTS:
   BIRDSNAP_L1_pred:
     <<: *birdsnap
     <<: *parts_l1p
-    annotations: L1_pred
     rescale_size: !!int 427
 
   BIRDSNAP_L1_full:
     <<: *birdsnap
     <<: *parts_l1f
-    annotations: L1_full
     rescale_size: !!int 427
 
   ####################################
@@ -473,12 +479,10 @@ PARTS:
   CARS_L1_pred:
     <<: *cars
     <<: *parts_l1p
-    annotations: L1_pred
 
   CARS_L1_full:
     <<: *cars
     <<: *parts_l1f
-    annotations: L1_full
 
 
   ####################################
@@ -488,12 +492,10 @@ PARTS:
   FLOWERS_L1_pred:
     <<: *flowers
     <<: *parts_l1p
-    annotations: L1_pred
 
   FLOWERS_L1_full:
     <<: *flowers
     <<: *parts_l1f
-    annotations: L1_full
 
 
   ####################################