refactored parts handling

cvdatasets/annotation/base.py

@@ -18,6 +18,33 @@ from cvdatasets.utils.decorators import only_with_info
 
 class BaseAnnotations(abc.ABC):
 
+	@classmethod
+	def extract_kwargs(cls, opts):
+		return dict(
+			root_or_infofile=opts.data,
+			load_strict=getattr(opts, "load_strict", False),
+		)
+
+	@classmethod
+	def new(cls, opts, **_kwargs):
+		kwargs = cls.extract_kwargs(opts)
+		kwargs.update(_kwargs)
+		kwargs_str = pretty_print_dict(kwargs)
+		try:
+			annot = cls(**kwargs)
+		except Exception as e:
+			logging.error(f"Failed to create \"{cls.__name__}\" annotations " + \
+				f"with following kwargs: \"{kwargs_str}\". " + \
+				f"Error was: {e}"
+			)
+			raise
+		else:
+			logging.info(f"Loaded \"{annot.dataset_key}\" annotations " + \
+				f"with following kwargs: \"{kwargs_str}\""
+			)
+			return annot
+
+
 	def __init__(self, *, root_or_infofile, dataset_key=None, images_folder="images", load_strict=True, **kwargs):
 
 		self.dataset_key = dataset_key
@@ -49,10 +76,26 @@ class BaseAnnotations(abc.ABC):
 	def data_root(self):
 		return Path(self.info.BASE_DIR) / self.info.DATA_DIR
 
+	@property
+	@only_with_info
+	def dataset_key(self):
+		if hasattr(self.__class__, "name"):
+			return self.__class__.name
+
+		elif self._dataset_key is not None:
+			return self._dataset_key
+
+		else:
+			return self.__class__.__name__
+
+	@dataset_key.setter
+	def dataset_key(self, value):
+		self._dataset_key = value
+
 	@property
 	@only_with_info
 	def dataset_info(self):
-		key = getattr(self.__class__, "name", self.dataset_key)
+		key = self.dataset_key
 
 		if key not in self.info.DATASETS:
 			raise ValueError(f"Cannot find dataset with key \"{key}\"")
@@ -114,7 +157,7 @@ class BaseAnnotations(abc.ABC):
 		# TODO: pass all scales
 		new_kwargs = {}
 
-		if "scales" in dataset_info:
+		if "scales" in dataset_info and len(dataset_info.scales):
 			new_kwargs["ratio"] = dataset_info.scales[0]
 
 		if "is_uniform" in dataset_info:

cvdatasets/annotation/mixins/features_mixin.py

@@ -5,6 +5,14 @@ from cvdatasets.utils import feature_file_name
 class FeaturesMixin(abc.ABC):
 	FEATURE_PHONY = dict(train=["train"], test=["test", "val"])
 
+	@classmethod
+	def extract_kwargs(cls, opts):
+		kwargs = super(FeaturesMixin, cls).extract_kwargs(opts)
+		kwargs.update(dict(
+			feature_model=getattr(opts, "feature_model", False),
+		))
+		return kwargs
+
 	def __init__(self, *args, feature_model=None, feature_folder="features", **kwargs):
 		super(FeaturesMixin, self).__init__(*args, **kwargs)
 

cvdatasets/annotation/mixins/parts_mixin.py

@@ -1,5 +1,6 @@
 import abc
 import logging
+import copy
 import numpy as np
 
 from collections import OrderedDict
@@ -8,6 +9,15 @@ from cvdatasets.utils.decorators import only_with_info
 
 class PartsMixin(abc.ABC):
 
+	@classmethod
+	def extract_kwargs(cls, opts):
+		kwargs = super(PartsMixin, cls).extract_kwargs(opts)
+		kwargs.update(dict(
+			parts=getattr(opts, "parts", None),
+		))
+		return kwargs
+
+
 	def __init__(self, *args, parts=None, **kwargs):
 		self.part_type = parts
 		self.part_names = OrderedDict()
@@ -27,10 +37,18 @@ class PartsMixin(abc.ABC):
 	@property
 	@only_with_info
 	def dataset_info(self):
+		ds_info = super(PartsMixin, self).dataset_info
 		if self.part_type is not None:
-			return self.info.PARTS[self.part_type]
-		else:
-			return super(PartsMixin, self).dataset_info
+			parts_key = f"{self.dataset_key}_{self.part_type}"
+			if parts_key in self.info.PARTS:
+				parts_info = self.info.PARTS[parts_key]
+			else:
+				parts_info = self.info.PART_TYPES[self.part_type]
+
+			ds_info = copy.deepcopy(ds_info)
+			ds_info.update(parts_info)
+
+		return ds_info
 
 	def check_dataset_kwargs(self, subset, **kwargs):
 		if self.dataset_info is None:

cvdatasets/annotation/types/__init__.py

@@ -1,3 +1,5 @@
+import logging
+
 from cvdatasets.annotation.types.file_list import FileListAnnotations
 from cvdatasets.annotation.types.folder_annotations import FolderAnnotations
 from cvdatasets.annotation.types.json_annotations import JSONAnnotations
@@ -19,6 +21,23 @@ class AnnotationType(BaseChoiceType, metaclass=AnnotationMetaType):
 
 	Default = FILE_LIST
 
+	@classmethod
+	def new_annotation(cls, opts):
+		annot = cls[opts.dataset].value
+		return annot.new(opts)
+
+	@classmethod
+	def as_choices(cls, add_phony=True):
+		choices = super(AnnotationType, cls).as_choices()
+		if not add_phony:
+			return choices
+
+		for key in cls:
+			for phony in cls.phony(key):
+				choices[phony.lower()] = choices[key.name.lower()]
+
+		return choices
+
 	@classmethod
 	def phony(cls, key):
 		""" returns for a key a list of datasets,
@@ -51,17 +70,5 @@ class AnnotationType(BaseChoiceType, metaclass=AnnotationMetaType):
 
 		}.get(key, [])
 
-	@classmethod
-	def as_choices(cls, add_phony=True):
-		choices = super(AnnotationType, cls).as_choices()
-		if not add_phony:
-			return choices
-
-		for key in cls:
-			for phony in cls.phony(key):
-				choices[phony.lower()] = choices[key.name.lower()]
-
-		return choices
-
 if __name__ == '__main__':
 	print(AnnotationType.as_choices().keys())

scripts/config.sh

@@ -1,4 +1 @@
-source ${HOME}/.miniconda3/etc/profile.d/conda.sh
-conda activate ${ENV:-chainer6}
-
-PYTHON="python" #-m cProfile -o profile"
+ #-m cProfile -o profile"

scripts/display.py

@@ -16,9 +16,11 @@ def main(args):
 	assert args.dataset in AnnotationType, \
 		f"AnnotationType is not known: \"{args.dataset}\""
 
-	annotation_cls = AnnotationType[args.dataset].value
-	logging.info(f"Loading \"{args.dataset}\" annnotations from \"{args.data}\"")
-	annot = annotation_cls(root_or_infofile=args.data, parts=args.parts, load_strict=False)
+	annot = AnnotationType.new_annotation(args)
+	# annotation_cls = AnnotationType[args.dataset].value
+	# logging.info(f"Loading \"{args.dataset}\" annnotations from \"{args.data}\"")
+	# annot = annotation_cls.new(args, )
+	# annot = annotation_cls(root_or_infofile=args.data, parts=args.parts, load_strict=False)
 
 	kwargs = {}
 	if annot.info is None:

scripts/display.sh

@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
-source config.sh
+source ${HOME}/.miniconda3/etc/profile.d/conda.sh
+conda activate ${ENV:-chainer6}
+
+PYTHON="python"
 
 ############## Possible calls ##############
 
 ##### displays GT parts of CUB200
 # ./display.sh /home/korsch1/korsch/datasets/birds/cub200_11 \
-# 	--dataset cub \
+# 	CUB200 \
 # 	-s600 -n5 \
 # 	--features /home/korsch1/korsch/datasets/birds/features/{train,val}_16parts_gt.npz \
 # 	--ratio 0.31
@@ -13,7 +16,7 @@ source config.sh
 
 ##### displays NAC parts of CUB200
 # ./display.sh /home/korsch1/korsch/datasets/birds/NAC/2017-bilinear/ \
-# 	--dataset cub \
+# 	CUB200 \
 # 	-s600 -n5 \
 # 	--features /home/korsch1/korsch/datasets/birds/features/{train,val}_20parts.npz \
 # 	--ratio 0.31 \
@@ -21,11 +24,13 @@ source config.sh
 
 ##### load from info file, displays Uniform parts of CUB200 that are in the GT bounding box
 # ./display.sh ~/DATA/info.yml \
-# 	-p UNI \
+# 	CUB200
+# 	CUB200_UNI \
 # 	-s600 -n3 \
 # 	--parts_in_bb
 
 ############################################
 
+export DATA=$1
 $PYTHON display.py $@
 

scripts/info_files/info.yml

@@ -59,7 +59,6 @@ DATASETS:
   IMAGENET_TOP_INAT20: &inet_top_inat20
     <<: *inet
     annotations: "TOP_INAT20"
-    annotation_type: FOLDER
     n_classes: 44
 
   CUB200:         &cub200
@@ -80,7 +79,6 @@ DATASETS:
     <<: *cub200
     folder: birds/cub200_google_images
     annotations: "sem_selected"
-    annotation_type: FILE_LIST
 
   NAB:         &nabirds
     folder: birds/nabirds
@@ -151,10 +149,12 @@ DATASETS:
   INAT19_MINI:    &inat19_mini
     <<: *inat19
     annotations: "2019_small"
+    feature_suffix: .mini
 
   INAT19_TEST:    &inat19_test
     <<: *inat19
     annotations: "2019_test"
+    feature_suffix: .test
 
   INAT18:         &inat18
     folder: inat
@@ -175,10 +175,8 @@ DATASETS:
     n_classes: 107
 
   TIGERS_TEST:         &tigers_test
-    folder: tigers
+    <<: *tigers
     annotations: "reid/test"
-    annotation_type: FILE_LIST
-    n_classes: 107
 
 ############ Existing Part Annotations and Part Features
 ### feature file name composition:
@@ -202,12 +200,15 @@ PART_TYPES:
      - 0.31
 
   GT2:            &parts_gt2
+    # TODO: should be changed, since GT2 parts can also be present for other datasets
+    annotations: cub200_11_regrouped
     feature_suffix: _5parts_gt
     rescale_size: !!int -1
     scales:
       - 0.31
 
   NAC:            &parts_nac
+    annotations: NAC/2017-bilinear
     feature_suffix: _20parts
     rescale_size: !!int 224
     scales:
@@ -217,287 +218,72 @@ PART_TYPES:
   L1_pred:        &parts_l1p
     annotations: L1_pred
     feature_suffix: _5parts_L1_pred
-    rescale_size: !!int 299
-    scales:
-      - 0.31
+    rescale_size: !!int 427
+    scales: []
 
   L1_full:        &parts_l1f
     annotations: L1_full
     feature_suffix: _5parts_L1_full
-    rescale_size: !!int 299
-    scales:
-      - 0.31
+    rescale_size: !!int 427
+    scales: []
 
   NTS:        &parts_nts
     annotations: NTS
     feature_suffix: _7parts_nts
     rescale_size: !!int 448
-    scales:
-      - 0.31
+    scales: []
 
   NTS2:        &parts_nts2
     annotations: NTS2
     feature_suffix: _5parts_nts
     rescale_size: !!int 448
-    scales:
-      - 0.31
+    scales: []
 
 PARTS:
-  #### No Parts Annotations
-
-  IMAGENET_GLOBAL:
-    <<: *inet
-    <<: *parts_global
-
-  IMAGENET_TOP_INAT20_GLOBAL:
-    <<: *inet_top_inat20
-    <<: *parts_global
-
-  CUB200_2FOLD_GLOBAL:
-    <<: *cub200_2fold
-    <<: *parts_global
-
-  CUB200_GLOBAL:
-    <<: *cub200
-    <<: *parts_global
-
-  CUB200_GOOGLE_GLOBAL:
-    <<: *cub200_google
-    <<: *parts_global
-
-  CUB200_GOOGLE_SEM_GLOBAL:
-    <<: *cub200_google_sem
-    <<: *parts_global
-
-
-  CARS_GLOBAL:
-    <<: *cars
-    <<: *parts_global
-
-  DOGS_GLOBAL:
-    <<: *dogs
-    <<: *parts_global
-
-  NAB_GLOBAL:
-    <<: *nabirds
-    <<: *parts_global
-
-  BIRDSNAP_GLOBAL:
-    <<: *birdsnap
-    <<: *parts_global
-
-  FLOWERS_GLOBAL:
-    <<: *flowers
-    <<: *parts_global
-
-  INAT18_GLOBAL:
-    <<: *inat18
-    <<: *parts_global
-
-  INAT19_GLOBAL:
-    <<: *inat19
-    <<: *parts_global
-
-  INAT19_MINI_GLOBAL:
-    <<: *inat19_mini
-    <<: *parts_global
-    feature_suffix: .mini
-
-  INAT19_TEST_GLOBAL:
-    <<: *inat19_test
-    <<: *parts_global
-    feature_suffix: .test
-
-  INAT20_GLOBAL:
-    <<: *inat20
-    <<: *parts_global
-
-  INAT20_TEST_GLOBAL:
-    <<: *inat20_test
-    <<: *parts_global
-
-  INAT20_IN_CLASS_GLOBAL:
-    <<: *inat20_in_class
-    <<: *parts_global
-
-  INAT20_OUT_CLASS_GLOBAL:
-    <<: *inat20_out_class
-    <<: *parts_global
-
-  INAT20_U_OUT_CLASS_GLOBAL:
-    <<: *inat20_u_out_class
-    <<: *parts_global
-
-  INAT20_NOISY_OUT_CLASS_GLOBAL:
-    <<: *inat20_noisy_out_class
-    <<: *parts_global
-
-  INAT20_NOISY_IN_CLASS_GLOBAL:
-    <<: *inat20_noisy_in_class
-    <<: *parts_global
-
-  HED_GLOBAL:
-    <<: *hed
-    <<: *parts_global
-
-  TIGERS_GLOBAL:
-    <<: *tigers
-    <<: *parts_global
-    rescale_size: !!int -1
-    scales:
-      - .31
-
-  TIGERS_TEST_GLOBAL:
-    <<: *tigers_test
-    <<: *parts_global
-
-  #### With Parts Annotations
-
-
-  ####################################
-  # CUB200-2011 2-Fold training set
-  ####################################
-
-  CUB200_2FOLD_L1_pred:
-    <<: *cub200_2fold
-    <<: *parts_l1p
-    rescale_size: !!int 427
-
-  CUB200_2FOLD_L1_full:
-    <<: *cub200_2fold
-    <<: *parts_l1f
-    rescale_size: !!int 427
+  # all <DATASET>_<PART_TYPES> combinations are created implicitely.
+  # if you want some changes in the configs, then crete here an entry
+  # and update the config values
 
   ####################################
   # CUB200-2011
   ####################################
 
-  CUB200_UNI:
-    <<: *cub200
-    <<: *parts_uni
-
-  CUB200_GT:
-    <<: *cub200
-    <<: *parts_gt
-
-  CUB200_GT2:
-    <<: *cub200
-    <<: *parts_gt2
-    annotations: cub200_11_regrouped
-
-  CUB200_NAC:
-    <<: *cub200
-    <<: *parts_nac
-    annotations: NAC/2017-bilinear
-
-  CUB200_NTS:
-    <<: *cub200
-    <<: *parts_nts
-
-  CUB200_NTS2:
-    <<: *cub200
-    <<: *parts_nts2
-
   CUB200_L1_pred:
     <<: *cub200
     <<: *parts_l1p
     annotations: cub200_11_L1_pred_old
-    # rescale_size: !!int 427
+    rescale_size: !!int 299
 
   CUB200_L1_full:
     <<: *cub200
     <<: *parts_l1f
     annotations: cub200_11_L1_full_old
-    # rescale_size: !!int 427
+    rescale_size: !!int 299
 
   CUB200_L1_pred_15:
     <<: *cub200
     <<: *parts_l1p
     annotations: L1_pred_15
-    rescale_size: !!int 427
     feature_suffix: _15parts_L1_pred
 
   CUB200_L1_full_15:
     <<: *cub200
     <<: *parts_l1f
     annotations: L1_full_15
-    rescale_size: !!int 427
     feature_suffix: _15parts_L1_full
 
   CUB200_L1_pred_2:
     <<: *cub200
     <<: *parts_l1p
     annotations: L1_pred_2
-    rescale_size: !!int 427
     feature_suffix: _2parts_L1_pred
 
   CUB200_L1_full_2:
     <<: *cub200
     <<: *parts_l1f
     annotations: L1_full_2
-    rescale_size: !!int 427
     feature_suffix: _2parts_L1_full
 
-
-  ####################################
-  # NA Birds
-  ####################################
-
-  NAB_GT:
-    <<: *nabirds
-    <<: *parts_gt
-
-  NAB_L1_pred:
-    <<: *nabirds
-    <<: *parts_l1p
-    rescale_size: !!int 427
-
-  NAB_L1_full:
-    <<: *nabirds
-    <<: *parts_l1f
-    rescale_size: !!int 427
-
-
-  ####################################
-  # BIRDSNAP
-  ####################################
-
-  BIRDSNAP_L1_pred:
-    <<: *birdsnap
-    <<: *parts_l1p
-    rescale_size: !!int 427
-
-  BIRDSNAP_L1_full:
-    <<: *birdsnap
-    <<: *parts_l1f
-    rescale_size: !!int 427
-
-  ####################################
-  # Stanford Cars
-  ####################################
-
-  CARS_L1_pred:
-    <<: *cars
-    <<: *parts_l1p
-
-  CARS_L1_full:
-    <<: *cars
-    <<: *parts_l1f
-
-
-  ####################################
-  # Flowers 102
-  ####################################
-
-  FLOWERS_L1_pred:
-    <<: *flowers
-    <<: *parts_l1p
-
-  FLOWERS_L1_full:
-    <<: *flowers
-    <<: *parts_l1f
-
-
   ####################################
   # iNaturalist 2019
   ####################################

scripts/utils/parser.py

@@ -29,7 +29,7 @@ def parse_args():
 				help=data_help),
 			Arg("dataset", choices=info_file.DATASETS.keys(),
 				help=dataset_help),
-			Arg("parts", default="CUB200_GLOBAL", choices=info_file.PARTS.keys(),
+			Arg("parts", choices=info_file.PART_TYPES.keys(),
 				help=parts_help),
 		], group_name="Dataset arguments")