Merge branch 'dev'

cvdatasets/_version.py

@@ -1 +1 @@
-__version__ = "0.8.0"
+__version__ = "0.9.2"

cvdatasets/annotation/base.py

@@ -19,7 +19,7 @@ from cvdatasets.utils.decorators import only_with_info
 class BaseAnnotations(abc.ABC):
 
 	@classmethod
-	def extract_kwargs(cls, opts):
+	def extract_kwargs(cls, opts, *args, **kwargs):
 		return dict(
 			root_or_infofile=opts.data,
 			load_strict=getattr(opts, "load_strict", False),
@@ -27,8 +27,8 @@ class BaseAnnotations(abc.ABC):
 		)
 
 	@classmethod
-	def new(cls, opts, **_kwargs):
-		kwargs = cls.extract_kwargs(opts)
+	def new(cls, opts,  *, ds_info=None, **_kwargs):
+		kwargs = cls.extract_kwargs(opts, ds_info)
 		kwargs.update(_kwargs)
 		kwargs_str = pretty_print_dict(kwargs)
 		try:

cvdatasets/annotation/mixins/features_mixin.py

@@ -6,8 +6,8 @@ class FeaturesMixin(abc.ABC):
 	FEATURE_PHONY = dict(train=["train"], test=["test", "val"])
 
 	@classmethod
-	def extract_kwargs(cls, opts):
-		kwargs = super(FeaturesMixin, cls).extract_kwargs(opts)
+	def extract_kwargs(cls, opts, *args, **kwargs):
+		kwargs = super(FeaturesMixin, cls).extract_kwargs(opts, *args, **kwargs)
 		kwargs.update(dict(
 			feature_model=getattr(opts, "feature_model", None),
 		))
@@ -27,7 +27,7 @@ class FeaturesMixin(abc.ABC):
 			tried = []
 			model_info = self.info.MODELS[self.feature_model]
 			for subset_phony in FeaturesMixin.FEATURE_PHONY[subset]:
-				features = feature_file_name(subset_phony, dataset_info, model_info)
+				features = feature_file_name(subset_phony, self.dataset_info, model_info)
 				feature_path = self.root / self.feature_folder / features
 				if feature_path.is_file(): break
 				tried.append(feature_path)

cvdatasets/annotation/mixins/parts_mixin.py

@@ -13,8 +13,8 @@ from cvdatasets.utils.decorators import only_with_info
 class PartsMixin(abc.ABC):
 
 	@classmethod
-	def extract_kwargs(cls, opts):
-		kwargs = super(PartsMixin, cls).extract_kwargs(opts)
+	def extract_kwargs(cls, opts, *args, **kwargs):
+		kwargs = super(PartsMixin, cls).extract_kwargs(opts, *args, **kwargs)
 		kwargs.update(dict(
 			parts=getattr(opts, "parts", None),
 		))

cvdatasets/annotation/types/__init__.py

@@ -17,16 +17,18 @@ class AnnotationType(BaseChoiceType):
 
 	@classmethod
 	def new_annotation(cls, opts, **kwargs):
+		info_file = read_info_file(opts.data)
+		ds_info = info_file.DATASETS[opts.dataset]
+
 		if opts.dataset in cls:
 			annot = cls[opts.dataset].value
+
 		else:
-			info_file = read_info_file(opts.data)
 			assert opts.dataset in info_file.DATASETS, \
-				f"No information was found about the dataset \"{args.dataset}\" in the info file \"{args.data}\""
-			ds_info = info_file.DATASETS[opts.dataset]
+				f"No information was found about the dataset \"{opts.dataset}\" in the info file \"{args.data}\""
 			annot = cls[ds_info.annotation_type.lower()].value
 
-		return annot.new(opts, **kwargs)
+		return annot.new(opts, ds_info=ds_info, **kwargs)
 
 	@classmethod
 	def as_choices(cls, add_phony=True):

cvdatasets/annotation/types/file_list.py

@@ -5,6 +5,16 @@ from cvdatasets.annotation.files import AnnotationFiles
 
 class FileListAnnotations(Annotations):
 
+	@classmethod
+	def extract_kwargs(cls, opts, ds_info=None, *args, **kwargs):
+		kwargs = super(FileListAnnotations, cls).extract_kwargs(opts, ds_info=ds_info, *args, **kwargs)
+		kwargs["test_fold_id"] = getattr(opts, "test_fold_id", 0)
+		return kwargs
+
+	def __init__(self, *args, test_fold_id=0, **kwargs):
+		self._test_fold_id = test_fold_id
+		super(FileListAnnotations, self).__init__(*args, **kwargs)
+
 	def load_files(self, file_obj) -> AnnotationFiles:
 		file_obj.load_files("images.txt", "labels.txt", "tr_ID.txt")
 		return file_obj
@@ -30,10 +40,11 @@ class FileListAnnotations(Annotations):
 
 		assert hasattr(self, "uuids"), \
 			"UUIDs were not parsed yet! Please call _parse_uuids before this method!"
-
 		uuid_to_split = {uuid: int(split) for uuid, split in zip(self.uuids, self.files.tr_ID)}
-		self.train_split = np.array([uuid_to_split[uuid] for uuid in self.uuids], dtype=bool)
-		self.test_split = np.logical_not(self.train_split)
+
+		split_ids = np.array([uuid_to_split[uuid] for uuid in self.uuids])
+		self.test_split = split_ids == self._test_fold_id
+		self.train_split = np.logical_not(self.test_split)
 
 if __name__ == '__main__':
 	annot = FileListAnnotations(

cvdatasets/annotation/types/folder_annotations.py

@@ -1,38 +1,71 @@
 import numpy as np
 
+from pathlib import Path
+
+from cvdatasets import utils
 from cvdatasets.annotation.base import Annotations
 from cvdatasets.annotation.files import AnnotationFiles
 
 class FolderAnnotations(Annotations):
+	_default_folders = dict(
+		train_images="train",
+		val_images="val",
+		test_images=("test", True)
+	)
+
+	@classmethod
+	def extract_kwargs(cls, opts, ds_info=None, *args, **kwargs):
+		kwargs = super(FolderAnnotations, cls).extract_kwargs(opts, ds_info=ds_info, *args, **kwargs)
+		if ds_info is None:
+			return kwargs
+
+		folders = dict(cls._default_folders)
+
+		for key in ["train", "val", "test"]:
+			key = f"{key}_images"
+
+			value = ds_info.get(key)
+			if value is None:
+				continue
+
+			folders[key] = value
+
+		kwargs["folders"] = folders
+		return kwargs
+
+	def __init__(self, *args, folders=_default_folders, **kwargs):
+		self._folders = folders
+		super(FolderAnnotations, self).__init__(*args, **kwargs)
 
 	def load_files(self, file_obj) -> AnnotationFiles:
-		file_obj.load_files(
-			train_images="ILSVRC2012_img_train",
-			val_images="ILSVRC2012_img_val",
-			test_images=("ILSVRC2012_img_test", True),
-		)
+		file_obj.load_files(**self._folders)
 		return file_obj
 
 	@property
 	def _has_test_set(self) -> bool:
 		return self.files.test_images is not None
 
+	def _uuid_from_path(self, fpath):
+		return "_".join(Path(fpath).relative_to(self.root).parts)
+		# return "_".join(Path(fpath).parts[-3:])
 
 	def _parse_uuids(self) -> None:
 		self.images_folder = ""
 
-		train_uuid_fnames = [(fpath.name, str(fpath.relative_to(self.root))) for
+		train_uuid_fnames = [(self._uuid_from_path(fpath), str(fpath.relative_to(self.root))) for
 			fpath in self.files.train_images]
 
-		val_uuid_fnames = [(fpath.name, str(fpath.relative_to(self.root))) for
+		val_uuid_fnames = [(self._uuid_from_path(fpath), str(fpath.relative_to(self.root))) for
 			fpath in self.files.val_images]
 
 		if self._has_test_set:
-			test_uuid_fnames = [(fpath.name, str(fpath.relative_to(self.root))) for
+			test_uuid_fnames = [(self._uuid_from_path(fpath), str(fpath.relative_to(self.root))) for
 				fpath in self.files.test_images]
 
 		uuid_fnames = train_uuid_fnames + val_uuid_fnames
 		self.uuids, self.image_names = map(np.array, zip(*uuid_fnames))
+
+		utils.dataset._uuid_check(self.uuids)
 		self.uuid_to_idx = {uuid: i for i, uuid in enumerate(self.uuids)}
 
 

cvdatasets/annotation/types/json_annotations.py

@@ -3,12 +3,10 @@ import hashlib
 import logging
 import numpy as np
 
+from cvdatasets import utils
 from cvdatasets.annotation.base import Annotations
 from cvdatasets.annotation.files import AnnotationFiles
 
-def _uuid_check(uuids):
-	return len(np.unique(uuids)) == len(uuids)
-
 def _uuid_entry(im_info):
 	return hashlib.md5(im_info["file_name"].encode()).hexdigest()
 
@@ -30,8 +28,7 @@ class JSONAnnotations(Annotations):
 		uuid_fnames = [(str(im["id"]), im["file_name"]) for im in self.files.trainval["images"]]
 		self.uuids, self.image_names = map(np.array, zip(*uuid_fnames))
 
-		assert _uuid_check(self.uuids) , \
-			"UUIDs are not unique!"
+		utils.dataset._uuid_check(self.uuids)
 
 		self.uuid_to_idx = {uuid: i for i, uuid in enumerate(self.uuids)}
 

cvdatasets/dataset/__init__.py

@@ -1,11 +1,14 @@
+from cvdatasets.dataset.mixins.base import BaseMixin
+from cvdatasets.dataset.mixins.bounding_box import BBCropMixin
+from cvdatasets.dataset.mixins.bounding_box import BBoxMixin
+from cvdatasets.dataset.mixins.bounding_box import MultiBoxMixin
 from cvdatasets.dataset.mixins.chainer_mixins import IteratorMixin
 from cvdatasets.dataset.mixins.features import PreExtractedFeaturesMixin
-from cvdatasets.dataset.mixins.parts import BBCropMixin
-from cvdatasets.dataset.mixins.parts import BBoxMixin
+from cvdatasets.dataset.mixins.image_profiler import ImageProfilerMixin
+from cvdatasets.dataset.mixins.parts import BasePartMixin
 from cvdatasets.dataset.mixins.parts import CroppedPartMixin
-from cvdatasets.dataset.mixins.parts import MultiBoxMixin
 from cvdatasets.dataset.mixins.parts import PartCropMixin
-from cvdatasets.dataset.mixins.parts import PartMixin
+from cvdatasets.dataset.mixins.parts import _PartMixin
 from cvdatasets.dataset.mixins.parts import PartRevealMixin
 from cvdatasets.dataset.mixins.parts import PartsInBBMixin
 from cvdatasets.dataset.mixins.parts import RandomBlackOutMixin
@@ -13,9 +16,10 @@ from cvdatasets.dataset.mixins.parts import RevealedPartMixin
 from cvdatasets.dataset.mixins.parts import UniformPartMixin
 from cvdatasets.dataset.mixins.reading import AnnotationsReadMixin
 from cvdatasets.dataset.mixins.reading import ImageListReadingMixin
+from cvdatasets.dataset.mixins.transform import TransformMixin
 
 
-class ImageWrapperDataset(PartMixin, PreExtractedFeaturesMixin, AnnotationsReadMixin, IteratorMixin):
+class ImageWrapperDataset(_PartMixin, PreExtractedFeaturesMixin, AnnotationsReadMixin, IteratorMixin):
 	pass
 
 class Dataset(ImageWrapperDataset):
@@ -23,3 +27,39 @@ class Dataset(ImageWrapperDataset):
 	def get_example(self, i):
 		im_obj = super(Dataset, self).get_example(i)
 		return im_obj.as_tuple()
+
+__all__ = [
+	"Dataset",
+	"ImageWrapperDataset",
+
+	### mixins ###
+	"BaseMixin",
+	# reading
+	"AnnotationsReadMixin",
+	"ImageListReadingMixin",
+
+	# features
+	"PreExtractedFeaturesMixin",
+
+	# image profiling
+	"ImageProfilerMixin",
+
+	# bounding boxes
+	"BBCropMixin",
+	"BBoxMixin",
+	"MultiBoxMixin",
+
+	# parts
+	"BasePartMixin",
+	"CroppedPartMixin",
+	"PartCropMixin",
+	"_PartMixin",
+	"PartRevealMixin",
+	"PartsInBBMixin",
+	"RandomBlackOutMixin",
+	"RevealedPartMixin",
+	"UniformPartMixin",
+
+	# transform mixin
+	"TransformMixin",
+]

cvdatasets/dataset/image/__init__.py

@@ -0,0 +1,8 @@
+from cvdatasets.dataset.image.image_wrapper import ImageWrapper
+from cvdatasets.dataset.image.size import Size
+
+
+__all__ = [
+	"ImageWrapper",
+	"Size",
+]

cvdatasets/dataset/image.py → cvdatasets/dataset/image/image_wrapper.py

@@ -1,14 +1,12 @@
-from PIL import Image
-from imageio import imread
-from os.path import isfile
-
 import copy
 import numpy as np
 
-from .part import Parts
-from .part import UniformParts
-from .part import SurrogateType
+from PIL import Image
+
 from cvdatasets import utils
+from cvdatasets.dataset.part import Parts
+from cvdatasets.dataset.part import SurrogateType
+from cvdatasets.dataset.part import UniformParts
 
 def should_have_parts(func):
 	def inner(self, *args, **kwargs):
@@ -17,31 +15,18 @@ def should_have_parts(func):
 	return inner
 
 class ImageWrapper(object):
-	@staticmethod
-	def read_image(im_path, mode="RGB", n_retries=5):
-		_read = lambda: Image.open(im_path, mode="r")
-		if n_retries <= 0:
-			assert isfile(im_path), "Image \"{}\" does not exist!".format(im_path)
-			return _read()
-		else:
-			error = None
-			for i in range(n_retries):
-				try:
-					return _read()
-				except Exception as e:
-					error = e
-
-			raise RuntimeError("Reading image \"{}\" failed after {} n_retries! ({})".format(im_path, n_retries, error))
 
 
 	def __init__(self, im_path, label,
 		parts=None,
 		mode="RGB",
+		uuid=None,
 		part_rescale_size=None,
 		part_surrogate_type=SurrogateType.MIDDLE,
 		center_cropped=True):
 
 		self.mode = mode
+		self.uuid = uuid
 		self._im = None
 		self._im_array = None
 
@@ -64,16 +49,21 @@ class ImageWrapper(object):
 	@property
 	def im_array(self):
 		if self._im_array is None:
+
 			if isinstance(self._im, Image.Image):
-				_im = self._im.convert(self.mode)
+				_im = utils.retry_operation(5, self._im.convert, self.mode)
 				self._im_array = utils.asarray(_im)
+
 			elif isinstance(self._im, np.ndarray):
 				if self.mode == "RGB" and self._im.ndim == 2:
 					self._im_array = np.stack((self._im,) * 3, axis=-1)
+
 				elif self._im.ndim in (3, 4):
 					self._im_array = self._im
+
 				else:
 					raise ValueError()
+
 			else:
 				raise ValueError()
 		return self._im_array
@@ -81,13 +71,13 @@ class ImageWrapper(object):
 	@property
 	def im(self):
 		if isinstance(self._im, Image.Image) and self._im.mode != self.mode:
-			self._im = self._im.convert(self.mode)
+			self._im = utils.retry_operation(5, self._im.convert, self.mode)
 		return self._im
 
 	@im.setter
 	def im(self, value):
 		if isinstance(value, str):
-			self._im = ImageWrapper.read_image(value, mode=self.mode)
+			self._im = utils.read_image(value, n_retries=5)
 			self._im_path = value
 		else:
 			self._im = value

cvdatasets/dataset/image/size.py

@@ -0,0 +1,50 @@
+import numpy as np
+
+from collections.abc import Iterable
+
+class Size(object):
+	dtype=np.int32
+
+	def __init__(self, value):
+		self._size = np.zeros(2, dtype=self.dtype)
+		if isinstance(value, int):
+			self._size[:] = value
+
+		elif isinstance(value, Size):
+			self._size[:] = value._size
+
+		elif isinstance(value, Iterable):
+			assert len(value) <= 2, \
+				"only iterables of maximum size 2 are supported, but was {}!".format(len(value))
+			self._size[:] = np.round(value)
+
+
+		else:
+			raise ValueError("Unsupported data type: {}!".format(type(value)))
+
+	def __str__(self):
+		return "<Size {}x{}>".format(*self._size)
+
+	def __repr__(self):
+		return str(self)
+
+	def __add__(self, other):
+		return self.__class__(self._size + other)
+
+	def __sub__(self, other):
+		return self.__class__(self._size - other)
+
+	def __mul__(self, other):
+		return self.__class__(self._size * other)
+
+	def __truediv__(self, other):
+		return self.__class__(self._size / other)
+
+	def __floordiv__(self, other):
+		return self.__class__(self._size // other)
+
+	def __iter__(self):
+		return iter(self._size)
+
+	def __len__(self):
+		return len(self._size)

cvdatasets/dataset/mixins/__init__.py

@@ -1,33 +0,0 @@
-from abc import ABC, abstractmethod
-
-import numpy as np
-import six
-
-from matplotlib.patches import Rectangle
-
-class BaseMixin(ABC):
-
-	@abstractmethod
-	def get_example(self, i):
-		s = super(BaseMixin, self)
-		if hasattr(s, "get_example"):
-			return s.get_example(i)
-
-	def plot_bounding_box(self, i, ax, fill=False, linestyle="--", **kwargs):
-		x, y, w, h = self.bounding_box(i)
-		ax.add_patch(Rectangle(
-			(x,y), w, h,
-			fill=False,
-			linestyle="-.",
-			**kwargs
-		))
-
-	def __getitem__(self, index):
-		if isinstance(index, slice):
-			current, stop, step = index.indices(len(self))
-			return [self.get_example(i) for i in
-					six.moves.range(current, stop, step)]
-		elif isinstance(index, list) or isinstance(index, np.ndarray):
-			return [self.get_example(i) for i in index]
-		else:
-			return self.get_example(index)

cvdatasets/dataset/mixins/base.py

@@ -0,0 +1,32 @@
+import abc
+import numpy as np
+import six
+
+from matplotlib.patches import Rectangle
+
+class BaseMixin(abc.ABC):
+
+	@abc.abstractmethod
+	def get_example(self, i):
+		s = super(BaseMixin, self)
+		if hasattr(s, "get_example"):
+			return s.get_example(i)
+
+	def plot_bounding_box(self, i, ax, fill=False, linestyle="--", **kwargs):
+		x, y, w, h = self.bounding_box(i)
+		ax.add_patch(Rectangle(
+			(x,y), w, h,
+			fill=False,
+			linestyle="-.",
+			**kwargs
+		))
+
+	def __getitem__(self, index):
+		if isinstance(index, slice):
+			current, stop, step = index.indices(len(self))
+			return [self.get_example(i) for i in
+					six.moves.range(current, stop, step)]
+		elif isinstance(index, list) or isinstance(index, np.ndarray):
+			return [self.get_example(i) for i in index]
+		else:
+			return self.get_example(index)

cvdatasets/dataset/mixins/bounding_box.py

@@ -0,0 +1,60 @@
+import numpy as np
+
+from cvdatasets.dataset.mixins.base import BaseMixin
+
+class BBoxMixin(BaseMixin):
+
+	def bounding_box(self, i):
+		bbox = self._get("bounding_box", i)
+		return [bbox[attr] for attr in "xywh"]
+
+class MultiBoxMixin(BaseMixin):
+	_all_keys=[
+		"x", "x0", "x1",
+		"y", "y0", "y1",
+		"w", "h",
+	]
+
+	def multi_box(self, i, keys=["x0","x1","y0","y1"]):
+		assert all([key in self._all_keys for key in keys]), \
+			f"unknown keys found: {keys}. Possible are: {self._all_keys}"
+
+		boxes = [
+			dict(
+				x=box["x0"], x0=box["x0"], x1=box["x1"],
+
+				y=box["y0"], y0=box["y0"], y1=box["y1"],
+
+				w=box["x1"] - box["x0"],
+				h=box["y1"] - box["y0"],
+			)
+			for box in self._get("multi_box", i)["objects"]
+		]
+
+		return [[box[key] for key in keys] for box in boxes]
+
+class BBCropMixin(BBoxMixin):
+
+	def __init__(self, *, crop_to_bb=False, crop_uniform=False, **kwargs):
+		super(BBCropMixin, self).__init__(**kwargs)
+		self.crop_to_bb = crop_to_bb
+		self.crop_uniform = crop_uniform
+
+	def bounding_box(self, i):
+		x,y,w,h = super(BBCropMixin, self).bounding_box(i)
+		if self.crop_uniform:
+			x0 = x + w//2
+			y0 = y + h//2
+
+			crop_size = max(w//2, h//2)
+
+			x,y = max(x0 - crop_size, 0), max(y0 - crop_size, 0)
+			w = h = crop_size * 2
+		return x,y,w,h
+
+	def get_example(self, i):
+		im_obj = super(BBCropMixin, self).get_example(i)
+		if self.crop_to_bb:
+			bb = self.bounding_box(i)
+			return im_obj.crop(*bb)
+		return im_obj

cvdatasets/dataset/mixins/chainer_mixins/base.py

@@ -5,9 +5,9 @@ except ImportError:
 else:
 	has_chainer = True
 
-from abc import ABC
+import abc
 
-class BaseChainerMixin(ABC):
+class BaseChainerMixin(abc.ABC):
 
 	def chainer_check(self):
 		global has_chainer

cvdatasets/dataset/mixins/features.py

@@ -2,7 +2,7 @@ import numpy as np
 
 from os.path import isfile
 
-from . import BaseMixin
+from cvdatasets.dataset.mixins.base import BaseMixin
 
 
 class PreExtractedFeaturesMixin(BaseMixin):

cvdatasets/dataset/mixins/image_profiler.py

@@ -0,0 +1,27 @@
+import numpy as np
+from contextlib import contextmanager
+
+from cvdatasets.dataset.mixins.base import BaseMixin
+
+
+class ImageProfilerMixin(BaseMixin):
+	def __init__(self, *args, **kwargs):
+		super(ImageProfilerMixin, self).__init__(*args, **kwargs)
+		self._profile_img_enabled = False
+
+	@contextmanager
+	def enable_img_profiler(self):
+		_dmp = self._profile_img_enabled
+		self._profile_img_enabled = True
+		yield
+		self._profile_img_enabled = _dmp
+
+	def _profile_img(self, img, tag):
+		if len(img) == 0: return
+		if self._profile_img_enabled:
+			print(f"[{tag:^30s}]",
+				" | ".join([
+					f"size: {str(img.shape):>20s}",
+					f"pixel values: ({img.min():+8.2f}, {img.max():+8.2f})"
+					])
+				)

cvdatasets/dataset/mixins/parts.py

@@ -1,65 +1,17 @@
 import numpy as np
 
-from cvdatasets.dataset.mixins import BaseMixin
+from cvdatasets.dataset.mixins.base import BaseMixin
+from cvdatasets.dataset.mixins.bounding_box import BBoxMixin
+from cvdatasets.dataset.mixins.bounding_box import BBCropMixin
 
-class BBoxMixin(BaseMixin):
+class BasePartMixin(BaseMixin):
 
-	def bounding_box(self, i):
-		bbox = self._get("bounding_box", i)
-		return [bbox[attr] for attr in "xywh"]
-
-class MultiBoxMixin(BaseMixin):
-	_all_keys=[
-		"x", "x0", "x1",
-		"y", "y0", "y1",
-		"w", "h",
-	]
-
-	def multi_box(self, i, keys=["x0","x1","y0","y1"]):
-		assert all([key in self._all_keys for key in keys]), \
-			f"unknown keys found: {keys}. Possible are: {self._all_keys}"
-
-		boxes = [
-			dict(
-				x=box["x0"], x0=box["x0"], x1=box["x1"],
-
-				y=box["y0"], y0=box["y0"], y1=box["y1"],
-
-				w=box["x1"] - box["x0"],
-				h=box["y1"] - box["y0"],
-			)
-			for box in self._get("multi_box", i)["objects"]
-		]
-
-		return [[box[key] for key in keys] for box in boxes]
-
-class BBCropMixin(BBoxMixin):
-
-	def __init__(self, *, crop_to_bb=False, crop_uniform=False, **kwargs):
-		super(BBCropMixin, self).__init__(**kwargs)
-		self.crop_to_bb = crop_to_bb
-		self.crop_uniform = crop_uniform
-
-	def bounding_box(self, i):
-		x,y,w,h = super(BBCropMixin, self).bounding_box(i)
-		if self.crop_uniform:
-			x0 = x + w//2
-			y0 = y + h//2
-
-			crop_size = max(w//2, h//2)
-
-			x,y = max(x0 - crop_size, 0), max(y0 - crop_size, 0)
-			w = h = crop_size * 2
-		return x,y,w,h
+	def __init__(self, ratio=None, *args, **kwargs):
+		super(BasePartMixin, self).__init__(*args, **kwargs)
+		self.ratio = ratio
 
-	def get_example(self, i):
-		im_obj = super(BBCropMixin, self).get_example(i)
-		if self.crop_to_bb:
-			bb = self.bounding_box(i)
-			return im_obj.crop(*bb)
-		return im_obj
+class PartsInBBMixin(BasePartMixin, BBoxMixin):
 
-class PartsInBBMixin(BBoxMixin):
 	def __init__(self, parts_in_bb=False, *args, **kwargs):
 		super(PartsInBBMixin, self).__init__(*args, **kwargs)
 		self.parts_in_bb = parts_in_bb
@@ -72,7 +24,7 @@ class PartsInBBMixin(BBoxMixin):
 			return im_obj.hide_parts_outside_bb(*bb)
 		return im_obj
 
-class PartCropMixin(BaseMixin):
+class PartCropMixin(BasePartMixin):
 
 	def __init__(self, return_part_crops=False, *args, **kwargs):
 		super(PartCropMixin, self).__init__(*args, **kwargs)
@@ -85,7 +37,7 @@ class PartCropMixin(BaseMixin):
 		return im_obj
 
 
-class PartRevealMixin(BaseMixin):
+class PartRevealMixin(BasePartMixin):
 
 	def __init__(self, reveal_visible=False, *args, **kwargs):
 		super(PartRevealMixin, self).__init__(*args, **kwargs)
@@ -93,18 +45,16 @@ class PartRevealMixin(BaseMixin):
 
 	def get_example(self, i):
 		im_obj = super(PartRevealMixin, self).get_example(i)
-		assert hasattr(self, "ratio"), "\"ratio\" attribute is missing!"
 		if self.reveal_visible:
 			return im_obj.reveal_visible(self.ratio)
 		return im_obj
 
 
-class UniformPartMixin(BaseMixin):
+class UniformPartMixin(BasePartMixin):
 
-	def __init__(self, uniform_parts=False, ratio=None, *args, **kwargs):
+	def __init__(self, uniform_parts=False, *args, **kwargs):
 		super(UniformPartMixin, self).__init__(*args, **kwargs)
 		self.uniform_parts = uniform_parts
-		self.ratio = ratio
 
 	def get_example(self, i):
 		im_obj = super(UniformPartMixin, self).get_example(i)
@@ -112,7 +62,7 @@ class UniformPartMixin(BaseMixin):
 			return im_obj.uniform_parts(self.ratio)
 		return im_obj
 
-class RandomBlackOutMixin(BaseMixin):
+class RandomBlackOutMixin(BasePartMixin):
 
 	def __init__(self, seed=None, rnd_select=False, blackout_parts=None, *args, **kwargs):
 		super(RandomBlackOutMixin, self).__init__(*args, **kwargs)
@@ -129,18 +79,18 @@ class RandomBlackOutMixin(BaseMixin):
 
 # some shortcuts
 
-class PartMixin(RandomBlackOutMixin, PartsInBBMixin, UniformPartMixin, BBCropMixin):
+class _PartMixin(RandomBlackOutMixin, PartsInBBMixin, UniformPartMixin, BBCropMixin):
 	"""
 		TODO!
 	"""
 
-class RevealedPartMixin(PartRevealMixin, PartMixin):
+class RevealedPartMixin(PartRevealMixin, _PartMixin):
 	"""
 		TODO!
 	"""
 
 
-class CroppedPartMixin(PartCropMixin, PartMixin):
+class CroppedPartMixin(PartCropMixin, _PartMixin):
 	"""
 		TODO!
 	"""

cvdatasets/dataset/mixins/reading.py

@@ -2,8 +2,8 @@ import numpy as np
 
 from os.path import join
 
-from . import BaseMixin
-from ..image import ImageWrapper
+from cvdatasets.dataset.mixins.base import BaseMixin
+from cvdatasets.dataset.image import ImageWrapper
 
 class AnnotationsReadMixin(BaseMixin):
 
@@ -31,6 +31,7 @@ class AnnotationsReadMixin(BaseMixin):
 
 		return ImageWrapper(im_path, int(label), parts,
 			mode=self.mode,
+			uuid=i,
 			part_rescale_size=self.part_rescale_size,
 			center_cropped=self.center_cropped)
 
@@ -62,7 +63,7 @@ class ImageListReadingMixin(BaseMixin):
 		im_file, label = self._pairs[i]
 		im_path = join(self._root, im_file)
 
-		return ImageWrapper(im_path, int(label))
+		return ImageWrapper(im_path, int(label), uuid=i)
 
 	@property
 	def labels(self):

cvdatasets/dataset/mixins/transform.py

@@ -0,0 +1,43 @@
+import abc
+import chainer
+
+from cvdatasets.dataset.image.size import Size
+from cvdatasets.dataset.mixins.base import BaseMixin
+
+class TransformMixin(BaseMixin):
+
+	def __init__(self, size, part_size=None, *args, **kwargs):
+		super(TransformMixin, self).__init__(*args, **kwargs)
+
+		self.size = size
+		self.part_size = size if part_size is None else part_size
+
+	@abc.abstractmethod
+	def transform(self, im_obj):
+		pass
+
+	def get_example(self, i):
+		im_obj = super(TransformMixin, self).get_example(i)
+		return self.transform(im_obj)
+
+	@property
+	def size(self):
+		if chainer.config.train:
+			return self._size // 0.875
+		else:
+			return self._size
+
+	@size.setter
+	def size(self, value):
+		self._size = Size(value)
+
+	@property
+	def part_size(self):
+		if chainer.config.train:
+			return self._part_size // 0.875
+		else:
+			return self._part_size
+
+	@part_size.setter
+	def part_size(self, value):
+		self._part_size = Size(value)

cvdatasets/dataset/part/base.py

@@ -7,16 +7,19 @@ from matplotlib import pyplot as plt
 from skimage.transform import resize
 
 from cvdatasets import utils
+from cvdatasets.dataset.part.surrogate import SurrogateType
 
 class BasePartCollection(ABC):
 
 	def __getitem__(self, i):
 		return self._parts[i]
 
-	def __len__(self, i):
+	def __len__(self):
 		return len(self._parts)
 
 	def __repr__(self):
+		if len(self._parts) == 0:
+			return repr(self._parts)
 		return repr(np.stack([p.as_annotation for p in self._parts]))
 
 	@property
@@ -76,7 +79,7 @@ class BasePartCollection(ABC):
 
 
 class BasePart(ABC):
-	_surrogate_type = None
+	_surrogate_type: SurrogateType = SurrogateType.DEFAULT
 
 	def __repr__(self):
 		return repr(self.as_annotation)

cvdatasets/dataset/part/surrogate.py

@@ -10,7 +10,10 @@ class SurrogateType(enum.Enum):
 	MIDDLE = enum.auto()
 	IMAGE = enum.auto()
 
+	DEFAULT = MIDDLE
+
 	def __call__(self, im, w, h, dtype=np.uint8):
+		im = utils.asarray(im)
 		if self is SurrogateType.BLANK:
 			return self._blank(im, w, h, dtype=dtype)
 
@@ -28,7 +31,7 @@ class SurrogateType(enum.Enum):
 		return np.zeros((h, w, c), dtype=dtype)
 
 	def _image(self, im, w, h, dtype):
-		_part_surrogate = resize(utils.asarray(im), (h, w),
+		_part_surrogate = resize(im, (h, w),
 			mode="constant",
 			anti_aliasing=True,
 			preserve_range=True)

cvdatasets/utils/__init__.py

@@ -62,6 +62,24 @@ class _MetaInfo(object):
 			setattr(self, name, value)
 		self.structure = []
 
+def retry_operation(n_retries, func, *args, **kwargs):
 
-from .image import asarray, dimensions, rescale
-from .dataset import new_iterator
+	if n_retries <= 0:
+		return func(*args, **kwargs)
+
+	error = None
+	for i in range(n_retries):
+		try:
+			return func(*args, **kwargs)
+		except Exception as e:
+			error = e
+
+	raise RuntimeError(f"Operation {func.__name__} failed after {n_retries} n_retries! ({error})")
+
+
+from cvdatasets.utils.dataset import new_iterator
+from cvdatasets.utils.image import asarray
+from cvdatasets.utils.image import read_image
+from cvdatasets.utils.transforms import color_jitter
+from cvdatasets.utils.transforms import dimensions
+from cvdatasets.utils.transforms import rescale

cvdatasets/utils/dataset.py

@@ -5,6 +5,12 @@ import warnings
 def _format_kwargs(kwargs):
 	return " ".join([f"{key}={value}" for key, value in kwargs.items()])
 
+def _uuid_check(uuids):
+	""" Checks whether the ids are unique """
+
+	assert len(np.unique(uuids)) == len(uuids), \
+		"UUIDs are not unique!"
+
 def new_iterator(data, n_jobs, batch_size, repeat=True, shuffle=True, n_prefetch=2):
 	from chainer.iterators import SerialIterator, MultiprocessIterator
 
@@ -17,6 +23,8 @@ def new_iterator(data, n_jobs, batch_size, repeat=True, shuffle=True, n_prefetch
 			pass
 
 		input_shape = getattr(data, "size", (512, 512))
+		n_parts = getattr(data, "n_parts", 1)
+
 		if isinstance(input_shape, int):
 			input_shape = (input_shape, input_shape)
 		elif not isinstance(input_shape, tuple):
@@ -27,8 +35,8 @@ def new_iterator(data, n_jobs, batch_size, repeat=True, shuffle=True, n_prefetch
 				input_shape = (512, 512)
 
 		shared_mem_shape = (3,) + input_shape
-		shared_mem = np.zeros(shared_mem_shape, dtype=np.float32).nbytes
-		logging.info(f"Using {shared_mem / 1024**2: .3f} MiB of shared memory")
+		shared_mem = (n_parts+1) * np.zeros(shared_mem_shape, dtype=np.float32).nbytes
+		logging.info(f"Using {batch_size * shared_mem / 1024**2: .3f} MiB of shared memory")
 
 		it_kwargs = dict(
 			n_processes=n_jobs,

cvdatasets/utils/image.py

@@ -1,39 +1,21 @@
 import numpy as np
-from PIL.Image import Image as PIL_Image
 
-def rescale(im, coords, rescale_size, center_cropped=True, no_offset=False):
-	h, w, c = dimensions(im)
+from os.path import isfile
+from PIL import Image
 
-	offset = 0
-	if center_cropped:
-		_min_val = min(w, h)
-		wh = np.array([_min_val, _min_val])
-		if not no_offset:
-			offset = (np.array([w, h]) - wh) / 2
-	else:
-		wh = np.array([w, h])
+from cvdatasets.utils import retry_operation
 
-	scale = wh / rescale_size
-	return coords * scale + offset
+def read_image(im_path, n_retries=5):
+	assert isfile(im_path), "Image \"{}\" does not exist!".format(im_path)
+	return retry_operation(n_retries, Image.open, im_path, mode="r")
 
-def dimensions(im):
-	if isinstance(im, np.ndarray):
-		if im.ndim != 3:
-			import pdb; pdb.set_trace()
-		assert im.ndim == 3, "Only RGB images are currently supported!"
-		return im.shape
-	elif isinstance(im, PIL_Image):
-		w, h = im.size
-		c = len(im.getbands())
-		# assert c == 3, "Only RGB images are currently supported!"
-		return h, w, c
-	else:
-		raise ValueError("Unknown image instance ({})!".format(type(im)))
 
 def asarray(im, dtype=np.uint8):
 	if isinstance(im, np.ndarray):
 		return im.astype(dtype)
-	elif isinstance(im, PIL_Image):
+
+	elif isinstance(im, Image.Image):
 		return np.asarray(im, dtype=dtype)
+
 	else:
 		raise ValueError("Unknown image instance ({})!".format(type(im)))

cvdatasets/utils/transforms.py

@@ -0,0 +1,148 @@
+import numpy as np
+import random
+
+from PIL import Image
+from functools import partial
+
+def dimensions(im):
+	if isinstance(im, np.ndarray):
+		if im.ndim != 3:
+			import pdb; pdb.set_trace()
+		assert im.ndim == 3, "Only RGB images are currently supported!"
+		return im.shape
+
+	elif isinstance(im, Image.Image):
+		w, h = im.size
+		c = len(im.getbands())
+		# assert c == 3, "Only RGB images are currently supported!"
+		return h, w, c
+
+	else:
+		raise ValueError("Unknown image instance ({})!".format(type(im)))
+
+def rescale(im, coords, rescale_size, center_cropped=True, no_offset=False):
+	h, w, c = dimensions(im)
+
+	offset = 0
+	if center_cropped:
+		_min_val = min(w, h)
+		wh = np.array([_min_val, _min_val])
+		if not no_offset:
+			offset = (np.array([w, h]) - wh) / 2
+
+	else:
+		wh = np.array([w, h])
+
+	scale = wh / rescale_size
+	return coords * scale + offset
+
+####################
+### Source: https://github.com/chainer/chainercv/blob/b52c71d9cd11dc9efdd5aaf327fed1a99df94d10/chainercv/transforms/image/color_jitter.py
+####################
+
+
+def _grayscale(img, channel_order="RGB"):
+	"""
+		from https://scikit-image.org/docs/dev/api/skimage.color.html#skimage.color.rgb2gray:
+			Y = 0.2125 R + 0.7154 G + 0.0721 B
+	"""
+
+	if channel_order == "RGB":
+		return 0.2125 * img[0] + 0.7154 * img[1] + 0.0721 * img[2]
+
+	elif channel_order == "BGR":
+		return 0.0721 * img[0] + 0.7154 * img[1] + 0.2125 * img[2]
+
+	else:
+		raise ValueError(f"Unknown channel order: {channel_order}")
+
+
+def _blend(img_a, img_b, alpha):
+	return alpha * img_a + (1 - alpha) * img_b
+
+
+def _brightness(img, var):
+	alpha = 1 + np.random.uniform(-var, var)
+	return _blend(img, np.zeros_like(img), alpha), alpha
+
+
+def _contrast(img, var, **kwargs):
+	gray = _grayscale(img, **kwargs)[0].mean()
+
+	alpha = 1 + np.random.uniform(-var, var)
+	return _blend(img, gray, alpha), alpha
+
+
+def _saturation(img, var, **kwargs):
+	gray = _grayscale(img, **kwargs)
+
+	alpha = 1 + np.random.uniform(-var, var)
+	return _blend(img, gray, alpha), alpha
+
+
+def color_jitter(img, brightness=0.4, contrast=0.4,
+				 saturation=0.4, return_param=False,
+				 min_value=0,
+				 max_value=255,
+				 channel_order="RGB"):
+	"""Data augmentation on brightness, contrast and saturation.
+	Args:
+		img (~numpy.ndarray): An image array to be augmented. This is in
+			CHW and RGB format.
+		brightness (float): Alpha for brightness is sampled from
+			:obj:`unif(-brightness, brightness)`. The default
+			value is 0.4.
+		contrast (float): Alpha for contrast is sampled from
+			:obj:`unif(-contrast, contrast)`. The default
+			value is 0.4.
+		saturation (float): Alpha for contrast is sampled from
+			:obj:`unif(-saturation, saturation)`. The default
+			value is 0.4.
+		return_param (bool): Returns parameters if :obj:`True`.
+	Returns:
+		~numpy.ndarray or (~numpy.ndarray, dict):
+		If :obj:`return_param = False`,
+		returns an color jittered image.
+		If :obj:`return_param = True`, returns a tuple of an array and a
+		dictionary :obj:`param`.
+		:obj:`param` is a dictionary of intermediate parameters whose
+		contents are listed below with key, value-type and the description
+		of the value.
+		* **order** (*list of strings*): List containing three strings: \
+			:obj:`'brightness'`, :obj:`'contrast'` and :obj:`'saturation'`. \
+			They are ordered according to the order in which the data \
+			augmentation functions are applied.
+		* **brightness_alpha** (*float*): Alpha used for brightness \
+			data augmentation.
+		* **contrast_alpha** (*float*): Alpha used for contrast \
+			data augmentation.
+		* **saturation_alpha** (*float*): Alpha used for saturation \
+			data augmentation.
+	"""
+	funcs = list()
+	if brightness > 0:
+		funcs.append(('brightness', partial(_brightness, var=brightness)))
+	if contrast > 0:
+		funcs.append(('contrast', partial(_contrast, var=contrast, channel_order=channel_order)))
+	if saturation > 0:
+		funcs.append(('saturation', partial(_saturation, var=saturation, channel_order=channel_order)))
+	random.shuffle(funcs)
+
+	params = {'order': [key for key, val in funcs],
+			  'brightness_alpha': 1,
+			  'contrast_alpha': 1,
+			  'saturation_alpha': 1}
+	for key, func in funcs:
+		img, alpha = func(img)
+		params[key + '_alpha'] = alpha
+
+	if min_value is not None:
+		img = np.maximum(img, min_value)
+
+	if max_value is not None:
+		img = np.minimum(img, max_value)
+
+	if return_param:
+		return img, params
+	else:
+		return img

scripts/display.py

@@ -10,7 +10,31 @@ import matplotlib.pyplot as plt
 from argparse import ArgumentParser
 
 from cvdatasets import AnnotationType
-from utils import parser, plot_crops
+from utils import parser
+
+def plot_crops(crops, spec, spec_offset, scatter_mid=False, names=None):
+
+	n_crops = len(crops)
+	if n_crops == 0: return
+	rows = int(np.ceil(np.sqrt(n_crops)))
+	cols = int(np.ceil(n_crops / rows))
+	dx, dy = spec_offset
+
+	for i, crop in enumerate(crops):
+		x, y = np.unravel_index(i, (rows, cols))
+
+		ax = plt.subplot(spec[x+dx, y+dy])
+
+		if names is not None:
+			ax.set_title(names[i])
+
+		ax.imshow(crop)
+		ax.axis("off")
+
+		if scatter_mid:
+			middle_h, middle_w = crop.shape[0] / 2, crop.shape[1] / 2
+			ax.scatter(middle_w, middle_h, marker="x")
+
 
 def main(args):
 	# assert args.dataset in AnnotationType, \
@@ -54,25 +78,34 @@ def main(args):
 		logging.info(f"Showing only images {start} - {end}")
 		idxs = range(start, end)
 
+
 	for i in idxs:
 		im, parts, label = data[i]
+		n_parts = len(parts)
+
+		assert n_parts != 0
+		rows = int(np.ceil(np.sqrt(n_parts)))
+		cols = int(np.ceil(n_parts / rows))
+		factor = 3 if args.rnd else 2
+		grid_spec = plt.GridSpec(rows, factor * cols)
+
+		fig = plt.figure()
 
-		fig1, axs = plt.subplots(1, 1, figsize=(16,9))
-		axs = [axs]
+		im_ax = plt.subplot(grid_spec[:, :cols])
 
-		axs[0].axis("off")
-		axs[0].set_title("Visible Parts")
-		axs[0].imshow(im)
+		im_ax.axis("off")
+		im_ax.set_title("Visible Parts")
+		im_ax.imshow(im)
 
 		if not args.crop_to_bb and not args.no_bboxes:
-			data.plot_bounding_box(i, axs[0])
+			data.plot_bounding_box(i, im_ax)
 
 		# axs[1].axis("off")
 		# axs[1].set_title("{}selected parts".format("randomly " if args.rnd else ""))
 		# axs[1].imshow(parts.reveal(im, ratio=data.ratio))
 
 		if not args.no_parts:
-			parts.plot(im=im, ax=axs[0], ratio=data.ratio, linewidth=3)
+			parts.plot(im=im, ax=im_ax, ratio=data.ratio, linewidth=3)
 			if data.uniform_parts:
 				crop_names = None
 			else:
@@ -83,10 +116,10 @@ def main(args):
 				parts.invert_selection()
 				action_crops = parts.visible_crops(im, ratio=data.ratio)
 
-			plot_crops(part_crops, f"{args.parts}: Selected parts", names=crop_names)
+			plot_crops(part_crops, grid_spec, spec_offset=(0, cols), names=crop_names)
 
 			if args.rnd:
-				plot_crops(action_crops, f"{args.parts}: Actions", names=crop_names)
+				plot_crops(action_crops, grid_spec, spec_offset=(0, 2*cols), names=crop_names)
 
 		plt.show()
 		plt.close()

scripts/display.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 source ${HOME}/.miniconda3/etc/profile.d/conda.sh
-conda activate ${ENV:-chainer6}
+conda activate ${CONDA_ENV:-chainer7cu11}
 
 PYTHON="python"
 

scripts/info_files/info.yml

@@ -6,46 +6,62 @@ MODEL_DIR: models
 ############ Existing models
 ### weights are used for fine-tuning
 MODELS:
-  efficientnet:    &efficientnet
-    folder: efficientnet
-    class_key: efficientnet
-    weights: model.imagenet.npz
-
-  inception_inat:    &inception_inat
-    folder: inception
-    class_key: inception
-    weights: model.inat.ckpt.npz
-
-  inception_imagenet:    &inception_inet
+  cvmodelz.InceptionV3:     &inception
     folder: inception
-    class_key: inception
-    weights: model.imagenet.ckpt.npz
-
-  inception:    &inception
-    <<: *inception_inat
-
-  inception_tf_inat:  &inception_tf_inat
-    folder: inception_tf
-    class_key: inception_tf
-    weights: inception_v3_iNat_299.ckpt
-
-  inception_tf_inet:  &inception_tf_inet
-    folder: inception_tf
-    class_key: inception_tf
-    weights: inception_v3_ILSVRC_299.ckpt
+    weights:
+      inat: model.inat.ckpt.npz
+      imagenet: model.imagenet.ckpt.npz
 
-  inception_tf:  &inception_tf
-    <<: *inception_tf_inat
-
-  resnet:       &resnet50
+  cvmodelz.ResNet50:     &resnet50
     folder: resnet
-    class_key: resnet
-    weights: model.npz
+    weights:
+      imagenet: model.npz
 
-  vgg19:       &vgg19
+  cvmodelz.VGG19:     &vgg19
     folder: vgg19
-    class_key: vgg19
-    weights: model.npz
+    weights:
+      imagenet: model.npz
+
+  # efficientnet:    &efficientnet
+  #   folder: efficientnet
+  #   class_key: efficientnet
+  #   weights: model.imagenet.npz
+
+  # inception_inat:    &inception_inat
+  #   folder: inception
+  #   class_key: inception
+  #   weights: model.inat.ckpt.npz
+
+  # inception_imagenet:    &inception_inet
+  #   folder: inception
+  #   class_key: inception
+  #   weights: model.imagenet.ckpt.npz
+
+  # inception:
+  #   <<: *inception_inat
+
+  # inception_tf_inat:  &inception_tf_inat
+  #   folder: inception_tf
+  #   class_key: inception_tf
+  #   weights: inception_v3_iNat_299.ckpt
+
+  # inception_tf_inet:  &inception_tf_inet
+  #   folder: inception_tf
+  #   class_key: inception_tf
+  #   weights: inception_v3_ILSVRC_299.ckpt
+
+  # inception_tf:  &inception_tf
+  #   <<: *inception_tf_inat
+
+  # resnet:       &resnet50
+  #   folder: resnet
+  #   class_key: resnet
+  #   weights: model.npz
+
+  # vgg19:       &vgg19
+  #   folder: vgg19
+  #   class_key: vgg19
+  #   weights: model.npz
 
 ############ Existing Datasets
 DATASETS:
@@ -54,6 +70,8 @@ DATASETS:
     folder: ImageNet
     annotations: "BJOERN"
     annotation_type: FOLDER
+    train_images: ILSVRC2012_img_train
+    val_images: ILSVRC2012_img_val
     n_classes: 1000
 
   IMAGENET_TOP_INAT20: &inet_top_inat20
@@ -67,6 +85,11 @@ DATASETS:
     annotation_type: FILE_LIST
     n_classes: 200
 
+  CUB10:         &cub10
+    <<: *cub200
+    annotations: "cub10"
+    n_classes: 10
+
   CUB200_2FOLD:   &cub200_2fold
     <<: *cub200
     folder: birds/cub200_2fold
@@ -110,6 +133,15 @@ DATASETS:
     annotation_type: FILE_LIST
     n_classes: 102
 
+  HERBA19:         &herba19
+    folder: herbarium
+    annotations: "2019"
+    annotation_type: FOLDER
+    train_images: small-train
+    val_images: small-validation
+    test_images: small-test
+    n_classes: 683
+
   INAT20:         &inat20
     folder: inat
     annotations: "2020/PLAIN"
@@ -219,25 +251,25 @@ PART_TYPES:
     annotations: L1_pred
     feature_suffix: _5parts_L1_pred
     rescale_size: !!int 427
-    scales: []
+    scales: [-1]
 
   L1_full:        &parts_l1f
     annotations: L1_full
     feature_suffix: _5parts_L1_full
     rescale_size: !!int 427
-    scales: []
+    scales: [-1]
 
   NTS:        &parts_nts
     annotations: NTS
     feature_suffix: _7parts_nts
     rescale_size: !!int 448
-    scales: []
+    scales: [-1]
 
   NTS2:        &parts_nts2
     annotations: NTS2
     feature_suffix: _5parts_nts
     rescale_size: !!int 448
-    scales: []
+    scales: [-1]
 
 PARTS:
   # all <DATASET>_<PART_TYPES> combinations are created implicitely.

scripts/tests.sh

@@ -4,4 +4,4 @@ source ${_root}/scripts/config.sh
 
 export BASE_DIR="${_root}/tests"
 
-$PYTHON ${BASE_DIR}/main.py $@
+python ${BASE_DIR}/main.py $@

scripts/utils/__init__.py

@@ -1,25 +0,0 @@
-import numpy as np
-import matplotlib.pyplot as plt
-
-
-def plot_crops(crops, title, scatter_mid=False, names=None):
-
-	n_crops = len(crops)
-	if n_crops == 0: return
-	rows = int(np.ceil(np.sqrt(n_crops)))
-	cols = int(np.ceil(n_crops / rows))
-
-	fig, axs = plt.subplots(rows, cols, figsize=(16,9))
-	fig.suptitle(title, fontsize=16)
-	[axs[np.unravel_index(i, axs.shape)].axis("off") for i in range(cols*rows)]
-
-	for i, crop in enumerate(crops):
-		ax = axs[np.unravel_index(i, axs.shape)]
-		if names is not None:
-			ax.set_title(names[i])
-		ax.imshow(crop)
-		if scatter_mid:
-			middle_h, middle_w = crop.shape[0] / 2, crop.shape[1] / 2
-			ax.scatter(middle_w, middle_h, marker="x")
-
-

tests/test_annotations.py

@@ -13,7 +13,6 @@ from cvdatasets import FileListAnnotations
 from cvdatasets.utils import read_info_file
 
 class MockAnnotation(FileListAnnotations):
-	name = "MOCK"
 	index_offset = 0
 
 	@property
@@ -107,7 +106,8 @@ class AnnotationTest(BaseAnnotationTest):
 			labels=[i % 5 for i in range(10)],
 			split=[int(i < 5) for i in range(10)],
 			bboxes=True,
-			n_parts=5
+			n_parts=5,
+			annot_params=dict(dataset_key="MOCK")
 		)
 		annot = self.create_annotations(**_annotation_params)
 
@@ -118,7 +118,8 @@ class AnnotationTest(BaseAnnotationTest):
 			labels=[i % 5 for i in range(10)],
 			split=[int(i < 5) for i in range(10)],
 			bboxes=True,
-			n_parts=5
+			n_parts=5,
+			annot_params=dict(dataset_key="MOCK")
 		)
 		annot = self.create_annotations(**_annotation_params)