Resolve "Flask-SQLAlchemy"

.coveragerc

@@ -0,0 +1,19 @@
+[run]
+source = pycs
+
+[report]
+
+# Regexes for lines to exclude from consideration
+exclude_lines =
+	# Have to re-enable the standard pragma
+	pragma: no cover
+
+	# Don't complain if tests don't hit defensive assertion code:
+	raise AssertionError
+	raise NotImplementedError
+	pass
+
+	# Don't complain if non-runnable code isn't run:
+	if 0:
+	if __name__ == .__main__.:
+

.gitignore

@@ -31,6 +31,7 @@ __pycache__/
 
 # coverage
 .coverage
+htmlcov/
 
 # projects and models
 /projects/

.gitlab-ci.yml

@@ -33,18 +33,13 @@ webui:
     - python -V
     - python -m venv env
     - source env/bin/activate
-    - pip install numpy opencv-python Pillow scipy
-    - pip install eventlet flask python-socketio
-    - pip install coverage pylint
+    - apt-get update
+    - apt-get install -y libgl1-mesa-glx
+    - pip install -r requirements.txt
+    - pip install -r requirements.dev.txt
   script:
-    - coverage run --source=pycs/ -m unittest discover test/
-    - "pylint --fail-under=9.5
-         --disable=duplicate-code
-         --disable=missing-module-docstring
-         --disable=too-many-instance-attributes
-         --extension-pkg-whitelist=cv2
-         --module-rgx='^[A-Za-z0-9]+$' --class-rgx='^[A-Za-z0-9]+$'
-         app.py pycs"
+    - make run_coverage
+    - make run_pylint
 
 tests_3.6:
   stage: test
@@ -71,9 +66,6 @@ tests_3.9:
   stage: test
   image: python:3.9
   <<: *python_test_definition
-  after_script:
-    - source env/bin/activate
-    - coverage report -m
 
 tests_3.10:
   stage: test

.pylintrc

@@ -0,0 +1,628 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
+extension-pkg-whitelist=cv2
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold to be exceeded before program exits with error.
+fail-under=9.5
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the ignore-list. The
+# regex matches against paths.
+ignore-paths=
+
+# Files or directories matching the regex patterns are skipped. The regex
+# matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=pylint_flask, pylint_flask_sqlalchemy
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        duplicate-code,
+        missing-module-docstring,
+        too-many-instance-attributes
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'error', 'warning', 'refactor', and 'convention'
+# which contain the number of messages in each category, as well as 'statement'
+# which is the total number of statements analyzed. This score is used by the
+# global evaluation report (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
+
+
+[LOGGING]
+
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
+
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style.
+#class-const-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+class-rgx=^[A-Za-z0-9]+$
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           id,
+           x, y, w, h, x0, y0, x1, y1,
+           Run,
+           _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+module-rgx=^[A-Za-z0-9\_]+$
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Ignore function signatures when computing similarities.
+ignore-signatures=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=app.logger
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,scoped_session
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+# Regular expression of note tags to take in consideration.
+#notes-rgx=
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it work,
+# install the 'python-enchant' package.
+spelling-dict=
+
+# List of comma separated words that should be considered directives if they
+# appear and the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[IMPORTS]
+
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=
+
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
+ext-import-graph=
+
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
+import-graph=
+
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
+
+
+[CLASSES]
+
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp,
+                      __post_init__
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "BaseException, Exception".
+overgeneral-exceptions=BaseException,
+                       Exception

Makefile

@@ -0,0 +1,20 @@
+run:
+	python app.py
+
+run_webui:
+	@cd webui && npm run serve
+
+install:
+	@echo "INSTALL MISSING!"
+
+run_tests:
+	python -m unittest discover tests/
+
+run_coverage:
+	@PYTHONWARNINGS="ignore::ResourceWarning" coverage run -m unittest discover tests/
+	@coverage run --concurrency=eventlet --append -m unittest tests.client.pipeline_tests
+	coverage html
+	coverage report -m
+
+run_pylint:
+	pylint --rcfile=.pylintrc app.py pycs

app.py

@@ -1,35 +1,9 @@
 #!/usr/bin/env python
 
-from json import load
-from os import mkdir, path
-
-from pycs.database.Database import Database
+from pycs import app
+from pycs import settings
 from pycs.frontend.WebServer import WebServer
-from pycs.jobs.JobRunner import JobRunner
-from pycs.util.PipelineCache import PipelineCache
 
 if __name__ == '__main__':
-    # load settings
-    print('- load settings')
-    with open('settings.json', 'r') as file:
-        settings = load(file)
-
-    # create projects folder
-    if not path.exists('projects/'):
-        mkdir('projects/')
-
-    # initialize database
-    print('- load database')
-    database = Database('data.sqlite3')
-
-    # start job runner
-    print('- start job runner')
-    jobs = JobRunner()
-
-    # create pipeline cache
-    print('- create pipeline cache')
-    pipelines = PipelineCache(jobs)
-
-    # start web server
-    print('- start web server')
-    web_server = WebServer(settings, database, jobs, pipelines)
+    server = WebServer(app, settings)
+    server.run()

labels/flat_moth_label_provider/moth_labels.py

@@ -1,6 +1,7 @@
 import json
 import typing as T
 
+from pycs import app
 from pycs.interfaces.LabelProvider import LabelProvider
 from pathlib import Path
 
@@ -20,8 +21,9 @@ class FlatMothLabels(LabelProvider):
         labels = []
         for key, entries in self.mapping.items():
             display_name = f"{entries['genus']} {entries['species']} ({entries['kr']})"
-            print(key, display_name)
-            label = self.create_label(key, display_name)
+            reference = entries["kr"]
+            app.logger.debug(f"{reference} -> {display_name}")
+            label = self.create_label(reference, display_name)
             labels.append(label)
 
         return labels

labels/lepiforum_version_7/Lepiforums-Europaliste_Schmetterlinge_Version_7_Stand_2020_01_01_bearbeitet_GBrehm.csv

@@ -292,7 +292,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Hepialoidea	Hepialidae						64	Triodia	adriaticus	(OSTHELDER, 1931)						
 		Hepialoidea	Hepialidae						65	Triodia	amasinus	(HERRICH-SCHÄFFER, [1851])						
 		Hepialoidea	Hepialidae			D	C	A	69	Korscheltellus	fusconebulosa	(DE GEER, 1778)					D-CH-A	Adlerfarn-Wurzelbohrer
-		Hepialoidea	Hepialidae						75	Korscheltellus	castillanus	(OBERTHÜR, 1883)			11/11/2018	Neukombination: Pharmacis ==> Korscheltellus		
+		Hepialoidea	Hepialidae						76	Korscheltellus	castillanus	(OBERTHÜR, 1883)			11/11/2018	Neukombination: Pharmacis ==> Korscheltellus		
 		Hepialoidea	Hepialidae			D	C	A	67	Korscheltellus	lupulina	(LINNAEUS, 1758)					D-CH-A	Kleiner Hopfen-Wurzelbohrer
 		Hepialoidea	Hepialidae			D	C	A	70	Pharmacis	carna	([DENIS & SCHIFFERMÜLLER], 1775)					D-CH-A	Espers Alpen-Wurzelbohrer, Schwärzlicher Wurzelbohrer
 		Hepialoidea	Hepialidae						73	Pharmacis	claudiae	KRISTAL & HIRNEISEN, 1994						
@@ -2339,7 +2339,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Elachistidae	Elachistinae					01893a	Elachista	paracollitella	NEL & VARENNE, 2016						
 		Gelechioidea	Elachistidae	Elachistinae		D	?	A	1905	Elachista	dispilella	ZELLER, 1839					D-CH-A	Schwarzpunkt-Grasminierfalter
 		Gelechioidea	Elachistidae	Elachistinae		D	C		01905b	Elachista	distigmatella	FREY, 1859					D-CH-A	
-		Gelechioidea	Elachistidae	Elachistinae					01905c	Elachista	candidella	SRUOGA & SINEV, 2017			10/27/2017	neu beschriebene Art	Nordkaukasus	
+		Gelechioidea	Elachistidae	Elachistinae					01905d	Elachista	candidella	SRUOGA & SINEV, 2017			10/27/2017	neu beschriebene Art	Nordkaukasus	
 		Gelechioidea	Elachistidae	Elachistinae					1914	Elachista	fasciola	PARENTI, 1983						
 		Gelechioidea	Elachistidae	Elachistinae					01915a	Elachista	purella	SRUOGA, 2000		zu löschen!	10/8/2019	zu streichen: jetzt Synonym zu Elachista levasi		
 		Gelechioidea	Elachistidae	Elachistinae		?	C	A	1915	Elachista	festucicolella	ZELLER, 1853					D-CH-A	
@@ -2397,7 +2397,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Elachistidae	Elachistinae		D			01894a	Elachista	constitella	FREY, 1859					D-CH-A	
 		Gelechioidea	Elachistidae	Elachistinae					1898	Elachista	curonensis	TRAUGOTT-OLSEN, 1990						
 		Gelechioidea	Elachistidae	Elachistinae					1900	Elachista	deceptricula	STAUDINGER, 1880						
-		Gelechioidea	Elachistidae	Elachistinae					01900a	Elachista	conferta	KAILA, 2017			11/3/2017	neu beschriebene Art		
+		Gelechioidea	Elachistidae	Elachistinae					01900c	Elachista	conferta	KAILA, 2017			11/3/2017	neu beschriebene Art		
 		Gelechioidea	Elachistidae	Elachistinae					1907	Elachista	drenovoi	PARENTI, 1981						
 		Gelechioidea	Elachistidae	Elachistinae					1908	Elachista	dumosa	PARENTI, 1981						
 		Gelechioidea	Elachistidae	Elachistinae		D			1913	Elachista	exigua	PARENTI, 1978					D-CH-A	Südlicher Grasminierfalter
@@ -3997,7 +3997,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				03337g	Monochroa	aenigma	ANIKIN & PISKUNOV, 2018			8/7/2019	neu beschriebene Art		
 		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				[non-KR]	Monochroa	rebeli	(M. HERING, 1927)					Kanaren	
 		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				[non-KR]	Monochroa	deserta	PISKUNOV, 1990			8/3/2018	neu für Europa (Zypern)	Zypern	
-		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				03337d	Spiniphallellus	desertus	BIDZILYA & KARSHOLT, 2008						
+		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				03337f	Spiniphallellus	desertus	BIDZILYA & KARSHOLT, 2008						
 		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				03337e	Spiniphallellus	chrysotosella	JUNNILAINEN, 2016						
 		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini				3336	Eulamprotes	nigromaculella	(MILLIÈRE, 1872)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Anomologini	D	C	A	3339	Eulamprotes	wilkella	(LINNAEUS, 1758)					D-CH-A	Großer Silberstreifen-Palpenfalter
@@ -4274,7 +4274,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3610	Scrobipalpa	kasyi	POVOLNÝ, 1966			12/30/2017	1968 ==> 1966, Klammern um Autor und Jahr entfernt		
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03617a	Scrobipalpa	notata	(POVOLNÝ, 2001)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D	C	A	3580	Scrobipalpa	acuminatella	(SIRCOM, 1850)					D-CH-A	
-		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03580a	Scrobipalpa	skulei	HUEMER & KARSHOLT, 2010						
+		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03580c	Scrobipalpa	skulei	HUEMER & KARSHOLT, 2010						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini			A	3606	Scrobipalpa	hungariae	(STAUDINGER, 1871)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini			A	03580b	Scrobipalpa	adaptata	(POVOLNÝ, 2001)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D			3588	Scrobipalpa	brahmiella	(HEYDEN, 1862)					D-CH-A	
@@ -4283,7 +4283,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3582	Scrobipalpa	amseli	POVOLNÝ, 1966						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03607b	Scrobipalpa	hyssopi	NEL, 2003						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03614a	Scrobipalpa	montanella	(CHRÉTIEN, 1910)						
-		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03614a	Scrobipalpa	corleyi	HUEMER & KARSHOLT, 2010						
+		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03614b	Scrobipalpa	corleyi	HUEMER & KARSHOLT, 2010						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D	C	A	3590	Scrobipalpa	chrysanthemella	(E. HOFMANN, 1867)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D	C	A	3623	Scrobipalpa	proclivella	(FUCHS, 1886)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03607a	Scrobipalpa	frugifera	POVOLNÝ, 1969						
@@ -4336,7 +4336,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3593	Scrobipalpa	dagmaris	POVOLNÝ, 1987						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3635	Scrobipalpa	suasella	(CONSTANT, 1895)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03635a	Scrobipalpa	hendrikseni	HUEMER & KARSHOLT, 2010						
-		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03642a	Scrobipalpa	halimifolia	BIDZILYA & BUDASHKIN, 2011						
+		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03642c	Scrobipalpa	halimifolia	BIDZILYA & BUDASHKIN, 2011						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3637	Scrobipalpa	traganella	(CHRÉTIEN, 1915)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3586	Scrobipalpa	bazae	POVOLNÝ, 1977						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D	C	A	3584	Scrobipalpa	artemisiella	(TREITSCHKE, 1833)					D-CH-A	
@@ -4356,7 +4356,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D			3592	Scrobipalpa	costella	(HUMPHREYS & WESTWOOD, 1845)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini			A	3607	Scrobipalpa	hyoscyamella	(STAINTON, 1869)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				3601	Scrobipalpa	portosanctana	(STAINTON, 1859)						
-		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03640a	Scrobipalpa	vicaria	(MEYRICK, 1921)						
+		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03640b	Scrobipalpa	vicaria	(MEYRICK, 1921)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D	C	A	3619	Scrobipalpa	ocellatella	(BOYD, 1858)					D-CH-A	Rüben-Palpenmotte, Rübenmotte
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03623a	Scrobipalpa	pulchra	POVOLNÝ, 1967						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03600a	Scrobipalpa	gecko	(WALSINGHAM, 1911)						
@@ -4371,7 +4371,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03631a	Scrobipalpa	spergulariella	(CHRÉTIEN, 1910)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini	D		A	03627b	Scrobipalpa	salicorniae	(HERING, 1889)					D-CH-A	
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03627c	Scrobipalpa	halimioniella	HUEMER & KARSHOLT, 2010						
-		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03636a	Scrobipalpa	thymelaeae	(AMSEL, 1939)						
+		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03636b	Scrobipalpa	thymelaeae	(AMSEL, 1939)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03605a	Scrobipalpa	halymella	(MILLIÈRE, 1864)						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03589a	Scrobipalpa	camphorosmella	NEL, 1999						
 		Gelechioidea	Gelechiidae	Gelechiinae	Gnorimoschemini				03589b	Scrobipalpa	stabilis	POVOLNÝ, 1977						
@@ -5131,7 +5131,6 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Tortricoidea	Tortricidae	Tortricinae	Cochylini	D	C	A	4359	Cochylis	posterana	ZELLER, 1847					D-CH-A	
 		Tortricoidea	Tortricidae	Tortricinae	Cochylini				4360	Cochylis	defessana	(MANN, 1861)						
 		Tortricoidea	Tortricidae	Tortricinae	Cochylini				04360a	Cochylis	millierana	PEYERIMHOFF, 1887			10/21/2017	Ersatzseite für Cochylis sannitica		
-		Tortricoidea	Tortricidae	Tortricinae	Cochylini				04360a	Cochylis	sannitica	TREMATERRA, 1995		zu löschen!	10/21/2017	Löschen: Synonym zu Cochylis millierana		
 		Tortricoidea	Tortricidae	Tortricinae	Cochylini	D	C	A	4355	Cochylichroa	atricapitana	(STEPHENS, 1852)			9/18/2019	Neukombination: Cochylis ==> Cochylichroa	D-CH-A	Rosenfarbiger Schwarzkopfwickler
 		Tortricoidea	Tortricidae	Tortricinae	Cochylini				04360b	Rolandylis	maiana	(KEARFOTT, 1907)						
 		Tortricoidea	Tortricidae	Tortricinae	Cochylini	D			4362	Cryptocochylis	conjunctana	(MANN, 1864)					D-CH-A	
@@ -6503,7 +6502,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini	D	C	A	7152	Cyaniris	semiargus	(ROTTEMBURG, 1775)					D-CH-A	Rotklee-Bläuling
 		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini	D	C	A	7131	Agriades	optilete	(KNOCH, 1781)					D-CH-A	Hochmoor-Bläuling
 		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini				7138	Agriades	pyrenaica	(BOISDUVAL, 1840)						
-		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini				7138	Agriades	dardanus	(FREYER, [1843])			05.01.2019 und 16.02.2019	neu als Art akzeptiertes Taxon (zuvor Unterart von A. pyrenaica); Jahr in eckige Klammern gesetzt		
+		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini				7138a	Agriades	dardanus	(FREYER, [1843])			05.01.2019 und 16.02.2019	neu als Art akzeptiertes Taxon (zuvor Unterart von A. pyrenaica); Jahr in eckige Klammern gesetzt		
 		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini	D	C	A	7139	Agriades	glandon	(DE PRUNNER, 1798)					D-CH-A	Dunkler Alpenbläuling
 		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini				07139a	Agriades	aquilo	(BOISDUVAL, 1832)						
 		Papilionoidea	Lycaenidae	Lycaeninae	Polyommatini				07139b	Agriades	zullichi	HEMMING, 1933						
@@ -7011,7 +7010,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Pyraloidea	Pyralidae	Phycitinae	Phycitini				05753b	Jerichoa	mediterranella	(AMSEL, 1935)			12/19/2019	neu für Europa		
 		Pyraloidea	Pyralidae	Phycitinae	Phycitini			A	5850	Oxybia	transversella	(DUPONCHEL, 1836)					D-CH-A	
 		Pyraloidea	Pyralidae	Phycitinae	Phycitini				5730	Denticera	divisella	(DUPONCHEL, [1843])			12/24/2018	1842 ==> [1843] (Jahr korrigiert und eckige Klammern um Jahr eingefügt)		
-		Pyraloidea	Pyralidae	Phycitinae	Phycitini				05730a	Sardzea	diviselloides	AMSEL, 1961		zu löschen!	22.09.2017 und 20.12.2019	Neu für Europa; wieder für Europa gestrichen (war Fehlbestimmung von Denticera divisella)		
+		Pyraloidea	Pyralidae	Phycitinae	Phycitini				05730b	Sardzea	diviselloides	AMSEL, 1961		zu löschen!	22.09.2017 und 20.12.2019	Neu für Europa; wieder für Europa gestrichen (war Fehlbestimmung von Denticera divisella)		
 		Pyraloidea	Pyralidae	Phycitinae	Phycitini	D	C	A	5751	Oncocera	semirubella	(SCOPOLI, 1763)					D-CH-A	
 		Pyraloidea	Pyralidae	Phycitinae	Phycitini				5789	Pseudophycita	deformella	(MÖSCHLER, 1866)						
 		Pyraloidea	Pyralidae	Phycitinae	Phycitini				05789a	Oedilepia	polygraphella	(JOANNIS, 1927)		Zitat noch nicht überprüft	12/19/2019	neu für Europa		
@@ -7717,7 +7716,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Pyraloidea	Crambidae	Pyraustinae					6581	Loxostege	ephippialis	(ZETTERSTEDT, 1839)						
 		Pyraloidea	Crambidae	Pyraustinae					06592a	Loxostege	expansalis	(EVERSMANN, 1852)						
 		Pyraloidea	Crambidae	Pyraustinae		D	C	A	6577	Loxostege	sticticalis	(LINNAEUS, [1760])			1/6/2019	1761 ==> [1760]	D-CH-A	Rübenzünsler
-		Pyraloidea	Crambidae	Pyraustinae					6577	Loxostege	ayhanana	KEMAL & KOÇAK, 2017			8/18/2018	neu für Europa		
+		Pyraloidea	Crambidae	Pyraustinae					6577a	Loxostege	ayhanana	KEMAL & KOÇAK, 2017			8/18/2018	neu für Europa		
 		Pyraloidea	Crambidae	Pyraustinae			C		6590	Ecpyrrhorrhoe	diffusalis	(GUENÉE, 1854)					D-CH-A	
 		Pyraloidea	Crambidae	Pyraustinae		D	C	A	6588	Ecpyrrhorrhoe	rubiginalis	(HÜBNER, 1796)			11/18/2018	[1796] ==> 1796	D-CH-A	
 		Pyraloidea	Crambidae	Pyraustinae					6583	Achyra	nudalis	(HÜBNER, 1796)			11/18/2018	[1796] ==> 1796		
@@ -7893,7 +7892,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Pyraloidea	Crambidae	Spilomelinae					06720b	Conogethes	punctiferalis	(GUENÉE, 1854)						
 		Pyraloidea	Crambidae	Spilomelinae		e			06720g	Sufetula	diminutalis	(WALKER, 1866)					D-CH-A	
 		Pyraloidea	Crambidae	Spilomelinae					06720h	Zebronia	phenice	(STOLL, [1782])			5/8/2017	eckige Klammern um Jahr		
-		Pyraloidea	Crambidae	Spilomelinae					06720i	Agathodes	designalis	GUENÉE, 1854			2/22/2019	neu für Europa		
+		Pyraloidea	Crambidae	Spilomelinae					06720j	Agathodes	designalis	GUENÉE, 1854			2/22/2019	neu für Europa		
 		Drepanoidea	Cimeliidae						6872	Axia	margarita	(HÜBNER, [1809-1813])						
 		Drepanoidea	Cimeliidae						6873	Axia	napoleona	SCHAWERDA, 1926						
 		Drepanoidea	Cimeliidae						6874	Axia	nesiota	REISSER, 1962						
@@ -8021,7 +8020,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Geometroidea	Geometridae	Desmobathrinae					7956	Myinodes	interpunctaria	(HERRICH-SCHÄFFER, 1839)						
 		Geometroidea	Geometridae	Desmobathrinae					07956b	Myinodes	constantina	HAUSMANN, 1994						
 		Geometroidea	Geometridae	Desmobathrinae					07956a	Myinodes	shohami	HAUSMANN, 1994						
-		Geometroidea	Geometridae	Desmobathrinae					07956b	Drepanopterula	limaria	(CHRISTOPH, 1885)			6/29/2019	Neu für Europa s.l. (russischer Kaukasus)	Nordkaukasus	
+		Geometroidea	Geometridae	Desmobathrinae					07956c	Drepanopterula	limaria	(CHRISTOPH, 1885)			6/29/2019	Neu für Europa s.l. (russischer Kaukasus)	Nordkaukasus	
 		Geometroidea	Geometridae	Geometrinae	Heliotheini				8009	Heliothea	discoidaria	BOISDUVAL, 1840						
 		Geometroidea	Geometridae	Geometrinae	Pseudoterpnini	D	C	A	7961	Aplasta	ononaria	(FUESSLY, 1783)					D-CH-A	Hauhechelspanner
 		Geometroidea	Geometridae	Geometrinae	Pseudoterpnini				7994	Holoterpna	pruinosata	(STAUDINGER, 1898)						
@@ -9486,7 +9485,7 @@ nur lokal	8	Noctuoidea	Noctuidae	Noctuinae	Noctuini	D	C	A	10098	Noctua	interposi
 		Noctuoidea	Nolidae	Chloephorinae					10457	Earias	albovenosana	OBERTHÜR, 1917						
 		Noctuoidea	Nolidae	Chloephorinae					10460	Earias	insulana	(BOISDUVAL, 1833)						
 		Noctuoidea	Nolidae	Chloephorinae					10461a	Earias	biplaga	WALKER, 1866						
-		Noctuoidea	Nolidae	Chloephorinae			e		10461a	Earias	roseifera	BUTLER, 1881			24.11.2018 und 27.05.2019	neu für Europa; CH-Einzelfähnchen	D-CH-A	
+		Noctuoidea	Nolidae	Chloephorinae			e		10461c	Earias	roseifera	BUTLER, 1881			24.11.2018 und 27.05.2019	neu für Europa; CH-Einzelfähnchen	D-CH-A	
 		Noctuoidea	Nolidae	Chloephorinae		e			10461b	Earias	vittella	(FABRICIUS, 1794)					D-CH-A	
 		Noctuoidea	Nolidae	Chloephorinae					9172	Xanthodes	albago	(FABRICIUS, 1794)						
 		Noctuoidea	Nolidae	Chloephorinae					10453	Pardoxia	graellsii	(FEISTHAMEL, 1837)						

labels/lepiforum_version_7/Provider.py

@@ -42,6 +42,7 @@ class Provider(LabelProvider):
             hierarchy_levels = (('family', 'Familie'),
                                 ('genus', 'Gattung'))
 
+        parents = set()
         for entry in entries:
             entry = entry.__dict__
             parent_reference = None
@@ -50,12 +51,16 @@ class Provider(LabelProvider):
             for level, level_name in hierarchy_levels:
                 if entry[level] is not None:
                     reference, name = f'{level}_{entry[level].lower()}', entry[level]
-                    result.append(self.create_label(reference, name, parent_reference, level_name))
+
+                    # parents should be added once
+                    if reference not in parents:
+                        result.append(self.create_label(reference, name, parent_reference, level_name))
+                        parents.add(reference)
 
                     parent_reference = reference
 
             # add element
-            if entry['kr_number'].isnumeric():
+            if entry['kr_number'].isalnum():
                 name = f'{entry["genus"]} {entry["species"]} ({entry["kr_number"]})'
                 reference = entry['kr_number']
             else:

migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

migrations/alembic.ini

@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = INFO
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

migrations/env.py

@@ -0,0 +1,91 @@
+from __future__ import with_statement
+
+import logging
+from logging.config import fileConfig
+
+from flask import current_app
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+config.set_main_option(
+    'sqlalchemy.url',
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
+        '%', '%%'))
+target_metadata = current_app.extensions['migrate'].db.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    connectable = current_app.extensions['migrate'].db.get_engine()
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

migrations/versions/b03df3e31b8d_.py

@@ -0,0 +1,126 @@
+"""empty message
+
+Revision ID: b03df3e31b8d
+Revises: 
+Create Date: 2021-08-11 12:46:17.757283
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b03df3e31b8d'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('label_provider',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=True),
+    sa.Column('root_folder', sa.String(), nullable=False),
+    sa.Column('configuration_file', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('root_folder', 'configuration_file')
+    )
+    op.create_table('model',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=True),
+    sa.Column('root_folder', sa.String(), nullable=False),
+    sa.Column('supports_encoded', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('root_folder')
+    )
+    op.create_table('project',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=True),
+    sa.Column('created', sa.DateTime(), nullable=False),
+    sa.Column('model_id', sa.Integer(), nullable=True),
+    sa.Column('label_provider_id', sa.Integer(), nullable=True),
+    sa.Column('root_folder', sa.String(), nullable=False),
+    sa.Column('external_data', sa.Boolean(), nullable=False),
+    sa.Column('data_folder', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['label_provider_id'], ['label_provider.id'], ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['model_id'], ['model.id'], ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('root_folder')
+    )
+    op.create_index(op.f('ix_project_created'), 'project', ['created'], unique=False)
+    op.create_table('collection',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('project_id', sa.Integer(), nullable=False),
+    sa.Column('reference', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=True),
+    sa.Column('position', sa.Integer(), nullable=False),
+    sa.Column('autoselect', sa.Boolean(), nullable=False),
+    sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('project_id', 'reference')
+    )
+    op.create_table('label',
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('project_id', sa.Integer(), nullable=False),
+    sa.Column('parent_id', sa.Integer(), nullable=True),
+    sa.Column('created', sa.DateTime(), nullable=False),
+    sa.Column('reference', sa.String(), nullable=True),
+    sa.Column('hierarchy_level', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['parent_id'], ['label.id'], ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('project_id', 'reference')
+    )
+    op.create_index(op.f('ix_label_created'), 'label', ['created'], unique=False)
+    op.create_table('file',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('uuid', sa.String(), nullable=False),
+    sa.Column('extension', sa.String(), nullable=False),
+    sa.Column('type', sa.String(), nullable=False),
+    sa.Column('size', sa.Integer(), nullable=False),
+    sa.Column('created', sa.DateTime(), nullable=False),
+    sa.Column('path', sa.String(), nullable=False),
+    sa.Column('frames', sa.Integer(), nullable=True),
+    sa.Column('fps', sa.Float(), nullable=True),
+    sa.Column('project_id', sa.Integer(), nullable=False),
+    sa.Column('collection_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('project_id', 'path')
+    )
+    op.create_index(op.f('ix_file_created'), 'file', ['created'], unique=False)
+    op.create_table('result',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('file_id', sa.Integer(), nullable=False),
+    sa.Column('origin', sa.String(), nullable=False),
+    sa.Column('type', sa.String(), nullable=False),
+    sa.Column('label_id', sa.Integer(), nullable=True),
+    sa.Column('data_encoded', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['file_id'], ['file.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['label_id'], ['label.id'], ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('result')
+    op.drop_index(op.f('ix_file_created'), table_name='file')
+    op.drop_table('file')
+    op.drop_index(op.f('ix_label_created'), table_name='label')
+    op.drop_table('label')
+    op.drop_table('collection')
+    op.drop_index(op.f('ix_project_created'), table_name='project')
+    op.drop_table('project')
+    op.drop_table('model')
+    op.drop_table('label_provider')
+    # ### end Alembic commands ###

models/fixed_model/Pipeline.py

@@ -2,6 +2,7 @@ from json import dump, load
 from os import path
 from time import sleep
 
+from pycs import app
 from pycs.interfaces.MediaFile import MediaFile
 from pycs.interfaces.MediaStorage import MediaStorage
 from pycs.interfaces.Pipeline import Pipeline as Interface
@@ -9,14 +10,14 @@ from pycs.interfaces.Pipeline import Pipeline as Interface
 
 class Pipeline(Interface):
     def __init__(self, root_folder, distribution):
-        print('fmv1 init')
+        app.logger.debug('fmv1 init')
         self.root_folder = root_folder
 
     def close(self):
-        print('fmv1 close')
+        app.logger.debug('fmv1 close')
 
     def execute(self, storage: MediaStorage, file: MediaFile):
-        print('fmv1 execute')
+        app.logger.debug('fmv1 execute')
 
         data_file = path.join(self.root_folder, 'data.json')
         if path.exists(data_file):
@@ -33,7 +34,7 @@ class Pipeline(Interface):
                     file.set_image_label(r['label'], r['frame'])
 
     def fit(self, storage: MediaStorage):
-        print('fmv1 fit')
+        app.logger.debug('fmv1 fit')
 
         for i in range(10):
             yield i / 10

models/haarcascade_frontalface_default/Pipeline.py

@@ -4,6 +4,7 @@ from urllib.request import urlretrieve
 
 import cv2
 
+from pycs import app
 from pycs.interfaces.MediaFile import MediaFile
 from pycs.interfaces.MediaStorage import MediaStorage
 from pycs.interfaces.Pipeline import Pipeline as Interface
@@ -13,7 +14,7 @@ class Pipeline(Interface):
     URL = 'https://raw.githubusercontent.com/opencv/opencv/master/data/haarcascades/haarcascade_frontalface_default.xml'
 
     def __init__(self, root_folder, distribution):
-        print('hcffdv1 init')
+        app.logger.debug('hcffdv1 init')
 
         # get path to xml file
         xml_file = path.join(root_folder, 'haarcascade_frontalface_default.xml')
@@ -26,7 +27,7 @@ class Pipeline(Interface):
         self.face_cascade = cv2.CascadeClassifier(xml_file)
 
     def close(self):
-        print('hcffdv1 close')
+        app.logger.debug('hcffdv1 close')
 
     def collections(self) -> List[dict]:
         return [
@@ -35,7 +36,7 @@ class Pipeline(Interface):
         ]
 
     def execute(self, storage: MediaStorage, file: MediaFile):
-        print('hcffdv1 execute')
+        app.logger.debug('hcffdv1 execute')
 
         # load file and analyze frames
         found = False

models/moth_scanner/configuration.json

@@ -26,6 +26,7 @@
     "model_type": "cvmodelz.InceptionV3",
     "input_size": 299,
     "weights": "classifier.npz",
-    "n_classes": 200
+    "n_classes": 200,
+    "mapping": "mapping.json"
   }
 }

models/moth_scanner/mapping.json

@@ -0,0 +1,2202 @@
+{
+  "0": {
+    "class_name": "abrostola_tripartita",
+    "kr": "9091",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Plusiinae",
+    "tribus": "Abrostolini",
+    "genus": "Abrostola",
+    "species": "tripartita",
+    "rarity": ""
+  },
+  "1": {
+    "class_name": "abrostola_triplasia",
+    "kr": "9093",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Plusiinae",
+    "tribus": "Abrostolini",
+    "genus": "Abrostola",
+    "species": "triplasia",
+    "rarity": ""
+  },
+  "2": {
+    "class_name": "achlya_flavicornis",
+    "kr": "7498",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Achlya",
+    "species": "flavicornis",
+    "rarity": ""
+  },
+  "3": {
+    "class_name": "acronicta_aceris",
+    "kr": "8778",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Acronictinae",
+    "tribus": "",
+    "genus": "Acronicta",
+    "species": "aceris",
+    "rarity": "5"
+  },
+  "4": {
+    "class_name": "acronicta_leporina",
+    "kr": "8779",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Acronictinae",
+    "tribus": "",
+    "genus": "Acronicta",
+    "species": "leporina",
+    "rarity": "3"
+  },
+  "5": {
+    "class_name": "acronicta_megacephala",
+    "kr": "8780",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Acronictinae",
+    "tribus": "",
+    "genus": "Acronicta",
+    "species": "megacephala",
+    "rarity": ""
+  },
+  "6": {
+    "class_name": "acronicta_psi",
+    "kr": "8777",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Acronictinae",
+    "tribus": "",
+    "genus": "Acronicta",
+    "species": "psi",
+    "rarity": "5"
+  },
+  "7": {
+    "class_name": "aglia_tau",
+    "kr": "6788",
+    "superfamily": "Bombycoidea",
+    "family": "Saturniidae",
+    "subfamily": "Agliinae",
+    "tribus": "",
+    "genus": "Aglia",
+    "species": "tau",
+    "rarity": "4"
+  },
+  "8": {
+    "class_name": "agriopis_aurantiaria",
+    "kr": "7695",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Agriopis",
+    "species": "aurantiaria",
+    "rarity": ""
+  },
+  "9": {
+    "class_name": "agriopis_leucophaearia",
+    "kr": "7693",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Agriopis",
+    "species": "leucophaearia",
+    "rarity": ""
+  },
+  "10": {
+    "class_name": "agriopis_marginaria",
+    "kr": "7696",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Agriopis",
+    "species": "marginaria",
+    "rarity": ""
+  },
+  "11": {
+    "class_name": "agrochola_litura",
+    "kr": "9586",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Agrochola",
+    "species": "litura",
+    "rarity": "4"
+  },
+  "12": {
+    "class_name": "agrochola_lunosa",
+    "kr": "9591",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Agrochola",
+    "species": "lunosa",
+    "rarity": ""
+  },
+  "13": {
+    "class_name": "agrochola_macilenta",
+    "kr": "9571",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Agrochola",
+    "species": "macilenta",
+    "rarity": ""
+  },
+  "14": {
+    "class_name": "agrotis_exclamationis",
+    "kr": "10348",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Agrotini",
+    "genus": "Agrotis",
+    "species": "exclamationis",
+    "rarity": "2"
+  },
+  "15": {
+    "class_name": "agrotis_ipsilon",
+    "kr": "10346",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Agrotini",
+    "genus": "Agrotis",
+    "species": "ipsilon",
+    "rarity": "5"
+  },
+  "16": {
+    "class_name": "agrotis_puta",
+    "kr": "10343",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Agrotini",
+    "genus": "Agrotis",
+    "species": "puta",
+    "rarity": ""
+  },
+  "17": {
+    "class_name": "alcis_repandata",
+    "kr": "7777",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Alcis",
+    "species": "repandata",
+    "rarity": ""
+  },
+  "18": {
+    "class_name": "alsophila_aescularia",
+    "kr": "7953",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Alsophilini",
+    "genus": "Alsophila",
+    "species": "aescularia",
+    "rarity": "3"
+  },
+  "19": {
+    "class_name": "amphipyra_pyramidea",
+    "kr": "9307",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Amphipyrinae",
+    "tribus": "",
+    "genus": "Amphipyra",
+    "species": "pyramidea",
+    "rarity": "5"
+  },
+  "20": {
+    "class_name": "anorthoa_munda",
+    "kr": "10050",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Orthosiini",
+    "genus": "Anorthoa",
+    "species": "munda",
+    "rarity": "4"
+  },
+  "21": {
+    "class_name": "apamea_monoglypha",
+    "kr": "9748",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Apameini",
+    "genus": "Apamea",
+    "species": "monoglypha",
+    "rarity": "3"
+  },
+  "22": {
+    "class_name": "apamea_scolopacina",
+    "kr": "9774",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Apameini",
+    "genus": "Apamea",
+    "species": "scolopacina",
+    "rarity": ""
+  },
+  "23": {
+    "class_name": "aplocera_plagiata",
+    "kr": "8620",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Chesiadini",
+    "genus": "Aplocera",
+    "species": "plagiata",
+    "rarity": "4"
+  },
+  "24": {
+    "class_name": "apocheima_hispidaria",
+    "kr": "7671",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Apocheima",
+    "species": "hispidaria",
+    "rarity": ""
+  },
+  "25": {
+    "class_name": "apoda_limacodes",
+    "kr": "3907",
+    "superfamily": "Zygaenoidea",
+    "family": "Limacodidae",
+    "subfamily": "",
+    "tribus": "",
+    "genus": "Apoda",
+    "species": "limacodes",
+    "rarity": "3"
+  },
+  "26": {
+    "class_name": "arctia_caja",
+    "kr": "10598",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Arctiini",
+    "genus": "Arctia",
+    "species": "caja",
+    "rarity": "3"
+  },
+  "27": {
+    "class_name": "asthena_albulata",
+    "kr": "8656",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Asthenini",
+    "genus": "Asthena",
+    "species": "albulata",
+    "rarity": ""
+  },
+  "28": {
+    "class_name": "atolmis_rubricollis",
+    "kr": "10483",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Lithosiini",
+    "genus": "Atolmis",
+    "species": "rubricollis",
+    "rarity": ""
+  },
+  "29": {
+    "class_name": "autographa_gamma",
+    "kr": "9056",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Plusiinae",
+    "tribus": "Plusiini",
+    "genus": "Autographa",
+    "species": "gamma",
+    "rarity": "5"
+  },
+  "30": {
+    "class_name": "axylia_putris",
+    "kr": "10082",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Axylia",
+    "species": "putris",
+    "rarity": "4"
+  },
+  "31": {
+    "class_name": "biston_betularia",
+    "kr": "7686",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Biston",
+    "species": "betularia",
+    "rarity": ""
+  },
+  "32": {
+    "class_name": "biston_strataria",
+    "kr": "7685",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Biston",
+    "species": "strataria",
+    "rarity": ""
+  },
+  "33": {
+    "class_name": "cabera_exanthemata",
+    "kr": "7826",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Caberini",
+    "genus": "Cabera",
+    "species": "exanthemata",
+    "rarity": "3"
+  },
+  "34": {
+    "class_name": "cabera_pusaria",
+    "kr": "7824",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Caberini",
+    "genus": "Cabera",
+    "species": "pusaria",
+    "rarity": "5"
+  },
+  "35": {
+    "class_name": "callimorpha_dominula",
+    "kr": "10603",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Arctiini",
+    "genus": "Callimorpha",
+    "species": "dominula",
+    "rarity": "4"
+  },
+  "36": {
+    "class_name": "calliteara_pudibunda",
+    "kr": "10387",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Lymantriinae",
+    "tribus": "Orgyiini",
+    "genus": "Calliteara",
+    "species": "pudibunda",
+    "rarity": "5"
+  },
+  "37": {
+    "class_name": "campaea_margaritaria",
+    "kr": "7836",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Campaeini",
+    "genus": "Campaea",
+    "species": "margaritaria",
+    "rarity": ""
+  },
+  "38": {
+    "class_name": "camptogramma_bilineata",
+    "kr": "8289",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Camptogramma",
+    "species": "bilineata",
+    "rarity": "5"
+  },
+  "39": {
+    "class_name": "caradrina_clavipalpis",
+    "kr": "9433",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Caradrini",
+    "genus": "Caradrina",
+    "species": "clavipalpis",
+    "rarity": "3"
+  },
+  "40": {
+    "class_name": "catocala_nupta",
+    "kr": "8874",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Erebinae",
+    "tribus": "Catocalini",
+    "genus": "Catocala",
+    "species": "nupta",
+    "rarity": "3"
+  },
+  "41": {
+    "class_name": "cerapteryx_graminis",
+    "kr": "10062",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Tholerini",
+    "genus": "Cerapteryx",
+    "species": "graminis",
+    "rarity": ""
+  },
+  "42": {
+    "class_name": "charanyca_trigrammica",
+    "kr": "9456",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Caradrini",
+    "genus": "Charanyca",
+    "species": "trigrammica",
+    "rarity": "3"
+  },
+  "43": {
+    "class_name": "chesias_legatella",
+    "kr": "8609",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Chesiadini",
+    "genus": "Chesias",
+    "species": "legatella",
+    "rarity": ""
+  },
+  "44": {
+    "class_name": "chloroclysta_siterata",
+    "kr": "8341",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Chloroclysta",
+    "species": "siterata",
+    "rarity": ""
+  },
+  "45": {
+    "class_name": "chloroclystis_v-ata",
+    "kr": "8601",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Chloroclystis",
+    "species": "v-ata",
+    "rarity": ""
+  },
+  "46": {
+    "class_name": "colocasia_coryli",
+    "kr": "10372",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Pantheinae",
+    "tribus": "",
+    "genus": "Colocasia",
+    "species": "coryli",
+    "rarity": ""
+  },
+  "47": {
+    "class_name": "colostygia_pectinataria",
+    "kr": "8385",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Colostygia",
+    "species": "pectinataria",
+    "rarity": ""
+  },
+  "48": {
+    "class_name": "colotois_pennaria",
+    "kr": "7663",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Colotoini",
+    "genus": "Colotois",
+    "species": "pennaria",
+    "rarity": "4"
+  },
+  "49": {
+    "class_name": "conistra_erythrocephala",
+    "kr": "9611",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Conistra",
+    "species": "erythrocephala",
+    "rarity": ""
+  },
+  "50": {
+    "class_name": "conistra_rubiginosa",
+    "kr": "9603",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Conistra",
+    "species": "rubiginosa",
+    "rarity": ""
+  },
+  "51": {
+    "class_name": "conistra_vaccinii",
+    "kr": "9600",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Conistra",
+    "species": "vaccinii",
+    "rarity": "5"
+  },
+  "52": {
+    "class_name": "cosmia_pyralina",
+    "kr": "9549",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Cosmia",
+    "species": "pyralina",
+    "rarity": ""
+  },
+  "53": {
+    "class_name": "cosmia_trapezina",
+    "kr": "9550",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Cosmia",
+    "species": "trapezina",
+    "rarity": "5"
+  },
+  "54": {
+    "class_name": "cosmorhoe_ocellata",
+    "kr": "8319",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Cosmorhoe",
+    "species": "ocellata",
+    "rarity": "3"
+  },
+  "55": {
+    "class_name": "craniophora_ligustri",
+    "kr": "8789",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Acronictinae",
+    "tribus": "",
+    "genus": "Craniophora",
+    "species": "ligustri",
+    "rarity": ""
+  },
+  "56": {
+    "class_name": "cryphia_algae",
+    "kr": "8801",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Bryophilinae",
+    "tribus": "",
+    "genus": "Cryphia",
+    "species": "algae",
+    "rarity": ""
+  },
+  "57": {
+    "class_name": "cybosia_mesomella",
+    "kr": "10477",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Lithosiini",
+    "genus": "Cybosia",
+    "species": "mesomella",
+    "rarity": "4"
+  },
+  "58": {
+    "class_name": "cyclophora_linearia",
+    "kr": "8024",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Cosymbiini",
+    "genus": "Cyclophora",
+    "species": "linearia",
+    "rarity": ""
+  },
+  "59": {
+    "class_name": "cyclophora_punctaria",
+    "kr": "8022",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Cosymbiini",
+    "genus": "Cyclophora",
+    "species": "punctaria",
+    "rarity": "3"
+  },
+  "60": {
+    "class_name": "cymatophorina_diluta",
+    "kr": "7492",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Cymatophorina",
+    "species": "diluta",
+    "rarity": ""
+  },
+  "61": {
+    "class_name": "deilephila_elpenor",
+    "kr": "6862",
+    "superfamily": "Bombycoidea",
+    "family": "Sphingidae",
+    "subfamily": "Macroglossinae",
+    "tribus": "Macroglossini",
+    "genus": "Deilephila",
+    "species": "elpenor",
+    "rarity": "3"
+  },
+  "62": {
+    "class_name": "deilephila_porcellus",
+    "kr": "6863",
+    "superfamily": "Bombycoidea",
+    "family": "Sphingidae",
+    "subfamily": "Macroglossinae",
+    "tribus": "Macroglossini",
+    "genus": "Deilephila",
+    "species": "porcellus",
+    "rarity": ""
+  },
+  "63": {
+    "class_name": "deileptenia_ribeata",
+    "kr": "7775",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Deileptenia",
+    "species": "ribeata",
+    "rarity": ""
+  },
+  "64": {
+    "class_name": "deltote_pygarga",
+    "kr": "9114",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Eustrotiinae",
+    "tribus": "",
+    "genus": "Deltote",
+    "species": "pygarga",
+    "rarity": "3"
+  },
+  "65": {
+    "class_name": "diarsia_brunnea",
+    "kr": "10092",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Diarsia",
+    "species": "brunnea",
+    "rarity": "4"
+  },
+  "66": {
+    "class_name": "diarsia_mendica",
+    "kr": "10089",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Diarsia",
+    "species": "mendica",
+    "rarity": "4"
+  },
+  "67": {
+    "class_name": "drepana_falcataria",
+    "kr": "7508",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Drepaninae",
+    "tribus": "",
+    "genus": "Drepana",
+    "species": "falcataria",
+    "rarity": "4"
+  },
+  "68": {
+    "class_name": "drymonia_obliterata",
+    "kr": "8723",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Drymonia",
+    "species": "obliterata",
+    "rarity": ""
+  },
+  "69": {
+    "class_name": "drymonia_ruficornis",
+    "kr": "8722",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Drymonia",
+    "species": "ruficornis",
+    "rarity": ""
+  },
+  "70": {
+    "class_name": "dysstroma_truncata",
+    "kr": "8348",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Dysstroma",
+    "species": "truncata",
+    "rarity": "4"
+  },
+  "71": {
+    "class_name": "ecliptopera_capitata",
+    "kr": "8339",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Ecliptopera",
+    "species": "capitata",
+    "rarity": ""
+  },
+  "72": {
+    "class_name": "ecliptopera_silaceata",
+    "kr": "8338",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Ecliptopera",
+    "species": "silaceata",
+    "rarity": ""
+  },
+  "73": {
+    "class_name": "ectropis_crepuscularia",
+    "kr": "7796",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Ectropis",
+    "species": "crepuscularia",
+    "rarity": ""
+  },
+  "74": {
+    "class_name": "eilema_complana",
+    "kr": "10490",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Lithosiini",
+    "genus": "Eilema",
+    "species": "complana",
+    "rarity": ""
+  },
+  "75": {
+    "class_name": "eilema_depressa",
+    "kr": "10487",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Lithosiini",
+    "genus": "Eilema",
+    "species": "depressa",
+    "rarity": ""
+  },
+  "76": {
+    "class_name": "eilema_sororcula",
+    "kr": "10499",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Lithosiini",
+    "genus": "Eilema",
+    "species": "sororcula",
+    "rarity": ""
+  },
+  "77": {
+    "class_name": "elaphria_venustula",
+    "kr": "9396",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Elaphriini",
+    "genus": "Elaphria",
+    "species": "venustula",
+    "rarity": ""
+  },
+  "78": {
+    "class_name": "ennomos_fuscantaria",
+    "kr": "7635",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Ennomini",
+    "genus": "Ennomos",
+    "species": "fuscantaria",
+    "rarity": ""
+  },
+  "79": {
+    "class_name": "epirrhoe_alternata",
+    "kr": "8275",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Epirrhoe",
+    "species": "alternata",
+    "rarity": "5"
+  },
+  "80": {
+    "class_name": "epirrita_autumnata",
+    "kr": "8444",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Operophterini",
+    "genus": "Epirrita",
+    "species": "autumnata",
+    "rarity": ""
+  },
+  "81": {
+    "class_name": "erannis_defoliaria",
+    "kr": "7699",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Erannis",
+    "species": "defoliaria",
+    "rarity": ""
+  },
+  "82": {
+    "class_name": "eupithecia_abbreviata",
+    "kr": "8578",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Eupithecia",
+    "species": "abbreviata",
+    "rarity": ""
+  },
+  "83": {
+    "class_name": "eupithecia_icterata",
+    "kr": "8538",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Eupithecia",
+    "species": "icterata",
+    "rarity": "3"
+  },
+  "84": {
+    "class_name": "eupithecia_succenturiata",
+    "kr": "8539",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Eupithecia",
+    "species": "succenturiata",
+    "rarity": "3"
+  },
+  "85": {
+    "class_name": "eupithecia_tantillaria",
+    "kr": "8596",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Eupithecia",
+    "species": "tantillaria",
+    "rarity": "3"
+  },
+  "86": {
+    "class_name": "euplexia_lucipara",
+    "kr": "9503",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Phlogophorini",
+    "genus": "Euplexia",
+    "species": "lucipara",
+    "rarity": "3"
+  },
+  "87": {
+    "class_name": "eupsilia_transversa",
+    "kr": "9596",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Eupsilia",
+    "species": "transversa",
+    "rarity": "5"
+  },
+  "88": {
+    "class_name": "eustroma_reticulata",
+    "kr": "8366",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Eustroma",
+    "species": "reticulata",
+    "rarity": ""
+  },
+  "89": {
+    "class_name": "geometra_papilionaria",
+    "kr": "7969",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Geometrinae",
+    "tribus": "Geometrini",
+    "genus": "Geometra",
+    "species": "papilionaria",
+    "rarity": ""
+  },
+  "90": {
+    "class_name": "gymnoscelis_rufifasciata",
+    "kr": "8599",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Gymnoscelis",
+    "species": "rufifasciata",
+    "rarity": ""
+  },
+  "91": {
+    "class_name": "habrosyne_pyritoides",
+    "kr": "7483",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Habrosyne",
+    "species": "pyritoides",
+    "rarity": ""
+  },
+  "92": {
+    "class_name": "hada_plebeja",
+    "kr": "9925",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Hadenini",
+    "genus": "Hada",
+    "species": "plebeja",
+    "rarity": "5"
+  },
+  "93": {
+    "class_name": "hemithea_aestivaria",
+    "kr": "7980",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Geometrinae",
+    "tribus": "Hemitheini",
+    "genus": "Hemithea",
+    "species": "aestivaria",
+    "rarity": ""
+  },
+  "94": {
+    "class_name": "herminia_grisealis",
+    "kr": "8846",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Herminiinae",
+    "tribus": "",
+    "genus": "Herminia",
+    "species": "grisealis",
+    "rarity": ""
+  },
+  "95": {
+    "class_name": "herminia_tarsicrinalis",
+    "kr": "8845",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Herminiinae",
+    "tribus": "",
+    "genus": "Herminia",
+    "species": "tarsicrinalis",
+    "rarity": ""
+  },
+  "96": {
+    "class_name": "herminia_tarsipennalis",
+    "kr": "8858",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Herminiinae",
+    "tribus": "",
+    "genus": "Herminia",
+    "species": "tarsipennalis",
+    "rarity": ""
+  },
+  "97": {
+    "class_name": "hoplodrina_octogenaria",
+    "kr": "9449",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Caradrini",
+    "genus": "Hoplodrina",
+    "species": "octogenaria",
+    "rarity": "3"
+  },
+  "98": {
+    "class_name": "horisme_tersata",
+    "kr": "8402",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Melanthiini",
+    "genus": "Horisme",
+    "species": "tersata",
+    "rarity": ""
+  },
+  "99": {
+    "class_name": "hydriomena_impluviata",
+    "kr": "8392",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Hydriomenini",
+    "genus": "Hydriomena",
+    "species": "impluviata",
+    "rarity": "3"
+  },
+  "100": {
+    "class_name": "hylaea_fasciaria",
+    "kr": "7839",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Campaeini",
+    "genus": "Hylaea",
+    "species": "fasciaria",
+    "rarity": "4"
+  },
+  "101": {
+    "class_name": "hypena_proboscidalis",
+    "kr": "8994",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Hypeninae",
+    "tribus": "",
+    "genus": "Hypena",
+    "species": "proboscidalis",
+    "rarity": "5"
+  },
+  "102": {
+    "class_name": "hypomecis_punctinalis",
+    "kr": "7784",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Hypomecis",
+    "species": "punctinalis",
+    "rarity": ""
+  },
+  "103": {
+    "class_name": "idaea_aversata",
+    "kr": "8184",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Sterrhini",
+    "genus": "Idaea",
+    "species": "aversata",
+    "rarity": "3"
+  },
+  "104": {
+    "class_name": "idaea_biselata",
+    "kr": "8132",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Sterrhini",
+    "genus": "Idaea",
+    "species": "biselata",
+    "rarity": "3"
+  },
+  "105": {
+    "class_name": "idaea_dimidiata",
+    "kr": "8161",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Sterrhini",
+    "genus": "Idaea",
+    "species": "dimidiata",
+    "rarity": ""
+  },
+  "106": {
+    "class_name": "idaea_seriata",
+    "kr": "8155",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Sterrhini",
+    "genus": "Idaea",
+    "species": "seriata",
+    "rarity": "3"
+  },
+  "107": {
+    "class_name": "lacanobia_oleracea",
+    "kr": "9917",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Hadenini",
+    "genus": "Lacanobia",
+    "species": "oleracea",
+    "rarity": "5"
+  },
+  "108": {
+    "class_name": "lampropteryx_otregiata",
+    "kr": "8317",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Lampropteryx",
+    "species": "otregiata",
+    "rarity": ""
+  },
+  "109": {
+    "class_name": "lampropteryx_suffumata",
+    "kr": "8316",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Lampropteryx",
+    "species": "suffumata",
+    "rarity": ""
+  },
+  "110": {
+    "class_name": "laothoe_populi",
+    "kr": "6824",
+    "superfamily": "Bombycoidea",
+    "family": "Sphingidae",
+    "subfamily": "Smerinthinae",
+    "tribus": "Smerinthini",
+    "genus": "Laothoe",
+    "species": "populi",
+    "rarity": "4"
+  },
+  "111": {
+    "class_name": "laspeyria_flexula",
+    "kr": "8975",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Boletobiinae",
+    "tribus": "Aventiini",
+    "genus": "Laspeyria",
+    "species": "flexula",
+    "rarity": ""
+  },
+  "112": {
+    "class_name": "ligdia_adustata",
+    "kr": "7530",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Abraxini",
+    "genus": "Ligdia",
+    "species": "adustata",
+    "rarity": ""
+  },
+  "113": {
+    "class_name": "lobophora_halterata",
+    "kr": "8665",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Trichopterygini",
+    "genus": "Lobophora",
+    "species": "halterata",
+    "rarity": ""
+  },
+  "114": {
+    "class_name": "lomaspilis_marginata",
+    "kr": "7527",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Cassymini",
+    "genus": "Lomaspilis",
+    "species": "marginata",
+    "rarity": "3"
+  },
+  "115": {
+    "class_name": "lomographa_temerata",
+    "kr": "7829",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Baptini",
+    "genus": "Lomographa",
+    "species": "temerata",
+    "rarity": ""
+  },
+  "116": {
+    "class_name": "lymantria_dispar",
+    "kr": "10376",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Lymantriinae",
+    "tribus": "Lymantriini",
+    "genus": "Lymantria",
+    "species": "dispar",
+    "rarity": "5"
+  },
+  "117": {
+    "class_name": "lymantria_monacha",
+    "kr": "10375",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Lymantriinae",
+    "tribus": "Lymantriini",
+    "genus": "Lymantria",
+    "species": "monacha",
+    "rarity": "6"
+  },
+  "118": {
+    "class_name": "macaria_alternata",
+    "kr": "7540",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Macariini",
+    "genus": "Macaria",
+    "species": "alternata",
+    "rarity": ""
+  },
+  "119": {
+    "class_name": "macaria_liturata",
+    "kr": "7542",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Macariini",
+    "genus": "Macaria",
+    "species": "liturata",
+    "rarity": ""
+  },
+  "120": {
+    "class_name": "macaria_notata",
+    "kr": "7539",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Macariini",
+    "genus": "Macaria",
+    "species": "notata",
+    "rarity": ""
+  },
+  "121": {
+    "class_name": "melanthia_procellata",
+    "kr": "8411",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Melanthiini",
+    "genus": "Melanthia",
+    "species": "procellata",
+    "rarity": ""
+  },
+  "122": {
+    "class_name": "mesapamea_secalis",
+    "kr": "9789",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Apameini",
+    "genus": "Mesapamea",
+    "species": "secalis",
+    "rarity": "3"
+  },
+  "123": {
+    "class_name": "mesoligia_furuncula",
+    "kr": "9786",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Apameini",
+    "genus": "Mesoligia",
+    "species": "furuncula",
+    "rarity": ""
+  },
+  "124": {
+    "class_name": "mimas_tiliae",
+    "kr": "6819",
+    "superfamily": "Bombycoidea",
+    "family": "Sphingidae",
+    "subfamily": "Smerinthinae",
+    "tribus": "Smerinthini",
+    "genus": "Mimas",
+    "species": "tiliae",
+    "rarity": "4"
+  },
+  "125": {
+    "class_name": "mythimna_albipuncta",
+    "kr": "10002",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Leucaniini",
+    "genus": "Mythimna",
+    "species": "albipuncta",
+    "rarity": "3"
+  },
+  "126": {
+    "class_name": "mythimna_ferrago",
+    "kr": "10001",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Leucaniini",
+    "genus": "Mythimna",
+    "species": "ferrago",
+    "rarity": "3"
+  },
+  "127": {
+    "class_name": "mythimna_impura",
+    "kr": "10006",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Leucaniini",
+    "genus": "Mythimna",
+    "species": "impura",
+    "rarity": ""
+  },
+  "128": {
+    "class_name": "mythimna_l-album",
+    "kr": "10022",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Leucaniini",
+    "genus": "Mythimna",
+    "species": "l-album",
+    "rarity": "4"
+  },
+  "129": {
+    "class_name": "noctua_comes",
+    "kr": "10099",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Noctua",
+    "species": "comes",
+    "rarity": ""
+  },
+  "130": {
+    "class_name": "noctua_fimbriata",
+    "kr": "10100",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Noctua",
+    "species": "fimbriata",
+    "rarity": ""
+  },
+  "131": {
+    "class_name": "noctua_interjecta",
+    "kr": "10105",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Noctua",
+    "species": "interjecta",
+    "rarity": ""
+  },
+  "132": {
+    "class_name": "noctua_janthina",
+    "kr": "10102",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Noctua",
+    "species": "janthina",
+    "rarity": ""
+  },
+  "133": {
+    "class_name": "noctua_pronuba",
+    "kr": "10096",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Noctua",
+    "species": "pronuba",
+    "rarity": "5"
+  },
+  "134": {
+    "class_name": "nola_confusalis",
+    "kr": "10429",
+    "superfamily": "Noctuoidea",
+    "family": "Nolidae",
+    "subfamily": "Nolinae",
+    "tribus": "Nolini",
+    "genus": "Nola",
+    "species": "confusalis",
+    "rarity": ""
+  },
+  "135": {
+    "class_name": "notodonta_dromedarius",
+    "kr": "8716",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Notodonta",
+    "species": "dromedarius",
+    "rarity": "4"
+  },
+  "136": {
+    "class_name": "nycteola_revayana",
+    "kr": "10441",
+    "superfamily": "Noctuoidea",
+    "family": "Nolidae",
+    "subfamily": "Chloephorinae",
+    "tribus": "",
+    "genus": "Nycteola",
+    "species": "revayana",
+    "rarity": ""
+  },
+  "137": {
+    "class_name": "ochropacha_duplaris",
+    "kr": "7490",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Ochropacha",
+    "species": "duplaris",
+    "rarity": ""
+  },
+  "138": {
+    "class_name": "ochropleura_plecta",
+    "kr": "10086",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Ochropleura",
+    "species": "plecta",
+    "rarity": "3"
+  },
+  "139": {
+    "class_name": "odontopera_bidentata",
+    "kr": "7647",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Odontoperini",
+    "genus": "Odontopera",
+    "species": "bidentata",
+    "rarity": ""
+  },
+  "140": {
+    "class_name": "oligia_fasciuncula",
+    "kr": "9784",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Apameini",
+    "genus": "Oligia",
+    "species": "fasciuncula",
+    "rarity": ""
+  },
+  "141": {
+    "class_name": "oligia_latruncula",
+    "kr": "9782",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Apameini",
+    "genus": "Oligia",
+    "species": "latruncula",
+    "rarity": "3"
+  },
+  "142": {
+    "class_name": "operophtera_brumata",
+    "kr": "8447",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Operophterini",
+    "genus": "Operophtera",
+    "species": "brumata",
+    "rarity": "1"
+  },
+  "143": {
+    "class_name": "opisthograptis_luteolata",
+    "kr": "7613",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Epionini",
+    "genus": "Opisthograptis",
+    "species": "luteolata",
+    "rarity": ""
+  },
+  "144": {
+    "class_name": "orthosia_cerasi",
+    "kr": "10044",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Orthosiini",
+    "genus": "Orthosia",
+    "species": "cerasi",
+    "rarity": "5"
+  },
+  "145": {
+    "class_name": "orthosia_cruda",
+    "kr": "10039",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Orthosiini",
+    "genus": "Orthosia",
+    "species": "cruda",
+    "rarity": ""
+  },
+  "146": {
+    "class_name": "orthosia_gothica",
+    "kr": "10038",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Orthosiini",
+    "genus": "Orthosia",
+    "species": "gothica",
+    "rarity": "3"
+  },
+  "147": {
+    "class_name": "orthosia_incerta",
+    "kr": "10037",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Orthosiini",
+    "genus": "Orthosia",
+    "species": "incerta",
+    "rarity": "3"
+  },
+  "148": {
+    "class_name": "ourapteryx_sambucaria",
+    "kr": "7659",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Ennomini",
+    "genus": "Ourapteryx",
+    "species": "sambucaria",
+    "rarity": "4"
+  },
+  "149": {
+    "class_name": "parectropis_similaria",
+    "kr": "7800",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Parectropis",
+    "species": "similaria",
+    "rarity": ""
+  },
+  "150": {
+    "class_name": "pareulype_berberata",
+    "kr": "8414",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Triphosini",
+    "genus": "Pareulype",
+    "species": "berberata",
+    "rarity": ""
+  },
+  "151": {
+    "class_name": "pasiphila_rectangulata",
+    "kr": "8603",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Eupitheciini",
+    "genus": "Pasiphila",
+    "species": "rectangulata",
+    "rarity": "3"
+  },
+  "152": {
+    "class_name": "peribatodes_rhomboidaria",
+    "kr": "7754",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Peribatodes",
+    "species": "rhomboidaria",
+    "rarity": ""
+  },
+  "153": {
+    "class_name": "peribatodes_secundaria",
+    "kr": "7762",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Peribatodes",
+    "species": "secundaria",
+    "rarity": ""
+  },
+  "154": {
+    "class_name": "perizoma_alchemillata",
+    "kr": "8456",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Perizomini",
+    "genus": "Perizoma",
+    "species": "alchemillata",
+    "rarity": "4"
+  },
+  "155": {
+    "class_name": "petrophora_chlorosata",
+    "kr": "7596",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Lithinini",
+    "genus": "Petrophora",
+    "species": "chlorosata",
+    "rarity": ""
+  },
+  "156": {
+    "class_name": "phalera_bucephala",
+    "kr": "8750",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Phalerinae",
+    "tribus": "",
+    "genus": "Phalera",
+    "species": "bucephala",
+    "rarity": "5"
+  },
+  "157": {
+    "class_name": "pheosia_gnoma",
+    "kr": "8728",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Pheosia",
+    "species": "gnoma",
+    "rarity": ""
+  },
+  "158": {
+    "class_name": "pheosia_tremula",
+    "kr": "8727",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Pheosia",
+    "species": "tremula",
+    "rarity": "4"
+  },
+  "159": {
+    "class_name": "phigalia_pilosaria",
+    "kr": "7672",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Boarmiini",
+    "genus": "Phigalia",
+    "species": "pilosaria",
+    "rarity": ""
+  },
+  "160": {
+    "class_name": "phlogophora_meticulosa",
+    "kr": "9505",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Phlogophorini",
+    "genus": "Phlogophora",
+    "species": "meticulosa",
+    "rarity": "3"
+  },
+  "161": {
+    "class_name": "phragmatobia_fuliginosa",
+    "kr": "10550",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Arctiini",
+    "genus": "Phragmatobia",
+    "species": "fuliginosa",
+    "rarity": "3"
+  },
+  "162": {
+    "class_name": "poecilocampa_populi",
+    "kr": "6728",
+    "superfamily": "Lasiocampoidea",
+    "family": "Lasiocampidae",
+    "subfamily": "Poecilocampinae",
+    "tribus": "Poecilocampini",
+    "genus": "Poecilocampa",
+    "species": "populi",
+    "rarity": ""
+  },
+  "163": {
+    "class_name": "polia_nebulosa",
+    "kr": "9993",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Hadenini",
+    "genus": "Polia",
+    "species": "nebulosa",
+    "rarity": "3"
+  },
+  "164": {
+    "class_name": "pseudoips_prasinana",
+    "kr": "10451",
+    "superfamily": "Noctuoidea",
+    "family": "Nolidae",
+    "subfamily": "Chloephorinae",
+    "tribus": "",
+    "genus": "Pseudoips",
+    "species": "prasinana",
+    "rarity": "3"
+  },
+  "165": {
+    "class_name": "pseudoterpna_pruinata",
+    "kr": "7965",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Geometrinae",
+    "tribus": "Pseudoterpnini",
+    "genus": "Pseudoterpna",
+    "species": "pruinata",
+    "rarity": ""
+  },
+  "166": {
+    "class_name": "pterostoma_palpina",
+    "kr": "8732",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Pterostoma",
+    "species": "palpina",
+    "rarity": "3"
+  },
+  "167": {
+    "class_name": "ptilodon_capucina",
+    "kr": "8738",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Notodontinae",
+    "tribus": "Notodontini",
+    "genus": "Ptilodon",
+    "species": "capucina",
+    "rarity": "5"
+  },
+  "168": {
+    "class_name": "pungeleria_capreolaria",
+    "kr": "7844",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Ennominae",
+    "tribus": "Campaeini",
+    "genus": "Pungeleria",
+    "species": "capreolaria",
+    "rarity": ""
+  },
+  "169": {
+    "class_name": "rivula_sericealis",
+    "kr": "9008",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Rivulinae",
+    "tribus": "",
+    "genus": "Rivula",
+    "species": "sericealis",
+    "rarity": "4"
+  },
+  "170": {
+    "class_name": "scopula_nigropunctata",
+    "kr": "8042",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Scopulini",
+    "genus": "Scopula",
+    "species": "nigropunctata",
+    "rarity": ""
+  },
+  "171": {
+    "class_name": "sphinx_ligustri",
+    "kr": "6832",
+    "superfamily": "Bombycoidea",
+    "family": "Sphingidae",
+    "subfamily": "Sphinginae",
+    "tribus": "Sphingini",
+    "genus": "Sphinx",
+    "species": "ligustri",
+    "rarity": "3"
+  },
+  "172": {
+    "class_name": "sphinx_pinastri",
+    "kr": "6834",
+    "superfamily": "Bombycoidea",
+    "family": "Sphingidae",
+    "subfamily": "Sphinginae",
+    "tribus": "Sphingini",
+    "genus": "Sphinx",
+    "species": "pinastri",
+    "rarity": "3"
+  },
+  "173": {
+    "class_name": "spilarctia_lutea",
+    "kr": "10566",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Arctiini",
+    "genus": "Spilarctia",
+    "species": "lutea",
+    "rarity": ""
+  },
+  "174": {
+    "class_name": "spilosoma_lubricipeda",
+    "kr": "10567",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Arctiini",
+    "genus": "Spilosoma",
+    "species": "lubricipeda",
+    "rarity": "3"
+  },
+  "175": {
+    "class_name": "stauropus_fagi",
+    "kr": "8758",
+    "superfamily": "Noctuoidea",
+    "family": "Notodontidae",
+    "subfamily": "Heterocampinae",
+    "tribus": "",
+    "genus": "Stauropus",
+    "species": "fagi",
+    "rarity": ""
+  },
+  "176": {
+    "class_name": "sunira_circellaris",
+    "kr": "9566",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Sunira",
+    "species": "circellaris",
+    "rarity": "4"
+  },
+  "177": {
+    "class_name": "tethea_or",
+    "kr": "7486",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Tethea",
+    "species": "or",
+    "rarity": "4"
+  },
+  "178": {
+    "class_name": "tetheella_fluctuosa",
+    "kr": "7488",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Tetheella",
+    "species": "fluctuosa",
+    "rarity": ""
+  },
+  "179": {
+    "class_name": "thera_variata",
+    "kr": "8357",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Cidariini",
+    "genus": "Thera",
+    "species": "variata",
+    "rarity": "5"
+  },
+  "180": {
+    "class_name": "tholera_decimalis",
+    "kr": "10065",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Hadeninae",
+    "tribus": "Tholerini",
+    "genus": "Tholera",
+    "species": "decimalis",
+    "rarity": "3"
+  },
+  "181": {
+    "class_name": "thumatha_senex",
+    "kr": "10466",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Arctiinae",
+    "tribus": "Lithosiini",
+    "genus": "Thumatha",
+    "species": "senex",
+    "rarity": ""
+  },
+  "182": {
+    "class_name": "thyatira_batis",
+    "kr": "7481",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Thyatirinae",
+    "tribus": "",
+    "genus": "Thyatira",
+    "species": "batis",
+    "rarity": "3"
+  },
+  "183": {
+    "class_name": "tiliacea_aurago",
+    "kr": "9557",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Xylenini",
+    "genus": "Tiliacea",
+    "species": "aurago",
+    "rarity": ""
+  },
+  "184": {
+    "class_name": "timandra_comae",
+    "kr": "8028",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Sterrhinae",
+    "tribus": "Timandrini",
+    "genus": "Timandra",
+    "species": "comae",
+    "rarity": ""
+  },
+  "185": {
+    "class_name": "trachea_atriplicis",
+    "kr": "9501",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Xyleninae",
+    "tribus": "Dypterygiini",
+    "genus": "Trachea",
+    "species": "atriplicis",
+    "rarity": "3"
+  },
+  "186": {
+    "class_name": "triodia_sylvina",
+    "kr": "63",
+    "superfamily": "Hepialoidea",
+    "family": "Hepialidae",
+    "subfamily": "",
+    "tribus": "",
+    "genus": "Triodia",
+    "species": "sylvina",
+    "rarity": "4"
+  },
+  "187": {
+    "class_name": "trisateles_emortualis",
+    "kr": "9169",
+    "superfamily": "Noctuoidea",
+    "family": "Erebidae",
+    "subfamily": "Boletobiinae",
+    "tribus": "Phytometrini",
+    "genus": "Trisateles",
+    "species": "emortualis",
+    "rarity": ""
+  },
+  "188": {
+    "class_name": "watsonalla_binaria",
+    "kr": "7503",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Drepaninae",
+    "tribus": "",
+    "genus": "Watsonalla",
+    "species": "binaria",
+    "rarity": ""
+  },
+  "189": {
+    "class_name": "watsonalla_cultraria",
+    "kr": "7505",
+    "superfamily": "Drepanoidea",
+    "family": "Drepanidae",
+    "subfamily": "Drepaninae",
+    "tribus": "",
+    "genus": "Watsonalla",
+    "species": "cultraria",
+    "rarity": "4"
+  },
+  "190": {
+    "class_name": "xanthorhoe_biriviata",
+    "kr": "8248",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Xanthorhoe",
+    "species": "biriviata",
+    "rarity": "4"
+  },
+  "191": {
+    "class_name": "xanthorhoe_designata",
+    "kr": "8249",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Xanthorhoe",
+    "species": "designata",
+    "rarity": ""
+  },
+  "192": {
+    "class_name": "xanthorhoe_ferrugata",
+    "kr": "8253",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Xanthorhoe",
+    "species": "ferrugata",
+    "rarity": "5"
+  },
+  "193": {
+    "class_name": "xanthorhoe_fluctuata",
+    "kr": "8256",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Xanthorhoe",
+    "species": "fluctuata",
+    "rarity": "5"
+  },
+  "194": {
+    "class_name": "xanthorhoe_montanata",
+    "kr": "8255",
+    "superfamily": "Geometroidea",
+    "family": "Geometridae",
+    "subfamily": "Larentiinae",
+    "tribus": "Xanthorhoini",
+    "genus": "Xanthorhoe",
+    "species": "montanata",
+    "rarity": "5"
+  },
+  "195": {
+    "class_name": "xestia_baja",
+    "kr": "10204",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Xestia",
+    "species": "baja",
+    "rarity": "4"
+  },
+  "196": {
+    "class_name": "xestia_c-nigrum",
+    "kr": "10199",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Xestia",
+    "species": "c-nigrum",
+    "rarity": "2"
+  },
+  "197": {
+    "class_name": "xestia_triangulum",
+    "kr": "10201",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Xestia",
+    "species": "triangulum",
+    "rarity": "4"
+  },
+  "198": {
+    "class_name": "xestia_xanthographa",
+    "kr": "10212",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Noctuinae",
+    "tribus": "Noctuini",
+    "genus": "Xestia",
+    "species": "xanthographa",
+    "rarity": "3"
+  },
+  "199": {
+    "class_name": "xylocampa_areola",
+    "kr": "9676",
+    "superfamily": "Noctuoidea",
+    "family": "Noctuidae",
+    "subfamily": "Psaphidinae",
+    "tribus": "Xylocampini",
+    "genus": "Xylocampa",
+    "species": "areola",
+    "rarity": ""
+  }
+}

models/moth_scanner/scanner/__init__.py

@@ -33,8 +33,8 @@ class Scanner(Interface):
             if not info.selected:
                 continue
             x0, y0, x1, y1 = bbox
-            cls_id = self.classifier(bbox.crop(im, enlarge=True))
-            label = labels.get(str(cls_id), cls_id)
+            cls_ref = self.classifier(bbox.crop(im, enlarge=True))
+            label = labels.get(cls_ref, cls_ref)
             file.add_bounding_box(x0, y0, bbox.w, bbox.h, label=label)
 
     def read_image(self, path: str, mode: int = cv2.IMREAD_COLOR) -> np.ndarray:

models/moth_scanner/scanner/classifier.py

@@ -1,4 +1,5 @@
 import chainer
+import json
 import numpy as np
 import typing as T
 
@@ -27,6 +28,10 @@ class Classifier(object):
                                          strict=True,
                                         )
 
+        with open(Path(root, config.mapping)) as f:
+            mapping = json.load(f)
+            self._cls_id2ref = {int(key): value["kr"] for key, value in mapping.items()}
+
     def _transform(self, im: np.ndarray):
         _prepare = self.backbone.meta.prepare_func
         size = (self.input_size, self.input_size)
@@ -55,4 +60,5 @@ class Classifier(object):
             pred = self.backbone(x)
         pred.to_cpu()
 
-        return int(np.argmax(pred.array, axis=1))
+        cls_id = int(np.argmax(pred.array, axis=1))
+        return self._cls_id2ref.get(cls_id, str(cls_id))

pycs/__init__.py

@@ -0,0 +1,47 @@
+import json
+import os
+import sys
+# pylint: disable=wrong-import-position,wrong-import-order
+import eventlet.tpool
+eventlet.tpool.set_num_threads(2)
+
+from munch import munchify
+from pathlib import Path
+
+from flask import Flask
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy import event
+from sqlalchemy import pool
+from sqlalchemy.engine import Engine
+
+print('=== Loading settings ===')
+with open('settings.json') as file:
+    settings = munchify(json.load(file))
+
+# create projects folder
+if not os.path.exists(settings.projects_folder):
+    os.mkdir(settings.projects_folder)
+
+DB_FILE = Path.cwd() / settings.database
+
+app = Flask(__name__)
+app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{DB_FILE}"
+app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+
+# pylint: disable=unused-argument
+@event.listens_for(Engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+    """ enables foreign keys on every established connection """
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+db = SQLAlchemy(app, engine_options=dict(
+    poolclass=pool.SingletonThreadPool,
+    connect_args=dict(
+        check_same_thread=False
+        )
+    )
+)
+migrate = Migrate(app, db)

pycs/database/Collection.py

@@ -1,70 +1,124 @@
-from contextlib import closing
-from typing import Iterator
+from __future__ import annotations
 
-from pycs.database.File import File
+import os
+import typing as T
 
 
-class Collection:
-    """
-    database class for collections
-    """
+from pycs import db
+from pycs.database.base import NamedBaseModel
+from pycs.database.util import commit_on_return
 
-    def __init__(self, database, row):
-        self.database = database
+class Collection(NamedBaseModel):
+    """ DB Model for collections """
 
-        self.identifier = row[0]
-        self.project_id = row[1]
-        self.reference = row[2]
-        self.name = row[3]
-        self.description = row[4]
-        self.position = row[5]
-        self.autoselect = row[6] > 0
+    # table columns
+    project_id = db.Column(
+        db.Integer,
+        db.ForeignKey("project.id", ondelete="CASCADE"),
+        nullable=False)
 
-    def set_name(self, name: str):
-        """
-        set this collection's name
+    reference = db.Column(
+        db.String, nullable=False)
 
-        :param name: new name
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE collections SET name = ? WHERE id = ?', (name, self.identifier))
-            self.name = name
+    description = db.Column(
+        db.String)
 
-    def remove(self) -> None:
-        """
-        remove this collection from the database
+    position = db.Column(
+        db.Integer, nullable=False)
 
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('DELETE FROM collections WHERE id = ?', [self.identifier])
+    autoselect = db.Column(
+        db.Boolean, nullable=False)
 
-    def count_files(self) -> int:
-        """
-        count files associated with this project
 
-        :return: count
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT COUNT(*) FROM files WHERE project = ? AND collection = ?',
-                           (self.project_id, self.identifier))
-            return cursor.fetchone()[0]
+    # contraints
+    __table_args__ = (
+        db.UniqueConstraint('project_id', 'reference'),
+    )
+
+    # relationships to other models
+    files = db.relationship("File", backref="collection", lazy="dynamic")
 
-    def files(self, offset: int = 0, limit: int = -1) -> Iterator[File]:
+    serialize_only = NamedBaseModel.serialize_only + (
+        "project_id",
+        "reference",
+        "description",
+        "position",
+        "autoselect",
+    )
+
+
+    def get_files(self, *filters, offset: int = 0, limit: int = -1):
         """
-        get an iterator of files associated with this collection
+        get an iterator of files associated with this project
 
         :param offset: file offset
         :param limit: file limit
         :return: iterator of files
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT * FROM files
-                WHERE project = ? AND collection = ?
-                ORDER BY id ASC LIMIT ? OFFSET ?
-                ''', (self.project_id, self.identifier, limit, offset))
-
-            for row in cursor:
-                yield File(self.database, row)
+
+        # pylint: disable=import-outside-toplevel, cyclic-import
+        from pycs.database.File import File
+        return self.files.filter(*filters).order_by(File.id).offset(offset).limit(limit)
+
+    # pylint: disable=too-many-arguments
+    @commit_on_return
+    def add_file(self,
+                 uuid: str,
+                 file_type: str,
+                 name: str,
+                 extension: str,
+                 size: int,
+                 filename: str,
+                 frames: int = None,
+                 fps: float = None) -> T.Tuple["File", bool]:
+        """
+        add a file to this collection
+
+        :param uuid: unique identifier which is used for temporary files
+        :param file_type: file type (either image or video)
+        :param name: file name
+        :param extension: file extension
+        :param size: file size
+        :param filename: actual name in filesystem
+        :param frames: frame count
+        :param fps: frames per second
+        :return: file
+        """
+        # pylint: disable=import-outside-toplevel, cyclic-import
+        from pycs.database.File import File
+
+        path = os.path.join(self.project.data_folder, f"{filename}{extension}")
+
+
+        file, is_new = File.get_or_create(
+            project_id=self.project_id, collection_id=self.id, path=path)
+
+        file.uuid = uuid
+        file.type = file_type
+        file.name = name
+        file.extension = extension
+        file.size = size
+        file.frames = frames
+        file.fps = fps
+
+        return file, is_new
+
+
+    @staticmethod
+    def update_autoselect(collections: T.List[Collection]) -> T.List[Collection]:
+        """ disable autoselect if there are no elements in the collection """
+
+        found = False
+
+        for collection in collections:
+            if not collection.autoselect:
+                continue
+
+            if found:
+                collection.autoselect = False
+
+            elif collection.files.count() == 0:
+                collection.autoselect = False
+                found = True
+
+        return collections

pycs/database/Database.py

@@ -1,338 +0,0 @@
-import sqlite3
-from contextlib import closing
-from time import time
-from typing import Optional, Iterator
-
-from pycs.database.Collection import Collection
-from pycs.database.File import File
-from pycs.database.LabelProvider import LabelProvider
-from pycs.database.Model import Model
-from pycs.database.Project import Project
-from pycs.database.Result import Result
-from pycs.database.discovery.LabelProviderDiscovery import discover as discover_label_providers
-from pycs.database.discovery.ModelDiscovery import discover as discover_models
-
-
-class Database:
-    """
-    opens an sqlite database and allows to access several objects
-    """
-
-    def __init__(self, path: str = ':memory:', initialization=True, discovery=True):
-        """
-        opens or creates a given sqlite database and creates all required tables
-
-        :param path: path to sqlite database
-        """
-        # save properties
-        self.path = path
-
-        # initialize database connection
-        self.con = sqlite3.connect(path)
-        self.con.execute("PRAGMA foreign_keys = ON")
-
-        if initialization:
-            # create tables
-            with self:
-                with closing(self.con.cursor()) as cursor:
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS models (
-                            id          INTEGER PRIMARY KEY,
-                            name        TEXT                NOT NULL,
-                            description TEXT,
-                            root_folder TEXT                NOT NULL UNIQUE,
-                            supports    TEXT                NOT NULL
-                        )
-                    ''')
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS label_providers (
-                            id                 INTEGER PRIMARY KEY,
-                            name               TEXT                NOT NULL,
-                            description        TEXT,
-                            root_folder        TEXT                NOT NULL,
-                            configuration_file TEXT                NOT NULL,
-                            UNIQUE(root_folder, configuration_file)
-                        )
-                    ''')
-
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS projects (
-                            id             INTEGER PRIMARY KEY,
-                            name           TEXT                NOT NULL,
-                            description    TEXT,
-                            created        INTEGER             NOT NULL,
-                            model          INTEGER,
-                            label_provider INTEGER,
-                            root_folder    TEXT                NOT NULL UNIQUE,
-                            external_data  BOOL                NOT NULL,
-                            data_folder    TEXT                NOT NULL,
-                            FOREIGN KEY (model) REFERENCES models(id)
-                                ON UPDATE CASCADE ON DELETE SET NULL,
-                            FOREIGN KEY (label_provider) REFERENCES label_providers(id)
-                                ON UPDATE CASCADE ON DELETE SET NULL
-                        )
-                    ''')
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS labels (
-                            id              INTEGER PRIMARY KEY,
-                            project         INTEGER             NOT NULL,
-                            parent          INTEGER,
-                            created         INTEGER             NOT NULL,
-                            reference       TEXT,
-                            name            TEXT                NOT NULL,
-                            hierarchy_level TEXT,
-                            FOREIGN KEY (project) REFERENCES projects(id)
-                                ON UPDATE CASCADE ON DELETE CASCADE,
-                            FOREIGN KEY (parent) REFERENCES labels(id)
-                                ON UPDATE CASCADE ON DELETE SET NULL,
-                            UNIQUE(project, reference)
-                        )
-                    ''')
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS collections (
-                            id          INTEGER          PRIMARY KEY,
-                            project     INTEGER NOT NULL,
-                            reference   TEXT    NOT NULL,
-                            name        TEXT    NOT NULL,
-                            description TEXT,
-                            position    INTEGER NOT NULL,
-                            autoselect  INTEGER NOT NULL,
-                            FOREIGN KEY (project) REFERENCES projects(id)
-                                ON UPDATE CASCADE ON DELETE CASCADE,
-                            UNIQUE(project, reference)
-                        )
-                    ''')
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS files (
-                            id         INTEGER PRIMARY KEY,
-                            uuid       TEXT                NOT NULL,
-                            project    INTEGER             NOT NULL,
-                            collection INTEGER,
-                            type       TEXT                NOT NULL,
-                            name       TEXT                NOT NULL,
-                            extension  TEXT                NOT NULL,
-                            size       INTEGER             NOT NULL,
-                            created    INTEGER             NOT NULL,
-                            path       TEXT                NOT NULL,
-                            frames     INTEGER,
-                            fps        FLOAT,
-                            FOREIGN KEY (project) REFERENCES projects(id)
-                                ON UPDATE CASCADE ON DELETE CASCADE,
-                            FOREIGN KEY (collection) REFERENCES collections(id)
-                                ON UPDATE CASCADE ON DELETE SET NULL,
-                            UNIQUE(project, path)
-                        )
-                    ''')
-                    cursor.execute('''
-                        CREATE TABLE IF NOT EXISTS results (
-                            id     INTEGER PRIMARY KEY,
-                            file   INTEGER             NOT NULL,
-                            origin TEXT                NOT NULL,
-                            type   TEXT                NOT NULL,
-                            label  INTEGER,
-                            data   TEXT,
-                            FOREIGN KEY (file) REFERENCES files(id)
-                                ON UPDATE CASCADE ON DELETE CASCADE
-                        )
-                    ''')
-
-        if discovery:
-            # run discovery modules
-            with self:
-                discover_models(self.con)
-                discover_label_providers(self.con)
-
-    def close(self):
-        """
-        close database file
-        """
-        self.con.close()
-
-    def copy(self):
-        """
-        Create a copy of this database object. This can be used to access the database
-        from another thread. Table initialization and model and label provider discovery is
-        disabled to speedup this function.
-
-        :return: Database
-        """
-        return Database(self.path, initialization=False, discovery=False)
-
-    def commit(self):
-        """
-        commit changes
-        """
-        self.con.commit()
-
-    def __enter__(self):
-        self.con.__enter__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.con.__exit__(exc_type, exc_val, exc_tb)
-
-    def models(self) -> Iterator[Model]:
-        """
-        get a list of all available models
-
-        :return: iterator of models
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM models')
-            for row in cursor:
-                yield Model(self, row)
-
-    def model(self, identifier: int) -> Optional[Model]:
-        """
-        get a model using its unique identifier
-
-        :param identifier: unique identifier
-        :return: model
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM models WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return Model(self, row)
-
-            return None
-
-    def label_providers(self) -> Iterator[LabelProvider]:
-        """
-        get a list of all available label providers
-
-        :return: iterator over label providers
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM label_providers')
-            for row in cursor:
-                yield LabelProvider(self, row)
-
-    def label_provider(self, identifier: int) -> Optional[LabelProvider]:
-        """
-        get a label provider using its unique identifier
-
-        :param identifier: unique identifier
-        :return: label provider
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM label_providers WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return LabelProvider(self, row)
-
-            return None
-
-    def projects(self) -> Iterator[Project]:
-        """
-        get a list of all available projects
-
-        :return: iterator over projects
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM projects')
-            for row in cursor:
-                yield Project(self, row)
-
-    def project(self, identifier: int) -> Optional[Project]:
-        """
-        get a project using its unique identifier
-
-        :param identifier: unique identifier
-        :return: project
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM projects WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return Project(self, row)
-
-            return None
-
-    def create_project(self,
-                       name: str,
-                       description: str,
-                       model: Model,
-                       label_provider: Optional[LabelProvider],
-                       root_folder: str,
-                       external_data: bool,
-                       data_folder: str):
-        """
-        insert a project into the database
-
-        :param name: project name
-        :param description: project description
-        :param model: used model
-        :param label_provider: used label provider (optional)
-        :param root_folder: path to project folder
-        :param external_data: whether an external data directory is used
-        :param data_folder: path to data folder
-        :return: created project
-        """
-        # prepare some values
-        created = int(time())
-        label_provider_id = label_provider.identifier if label_provider is not None else None
-
-        # insert statement
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO projects (
-                    name, description, created, model, label_provider, root_folder, external_data, 
-                    data_folder
-                )
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-            ''', (name, description, created, model.identifier, label_provider_id, root_folder,
-                  external_data, data_folder))
-
-            return self.project(cursor.lastrowid)
-
-    def collection(self, identifier: int) -> Optional[Collection]:
-        """
-        get a collection using its unique identifier
-
-        :param identifier: unique identifier
-        :return: collection
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM collections WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return Collection(self, row)
-
-            return None
-
-    def file(self, identifier) -> Optional[File]:
-        """
-        get a file using its unique identifier
-
-        :param identifier: unique identifier
-        :return: file
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM files WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return File(self, row)
-
-            return None
-
-    def result(self, identifier) -> Optional[Result]:
-        """
-        get a result using its unique identifier
-
-        :param identifier: unique identifier
-        :return: result
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM results WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return Result(self, row)
-
-            return None

pycs/database/File.py

@@ -1,256 +1,239 @@
-from contextlib import closing
+from __future__ import annotations
+
 import os
-from json import dumps
-from typing import List
-from typing import Optional
+import typing as T
+import warnings
+
+from datetime import datetime
+from pathlib import Path
 
+from pycs import db
+from pycs.database.Collection import Collection
 from pycs.database.Result import Result
+from pycs.database.Label import Label
+from pycs.database.base import NamedBaseModel
+from pycs.database.util import commit_on_return
+
+
+class File(NamedBaseModel):
+    """ DB Model for files """
+
+    # table columns
+    uuid = db.Column(db.String, nullable=False)
+
+    extension = db.Column(db.String, nullable=False)
+
+    type = db.Column(db.String, nullable=False)
+
+    size = db.Column(db.Integer, nullable=False)
+
+    created = db.Column(db.DateTime, default=datetime.utcnow,
+        index=True, nullable=False)
+
+    path = db.Column(db.String, nullable=False)
 
+    frames = db.Column(db.Integer)
 
-class File:
-    """
-    database class for files
-    """
+    fps = db.Column(db.Float)
 
-    def __init__(self, database, row):
-        self.database = database
+    project_id = db.Column(
+        db.Integer,
+        db.ForeignKey("project.id", ondelete="CASCADE"),
+        nullable=False)
 
-        self.identifier = row[0]
-        self.uuid = row[1]
-        self.project_id = row[2]
-        self.collection_id = row[3]
-        self.type = row[4]
-        self.name = row[5]
-        self.extension = row[6]
-        self.size = row[7]
-        self.created = row[8]
-        self.path = row[9]
-        self.frames = row[10]
-        self.fps = row[11]
+    collection_id = db.Column(
+        db.Integer,
+        db.ForeignKey("collection.id", ondelete="SET NULL"))
 
+    # contraints
+    __table_args__ = (
+        db.UniqueConstraint('project_id', 'path'),
+    )
+
+
+    results = db.relationship("Result",
+        backref="file",
+        lazy="dynamic",
+        passive_deletes=True,
+    )
+
+
+    serialize_only = NamedBaseModel.serialize_only + (
+        "uuid",
+        "extension",
+        "type",
+        "size",
+        "created",
+        "path",
+        "frames",
+        "fps",
+        "project_id",
+        "collection_id",
+    )
+
+    @property
+    def filename(self):
+        """ filename consisting of a name and an extension """
+        return f"{self.name}{self.extension}"
 
     @property
-    def absolute_path(self):
-        if os.path.isabs(self.path):
-            return self.path
+    def absolute_path(self) -> str:
+        """ returns an absolute of the file """
+        path = Path(self.path)
 
-        return os.path.join(os.getcwd(), self.path)
+        if path.is_absolute():
+            return str(path)
 
-    def project(self):
-        """
-        get the project associated with this file
+        return str(Path.cwd() / path)
 
-        :return: project
+    # pylint: disable=arguments-differ
+    def delete(self, commit: bool = True):
         """
-        return self.database.project(self.project_id)
-
-    def collection(self):
+            after the object is deleted, the according physical file
+            is also delete if commit was True
         """
-        get the collection associated with this file
 
-        :return: collection
-        """
-        if self.collection_id is None:
-            return None
+        # pylint: disable=unexpected-keyword-arg
+        dump = super().delete(commit=commit)
+
+        if commit:
+            os.remove(self.path)
 
-        return self.database.collection(self.collection_id)
+        # TODO: remove temp files
+        warnings.warn("Temporary files may still exist!")
+        return dump
 
-    def set_collection(self, collection_id: Optional[int]):
+    @commit_on_return
+    def set_collection(self, collection_id: T.Optional[int]):
         """
         set this file's collection
 
-        :param collection_id: new collection
+        :param collection_id: new collection id
         :return:
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE files SET collection = ? WHERE id = ?',
-                           (collection_id, self.identifier))
-            self.collection_id = collection_id
 
-    def set_collection_by_reference(self, collection_reference: Optional[str]):
+        self.collection_id = collection_id
+
+    @commit_on_return
+    def set_collection_by_reference(self, collection_reference: T.Optional[str]):
         """
         set this file's collection
 
         :param collection_reference: collection reference
         :return:
         """
-        if collection_reference is None:
+        if self.collection_reference is None:
             self.set_collection(None)
-            return
-
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT id FROM collections WHERE reference = ?', [collection_reference])
-            row = cursor.fetchone()
 
-        self.set_collection(row[0] if row is not None else None)
+        collection = Collection.query.filter_by(reference=collection_reference).one()
+        self.collection_id = collection.id
 
-    def remove(self) -> None:
+    def _get_another_file(self, *query) -> T.Optional[File]:
         """
-        remove this file from the database
+        get the first file matching the query ordered by descending id
 
-        :return:
+        :return: another file or None
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('DELETE FROM files WHERE id = ?', [self.identifier])
+        return File.query.filter(File.project_id == self.project_id, *query)
 
-    def previous(self):
+    def next(self) -> T.Optional[File]:
         """
-        get the predecessor of this file
+        get the successor of this file
 
-        :return: another file
+        :return: another file or None
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT * FROM files WHERE id < ? AND project = ? ORDER BY id DESC LIMIT 1
-            ''', (self.identifier, self.project_id))
-            row = cursor.fetchone()
 
-            if row is not None:
-                return File(self.database, row)
+        return self._get_another_file(File.id > self.id)\
+            .order_by(File.id).first()
 
-            return None
 
-    def next(self):
+    def previous(self) -> T.Optional[File]:
         """
-        get the successor of this file
+        get the predecessor of this file
 
-        :return: another file
+        :return: another file or None
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                    SELECT * FROM files WHERE id > ? AND project = ? ORDER BY id ASC LIMIT 1
-                ''', (self.identifier, self.project_id))
-            row = cursor.fetchone()
 
-            if row is not None:
-                return File(self.database, row)
+        # pylint: disable=no-member
+        return self._get_another_file(File.id < self.id)\
+            .order_by(File.id.desc()).first()
 
-            return None
 
-    def previous_in_collection(self):
+    def next_in_collection(self) -> T.Optional[File]:
         """
         get the predecessor of this file
 
-        :return: another file
+        :return: another file or None
         """
-        with closing(self.database.con.cursor()) as cursor:
-            if self.collection_id is None:
-                cursor.execute('''
-                    SELECT * FROM files
-                    WHERE id < ? AND project = ? AND collection IS NULL
-                    ORDER BY id DESC
-                    LIMIT 1
-                ''', (self.identifier, self.project_id))
-            else:
-                cursor.execute('''
-                    SELECT * FROM files
-                    WHERE id < ? AND project = ? AND collection = ?
-                    ORDER BY id DESC
-                    LIMIT 1
-                ''', (self.identifier, self.project_id, self.collection_id))
-
-            row = cursor.fetchone()
-            if row is not None:
-                return File(self.database, row)
-
-            return None
-
-    def next_in_collection(self):
+        return self._get_another_file(
+            File.id > self.id, File.collection_id == self.collection_id)\
+            .order_by(File.id).first()
+
+
+    def previous_in_collection(self) -> T.Optional[File]:
         """
-        get the successor of this file
+        get the predecessor of this file
 
-        :return: another file
+        :return: another file or None
         """
-        with closing(self.database.con.cursor()) as cursor:
-            if self.collection_id is None:
-                cursor.execute('''
-                    SELECT * FROM files
-                    WHERE id > ? AND project = ? AND collection IS NULL
-                    ORDER BY id ASC
-                    LIMIT 1
-                ''', (self.identifier, self.project_id))
-            else:
-                cursor.execute('''
-                    SELECT * FROM files
-                    WHERE id > ? AND project = ? AND collection = ?
-                    ORDER BY id ASC
-                    LIMIT 1
-                ''', (self.identifier, self.project_id, self.collection_id))
-
-            row = cursor.fetchone()
-            if row is not None:
-                return File(self.database, row)
-
-            return None
-
-    def results(self) -> List[Result]:
+
+        # pylint: disable=no-member
+        return self._get_another_file(
+            File.id < self.id, File.collection_id == self.collection_id)\
+            .order_by(File.id.desc()).first()
+
+
+    def result(self, identifier: int) -> T.Optional[Result]:
         """
-        get a list of all results associated with this file
+        get one of the file's results
 
-        :return: list of results
+        :return: result object or None
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM results WHERE file = ?', [self.identifier])
-            return list(map(
-                lambda row: Result(self.database, row),
-                cursor.fetchall()
-            ))
-
-    def result(self, identifier) -> Optional[Result]:
+        return self.results.get(identifier)
+
+
+    @commit_on_return
+    def create_result(self,
+                      origin: str,
+                      result_type: str,
+                      label: T.Optional[T.Union[Label, int]] = None,
+                      data: T.Optional[dict] = None) -> Result:
         """
-        get a specific result using its unique identifier
+        Creates a result and returns the created object
 
-        :param identifier: unique identifier
-        :return: result
+        :return: result object
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT * FROM results WHERE id = ? AND file = ?
-            ''', (identifier, self.identifier))
-            row = cursor.fetchone()
 
-            if row is not None:
-                return Result(self.database, row)
+        result = Result.new(commit=False,
+                            file_id=self.id,
+                            origin=origin,
+                            type=result_type)
 
-            return None
+        result.data = data
 
-    def create_result(self, origin, result_type, label, data=None):
-        """
-        create a result
+        if label is not None:
+            assert isinstance(label, (int, Label)), f"Wrong label type: {type(label)}"
 
-        :param origin:
-        :param result_type:
-        :param label:
-        :param data:
-        :return:
-        """
-        if data is not None:
-            data = dumps(data)
+            if isinstance(label, Label):
+                label = label.id
+
+            result.label_id = label
 
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO results (file, origin, type, label, data)
-                VALUES              (   ?,      ?,    ?,     ?,    ?)
-            ''', (self.identifier, origin, result_type, label, data))
+        return result
 
-            return self.result(cursor.lastrowid)
 
-    def remove_results(self, origin='pipeline') -> List[Result]:
+    def remove_results(self, origin='pipeline') -> T.List[Result]:
         """
-        remove all results with the specified origin
+        Remove assigned results, but return them.
 
-        :param origin: either 'pipeline' or 'user'
-        :return: list of removed results
+        :return: list of result objects
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT * FROM results WHERE file = ? AND origin = ?
-            ''', (self.identifier, origin))
 
-            results = list(map(lambda row: Result(self.database, row), cursor.fetchall()))
+        results = Result.query.filter(
+            Result.file_id == self.id,
+            Result.origin == origin)
 
-            cursor.execute('''
-                DELETE FROM results WHERE file = ? AND origin = ?
-            ''', (self.identifier, origin))
+        _results = [r.serialize() for r in results.all()]
+        results.delete()
 
-            return results
+        return _results

pycs/database/Label.py

@@ -1,107 +1,97 @@
-from contextlib import closing
+from __future__ import annotations
 
+import typing as T
 
-class Label:
-    """
-    database class for labels
-    """
+from datetime import datetime
 
-    def __init__(self, database, row):
-        self.database = database
+from pycs import db
+from pycs.database.base import NamedBaseModel
+from pycs.database.util import commit_on_return
 
-        self.identifier = row[0]
-        self.project_id = row[1]
-        self.parent_id = row[2]
-        self.created = row[3]
-        self.reference = row[4]
-        self.name = row[5]
-        self.hierarchy_level = row[6]
+def compare_children(start_label: Label, identifier: int) -> bool:
+    """ check for cyclic relationships """
 
-    def project(self):
-        """
-        get the project this label is associated with
+    labels_to_check = [start_label]
 
-        :return: project
-        """
-        return self.database.project(self.project_id)
+    while labels_to_check:
+        label = labels_to_check.pop(0)
 
-    def set_name(self, name: str):
-        """
-        set this labels name
+        if label.id == identifier:
+            return False
 
-        :param name: new name
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE labels SET name = ? WHERE id = ?', (name, self.identifier))
-            self.name = name
+        labels_to_check.extend(label.children)
 
-    def set_parent(self, parent_id: int):
-        """
-        set this labels parent
+    return True
 
-        :param parent_id: parent's id
-        :return:
-        """
+def _label_id():
+    return Label.id
 
-        # check for cyclic relationships
-        def compare_children(label, identifier):
-            if label.identifier == identifier:
-                return False
+class Label(NamedBaseModel):
+    """ DB Model for labels """
 
-            for child in label.children():
-                if not compare_children(child, identifier):
-                    return False
+    id = db.Column(db.Integer, primary_key=True)
 
-            return True
+    project_id = db.Column(
+        db.Integer,
+        db.ForeignKey("project.id", ondelete="CASCADE"),
+        nullable=False)
 
-        if not compare_children(self, parent_id):
-            raise ValueError('parent_id')
+    parent_id = db.Column(
+        db.Integer,
+        db.ForeignKey("label.id", ondelete="SET NULL"))
 
-        # insert parent id
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE labels SET parent = ? WHERE id = ?',
-                           (parent_id, self.identifier))
-            self.parent_id = parent_id
+    created = db.Column(db.DateTime, default=datetime.utcnow,
+        index=True, nullable=False)
 
-    def remove(self):
-        """
-        remove this label from the database
+    reference = db.Column(db.String)
+    hierarchy_level = db.Column(db.String)
 
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('DELETE FROM labels WHERE id = ?', [self.identifier])
+    # contraints
+    __table_args__ = (
+        db.UniqueConstraint('project_id', 'reference'),
+    )
 
-    def parent(self):
-        """
-        get this labels parent from the database
+    # relationships to other models
+    parent = db.relationship("Label",
+        backref="children",
+        remote_side=_label_id,
+    )
 
-        :return: parent or None
-        """
-        if self.parent_id is None:
-            return None
+    results = db.relationship("Result",
+        backref="label",
+        passive_deletes=True,
+        lazy="dynamic",
+    )
 
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM labels WHERE id = ? AND project = ?',
-                           (self.parent_id, self.project_id))
-            row = cursor.fetchone()
+    serialize_only = NamedBaseModel.serialize_only + (
+        "project_id",
+        "parent_id",
+        "reference",
+        "hierarchy_level",
+        # "children",
+    )
 
-            if row is not None:
-                return Label(self.database, row)
+    @commit_on_return
+    def set_parent(self, parent: T.Optional[T.Union[int, str, Label]] = None) -> None:
 
-        return None
-
-    def children(self):
         """
-        get this labels children from the database
+        set this labels parent
 
-        :return: list of children
+        :param parent: parent label. Can be a reference, an id or a Label instance
+        :return:
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM labels WHERE parent = ? AND project = ?',
-                           (self.identifier, self.project_id))
-            return list(map(
-                lambda row: Label(self.database, row),
-                cursor.fetchall()
-            ))
+        parent_id = None
+        if parent is not None:
+            if isinstance(parent, Label):
+                parent_id = parent.id
+
+            elif isinstance(parent, str):
+                parent_id = Label.query.filter(Label.reference == parent).one().id
+
+            elif isinstance(parent, int):
+                parent_id = parent
+
+            if not compare_children(self, parent_id):
+                raise ValueError('Cyclic relationship detected!')
+
+        self.parent_id = parent_id

pycs/database/LabelProvider.py

@@ -1,26 +1,71 @@
 import json
-from os import path
+import re
 
+from pathlib import Path
+
+from pycs import db
+from pycs.database.base import NamedBaseModel
 from pycs.interfaces.LabelProvider import LabelProvider as LabelProviderInterface
 
 
-class LabelProvider:
+class LabelProvider(NamedBaseModel):
     """
-    database class for label providers
+    DB model for label providers
     """
 
-    def __init__(self, database, row):
-        self.database = database
+    description = db.Column(db.String)
+    root_folder = db.Column(db.String, nullable=False)
+    configuration_file = db.Column(db.String, nullable=False)
+
+    # relationships to other models
+    projects = db.relationship("Project", backref="label_provider", lazy="dynamic")
+
+    # contraints
+    __table_args__ = (
+        db.UniqueConstraint('root_folder', 'configuration_file'),
+    )
+
+    serialize_only = NamedBaseModel.serialize_only + (
+        "description",
+        "root_folder",
+        "configuration_file",
+    )
+
+    @classmethod
+    def discover(cls, root: Path):
+        """
+            searches for label providers under the given path
+            and stores them in the database
+        """
+
+        for folder, conf_path in _find_files(root):
+            with open(conf_path) as conf_file:
+                config = json.load(conf_file)
+
+            provider, _ = cls.get_or_create(
+                root_folder=str(folder),
+                configuration_file=conf_path.name
+            )
+
+            provider.name = config['name']
+
+            # returns None if not present
+            provider.description = config.get('description')
 
-        self.identifier = row[0]
-        self.name = row[1]
-        self.description = row[2]
-        self.root_folder = row[3]
-        self.configuration_file = row[4]
+            provider.flush()
+        db.session.commit()
 
     @property
-    def configuration_path(self):
-        return path.join(self.root_folder, self.configuration_file)
+    def root(self) -> Path:
+        """ returns the root folder as Path object """
+        return Path(self.root_folder)
+
+
+    @property
+    def configuration_file_path(self) -> str:
+        """ return the configuration file as Path object """
+        return str(self.root / self.configuration_file)
+
 
     def load(self) -> LabelProviderInterface:
         """
@@ -29,11 +74,11 @@ class LabelProvider:
         :return: LabelProvider instance
         """
         # load configuration.json
-        with open(self.configuration_path, 'r') as configuration_file:
+        with open(self.configuration_file_path) as configuration_file:
             configuration = json.load(configuration_file)
 
         # load code
-        code_path = path.join(self.root_folder, configuration['code']['module'])
+        code_path = str(self.root / configuration['code']['module'])
         module_name = code_path.replace('/', '.').replace('\\', '.')
         class_name = configuration['code']['class']
 
@@ -42,3 +87,21 @@ class LabelProvider:
 
         # return instance
         return class_attr(self.root_folder, configuration)
+
+
+def _find_files(root: str, config_regex=re.compile(r'^configuration(\d+)?\.json$')):
+    """ generator for config files found under the given path """
+
+    for folder in Path(root).glob('*'):
+        # list files
+        for file_path in folder.iterdir():
+
+            # filter configuration files
+            if not file_path.is_file():
+                continue
+
+            if config_regex.match(file_path.name) is None:
+                continue
+
+            # yield element
+            yield folder, file_path

pycs/database/Model.py

@@ -1,56 +1,87 @@
-from contextlib import closing
-from json import loads, dumps
+import json
 
+from pathlib import Path
 
-class Model:
+from pycs import db
+from pycs.database.base import NamedBaseModel
+from pycs.database.util import commit_on_return
+
+class Model(NamedBaseModel):
     """
-    database class for label providers
+    DB model for ML models
     """
 
-    def __init__(self, database, row):
-        self.database = database
+    description = db.Column(db.String)
+    root_folder = db.Column(db.String, nullable=False, unique=True)
+    supports_encoded = db.Column(db.String, nullable=False)
 
-        self.identifier = row[0]
-        self.name = row[1]
-        self.description = row[2]
-        self.root_folder = row[3]
-        self.supports = loads(row[4])
+    # relationships to other models
+    projects = db.relationship("Project", backref="model", lazy="dynamic")
 
-    def copy_to(self, name: str, root_folder: str):
-        """
-        copies the models database entry while changing name and root_folder
+    serialize_only = NamedBaseModel.serialize_only + (
+        "description",
+        "root_folder",
+    )
+
+
+    def serialize(self):
+        result = super().serialize()
+        result["supports"] = self.supports
+        return result
 
-        :param name: copy name
-        :param root_folder: copy root folder
-        :return: copy
+    @classmethod
+    def discover(cls, root: Path, config_name: str = "configuration.json"):
         """
-        supports = dumps(self.supports)
-
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO models (name, description, root_folder, supports)
-                VALUES (?, ?, ?, ?)
-                ON CONFLICT (root_folder)
-                DO UPDATE SET name = ?, description = ?, supports = ?
-            ''', (name, self.description, root_folder, supports, name, self.description, supports))
-
-            # lastrowid is 0 if on conflict clause applies.
-            # If this is the case we do an extra query to receive the row id.
-            if cursor.lastrowid > 0:
-                row_id = cursor.lastrowid
-                insert = True
-            else:
-                cursor.execute('SELECT id FROM models WHERE root_folder = ?', [root_folder])
-                row_id = cursor.fetchone()[0]
-                insert = False
-
-        return self.database.model(row_id), insert
-
-    def remove(self):
+            searches for models under the given path
+            and stores them in the database
         """
-        remove this model from the database
+        for folder in Path(root).glob("*"):
+            with open(folder / config_name) as config_file:
+                config = json.load(config_file)
+
+            # extract data
+            name = config['name']
+            description = config.get('description', None)
+            supports = config['supports']
+
+            model, _ = cls.get_or_create(root_folder=str(folder))
+
+            model.name = name
+            model.description = description
+            model.supports = supports
+
+            model.flush()
+        db.session.commit()
+
+    @property
+    def supports(self) -> dict:
+        """ getter for the 'supports' attribute """
+        return json.loads(self.supports_encoded)
 
-        :return:
+    @supports.setter
+    def supports(self, value):
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('DELETE FROM models WHERE id = ?', [self.identifier])
+            setter for the 'supports' attribute.
+            The attribute is encoded property before assigned to the object.
+        """
+
+        if isinstance(value, str):
+            self.supports_encoded = value
+
+        elif isinstance(value, (dict, list)):
+            self.supports_encoded = json.dumps(value)
+
+        else:
+            raise ValueError(f"Not supported type: {type(value)}")
+
+    @commit_on_return
+    def copy_to(self, name: str, root_folder: str):
+        """ copies current model to another folder and updates the name """
+
+        model, is_new = Model.get_or_create(root_folder=root_folder)
+
+        model.name = name
+        model.description = self.description
+        model.supports_encoded = self.supports_encoded
+
+        return model, is_new

pycs/database/Project.py

@@ -1,230 +1,235 @@
-from contextlib import closing
-from os.path import join
-from time import time
-from typing import List, Optional, Tuple, Iterator, Union
+import os
+import shutil
+import typing as T
+
+from datetime import datetime
+
+from pycs import app
+from pycs import db
+from pycs.database.base import NamedBaseModel
 
 from pycs.database.Collection import Collection
 from pycs.database.File import File
 from pycs.database.Label import Label
-from pycs.database.LabelProvider import LabelProvider
-from pycs.database.Model import Model
-from pycs.database.util.TreeNodeLabel import TreeNodeLabel
+from pycs.database.util import commit_on_return
 
 
-class Project:
-    """
-    database class for projects
-    """
+class Project(NamedBaseModel):
+    """ DB Model for projects """
 
-    def __init__(self, database, row):
-        self.database = database
+    description = db.Column(db.String)
 
-        self.identifier = row[0]
-        self.name = row[1]
-        self.description = row[2]
-        self.created = row[3]
-        self.model_id = row[4]
-        self.label_provider_id = row[5]
-        self.root_folder = row[6]
-        self.external_data = bool(row[7])
-        self.data_folder = row[8]
+    created = db.Column(db.DateTime, default=datetime.utcnow,
+        index=True, nullable=False)
 
-    def model(self) -> Model:
-        """
-        get the model this project is associated with
+    model_id = db.Column(
+        db.Integer,
+        db.ForeignKey("model.id", ondelete="SET NULL"))
 
-        :return: model
-        """
-        return self.database.model(self.model_id)
+    label_provider_id = db.Column(
+        db.Integer,
+        db.ForeignKey("label_provider.id", ondelete="SET NULL"))
 
-    def label_provider(self) -> Optional[LabelProvider]:
-        """
-        get the label provider this project is associated with
+    root_folder = db.Column(db.String, nullable=False, unique=True)
 
-        :return: label provider
-        """
-        if self.label_provider_id is not None:
-            return self.database.label_provider(self.label_provider_id)
+    external_data = db.Column(db.Boolean, nullable=False)
 
-        return None
+    data_folder = db.Column(db.String, nullable=False)
 
-    def labels(self) -> List[Label]:
-        """
-        get a list of labels associated with this project
+    # contraints
+    __table_args__ = ()
 
-        :return: list of labels
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM labels WHERE project = ?', [self.identifier])
-            return list(map(
-                lambda row: Label(self.database, row),
-                cursor.fetchall()
-            ))
+    # relationships to other models
+    files = db.relationship(
+        "File",
+        backref="project",
+        lazy="dynamic",
+        passive_deletes=True,
+    )
 
-    def label_tree(self) -> List[TreeNodeLabel]:
-        """
-        get a list of root labels associated with this project
+    labels = db.relationship(
+        "Label",
+        backref="project",
+        lazy="dynamic",
+        passive_deletes=True,
+    )
 
-        :return: list of labels
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                WITH RECURSIVE
-                    tree AS (
-                        SELECT labels.* FROM labels
-                            WHERE project = ? AND parent IS NULL
-                        UNION ALL
-                        SELECT labels.* FROM labels
-                            JOIN tree ON labels.parent = tree.id
-                    )
-                SELECT * FROM tree
-            ''', [self.identifier])
+    collections = db.relationship(
+        "Collection",
+        backref="project",
+        lazy="dynamic",
+        passive_deletes=True,
+    )
+
+
+    serialize_only = NamedBaseModel.serialize_only + (
+        "created",
+        "description",
+        "model_id",
+        "label_provider_id",
+        "root_folder",
+        "external_data",
+        "data_folder",
+    )
+
+    @commit_on_return
+    def delete(self) -> T.Tuple[dict, dict]:
+        # pylint: disable=unexpected-keyword-arg
+        dump = super().delete(commit=False)
 
-            result = []
-            lookup = {}
+        model_dump = {}
+        if self.model_id is not None:
+            # pylint: disable=unexpected-keyword-arg
+            model_dump = self.model.delete(commit=False)
 
-            for row in cursor.fetchall():
-                label = TreeNodeLabel(self.database, row)
-                lookup[label.identifier] = label
+        if os.path.exists(self.root_folder):
+            # remove from file system
+            shutil.rmtree(self.root_folder)
 
-                if label.parent_id is None:
-                    result.append(label)
-                else:
-                    lookup[label.parent_id].children.append(label)
+        return dump, model_dump
 
-            return result
 
-    def label(self, identifier: int) -> Optional[Label]:
+    def label(self, identifier: int) -> T.Optional[Label]:
         """
         get a label using its unique identifier
 
         :param identifier: unique identifier
         :return: label
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM labels WHERE id = ? AND project = ?',
-                           (identifier, self.identifier))
-            row = cursor.fetchone()
+        return self.labels.filter(Label.id == identifier).one_or_none()
 
-            if row is not None:
-                return Label(self.database, row)
 
-            return None
-
-    def label_by_reference(self, reference: str) -> Optional[Label]:
+    def label_by_reference(self, reference: str) -> T.Optional[Label]:
         """
         get a label using its reference string
 
         :param reference: reference string
         :return: label
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM labels WHERE reference = ? AND project = ?',
-                           (reference, self.identifier))
-            row = cursor.fetchone()
+        return self.labels.filter(Label.reference == reference).one_or_none()
+
+
+    def file(self, identifier: int) -> T.Optional[Label]:
+        """
+        get a file using its unique identifier
+
+        :param identifier: unique identifier
+        :return: file
+        """
+        return self.files.filter(File.id == identifier).one_or_none()
+
+
+    def collection(self, identifier: int) -> T.Optional[Collection]:
+        """
+        get a collection using its unique identifier
 
-            if row is not None:
-                return Label(self.database, row)
+        :param identifier: unique identifier
+        :return: collection
+        """
+        return self.collections.filter(Collection.id == identifier).one_or_none()
 
-            return None
 
-    def create_label(self, name: str, reference: str = None,
-                     parent: Union[Label, int, str] = None,
-                     hierarchy_level: str = None) -> Tuple[Optional[Label], bool]:
+    def collection_by_reference(self, reference: str) -> T.Optional[Collection]:
+        """
+        get a collection using its unique identifier
+
+        :param identifier: unique identifier
+        :return: collection
+        """
+        return self.collections.filter(Collection.reference == reference).one_or_none()
+
+
+    @commit_on_return
+    def create_label(self, name: str,
+                     reference: str = None,
+                     parent: T.Optional[T.Union[int, str, Label]] = None,
+                     hierarchy_level: str = None) -> T.Tuple[T.Optional[Label], bool]:
         """
         create a label for this project. If there is already a label with the same reference
         in the database its name is updated.
 
         :param name: label name
         :param reference: label reference
-        :param parent: either parent identifier, parent reference string or `Label` object
+        :param parent: parent label. Either a reference string, a Label id or a Label instance
         :param hierarchy_level: hierarchy level name
         :return: created or edited label, insert
         """
-        created = int(time())
+        label = None
+        is_new = False
 
-        if isinstance(parent, str):
-            parent = self.label_by_reference(parent)
-        if isinstance(parent, Label):
-            parent = parent.identifier
+        if reference is not None:
+            label = Label.query.filter_by(project_id=self.id, reference=reference).one_or_none()
 
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO labels (project, parent, created, reference, name, hierarchy_level)
-                VALUES (?, ?, ?, ?, ?, ?)
-                ON CONFLICT (project, reference) DO
-                UPDATE SET parent = ?, name = ?, hierarchy_level = ?
-            ''', (self.identifier, parent, created, reference, name, hierarchy_level,
-                  parent, name, hierarchy_level))
+        if label is None:
+            label = Label.new(commit=False, project_id=self.id, reference=reference)
+            is_new = True
 
-            # lastrowid is 0 if on conflict clause applies.
-            # If this is the case we do an extra query to receive the row id.
-            if cursor.lastrowid > 0:
-                row_id = cursor.lastrowid
-                insert = True
-            else:
-                cursor.execute('SELECT id FROM labels WHERE project = ? AND reference = ?',
-                               (self.identifier, reference))
-                row_id = cursor.fetchone()[0]
-                insert = False
+        label.set_name(name, commit=False)
+        label.set_parent(parent, commit=False)
+        label.hierarchy_level = hierarchy_level
 
-        return self.label(row_id), insert
+        return label, is_new
 
-    def collections(self) -> List[Collection]:
+    @commit_on_return
+    def bulk_create_labels(self, labels: T.List[T.Dict], clean_old_labels: bool = True):
         """
-        get a list of collections associated with this project
+            Inserts a all labels at once.
 
-        :return: list of collections
+            :raises:
+                - AssertionError if project_id and reference are not unique
+                - ValueError if a cycle in the hierarchy is found
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM collections WHERE project = ? ORDER BY position ASC',
-                           [self.identifier])
+        if clean_old_labels:
+            self.labels.delete()
 
-            return list(map(
-                lambda row: Collection(self.database, row),
-                cursor.fetchall()
-            ))
+        for label in labels:
+            label["project_id"] = self.id
 
-    def collection(self, identifier: int) -> Optional[Collection]:
-        """
-        get a collection using its unique identifier
+        self.__check_labels(labels)
+        app.logger.info(f"Inserting {len(labels):,d} labels")
+        db.engine.execute(Label.__table__.insert(), labels)
+        self.__set_parents(labels)
 
-        :param identifier: unique identifier
-        :return: collection
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM collections WHERE id = ? AND project = ?',
-                           (identifier, self.identifier))
-            row = cursor.fetchone()
+        return labels
 
-            if row is not None:
-                return Collection(self.database, row)
+    def __set_parents(self, labels):
+        """ after the bul insert, we need to set correct parent_ids """
+        app.logger.info("Setting parents of the labels")
 
-            return None
+        self.flush()
+        for label in labels:
+            if label["parent"] is None:
+                continue
 
-    def collection_by_reference(self, reference: str):
-        """
-        get a collection using its reference string
+            label_obj = self.label_by_reference(label["reference"])
+            parent_label_obj = self.label_by_reference(label["parent"])
 
-        :param reference: reference string
-        :return: collection
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM collections WHERE reference = ? AND project = ?',
-                           (reference, self.identifier))
-            row = cursor.fetchone()
+            label_obj.parent_id = parent_label_obj.id
+
+    # pylint: disable=no-self-use
+    def __check_labels(self, labels):
+        """ check labels for unique keys and cycles """
+
+        unique_keys = dict()
+
+        for label in labels:
+            key = (label["project_id"], label["reference"])
+
+            assert key not in unique_keys, \
+                f"{key} was not unique: ({label=} vs {unique_keys[key]=})!"
 
-            if row is not None:
-                return Collection(self.database, row)
+            unique_keys[key] = label
 
-            return None
 
+
+    # pylint: disable=too-many-arguments
+    @commit_on_return
     def create_collection(self,
                           reference: str,
                           name: str,
                           description: str,
                           position: int,
-                          autoselect: bool) -> Tuple[Collection, bool]:
+                          autoselect: bool) -> T.Tuple[Collection, bool]:
         """
         create a new collection associated with this project
 
@@ -236,74 +241,60 @@ class Project:
 
         :return: collection object, insert
         """
-        autoselect = 1 if autoselect else 0
 
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO collections
-                    (project, reference, name, description, position, autoselect)
-                VALUES (?, ?, ?, ?, ?, ?)
-                ON CONFLICT (project, reference) DO
-                UPDATE SET name = ?, description = ?, position = ?, autoselect = ?
-            ''', (self.identifier, reference, name, description, position, autoselect,
-                  name, description, position, autoselect))
 
-            # lastrowid is 0 if on conflict clause applies.
-            # If this is the case we do an extra query to receive the row id.
-            if cursor.lastrowid > 0:
-                row_id = cursor.lastrowid
-                insert = True
-            else:
-                cursor.execute('SELECT id FROM collections WHERE project = ? AND reference = ?',
-                               (self.identifier, reference))
-                row_id = cursor.fetchone()[0]
-                insert = False
+        collection, is_new = Collection.get_or_create(
+            project_id=self.id, reference=reference)
 
-        return self.collection(row_id), insert
+        collection.name = name
+        collection.description = description
+        collection.position = position
+        collection.autoselect = autoselect
 
-    def remove(self) -> None:
-        """
-        remove this project from the database
+        return collection, is_new
 
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('DELETE FROM projects WHERE id = ?', [self.identifier])
 
-    def set_name(self, name: str) -> None:
+    # pylint: disable=too-many-arguments
+    @commit_on_return
+    def add_file(self,
+                 uuid: str,
+                 file_type: str,
+                 name: str,
+                 extension: str,
+                 size: int,
+                 filename: str,
+                 frames: int = None,
+                 fps: float = None) -> T.Tuple[File, bool]:
         """
-        set this projects name
+        add a file to this project
 
-        :param name: new name
-        :return:
+        :param uuid: unique identifier which is used for temporary files
+        :param file_type: file type (either image or video)
+        :param name: file name
+        :param extension: file extension
+        :param size: file size
+        :param filename: actual name in filesystem
+        :param frames: frame count
+        :param fps: frames per second
+        :return: file
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE projects SET name = ? WHERE id = ?', (name, self.identifier))
-            self.name = name
+        path = os.path.join(self.data_folder, f"{filename}{extension}")
 
-    def set_description(self, description: str) -> None:
-        """
-        set this projects description
+        file, is_new = File.get_or_create(
+            project_id=self.id, path=path)
 
-        :param description: new description
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE projects SET description = ? WHERE id = ?',
-                           (description, self.identifier))
-            self.description = description
+        file.uuid = uuid
+        file.type = file_type
+        file.name = name
+        file.extension = extension
+        file.size = size
+        file.frames = frames
+        file.fps = fps
 
-    def count_files(self) -> int:
-        """
-        count files associated with this project
+        return file, is_new
 
-        :return: count
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT COUNT(*) FROM files WHERE project = ?', [self.identifier])
-            return cursor.fetchone()[0]
 
-    def files(self, offset: int = 0, limit: int = -1) -> Iterator[File]:
+    def get_files(self, *filters, offset: int = 0, limit: int = -1) -> T.List[File]:
         """
         get an iterator of files associated with this project
 
@@ -311,133 +302,61 @@ class Project:
         :param limit: file limit
         :return: iterator of files
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM files WHERE project = ? ORDER BY id ASC LIMIT ? OFFSET ?',
-                           (self.identifier, limit, offset))
-
-            return map(
-                lambda row: File(self.database, row),
-                cursor.fetchall()
-            )
 
-    def count_files_without_results(self) -> int:
-        """
-        count files without associated results
+        return self.files.filter(*filters).order_by(File.id).offset(offset).limit(limit)
 
-        :return: count
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT COUNT(*)
-                FROM files
-                LEFT JOIN results ON files.id = results.file
-                WHERE files.project = ? AND results.id IS NULL
-            ''', [self.identifier])
-            return cursor.fetchone()[0]
 
-    def files_without_results(self) -> Iterator[File]:
+    def _files_without_results(self):
         """
-        get an iterator of files without associated results
+        get files without any results
 
-        :return: list of files
+        :return: a query object
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT files.*
-                FROM files
-                LEFT JOIN results ON files.id = results.file
-                WHERE files.project = ? AND results.id IS NULL
-                ORDER BY id ASC
-            ''', [self.identifier])
+        # pylint: disable=no-member
+        return self.files.filter(~File.results.any())
 
-            for row in cursor:
-                yield File(self.database, row)
 
-    def count_files_without_collection(self) -> int:
+    def count_files_without_results(self) -> int:
         """
-        count files associated with this project but with no collection
+        count files without associated results
 
         :return: count
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT COUNT(*) FROM files WHERE project = ? AND collection IS NULL',
-                           [self.identifier])
-            return cursor.fetchone()[0]
 
-    def files_without_collection(self, offset=0, limit=-1) -> Iterator[File]:
+        return self._files_without_results().count()
+
+
+    def files_without_results(self) -> T.List[File]:
         """
-        get an iterator of files without not associated with any collection
+        get a list of files without associated results
 
         :return: list of files
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                SELECT * FROM files
-                WHERE files.project = ? AND files.collection IS NULL
-                ORDER BY id ASC
-                LIMIT ? OFFSET ?
-            ''', (self.identifier, limit, offset))
+        return self._files_without_results().all()
 
-            for row in cursor:
-                yield File(self.database, row)
 
-    def file(self, identifier) -> Optional[File]:
+    def _files_without_collection(self, offset: int = 0, limit: int = -1):
         """
-        get a file using its unique identifier
+        get files without a collection
 
-        :param identifier: unique identifier
-        :return: file
+        :return: a query object
+        """
+        # pylint: disable=no-member
+        return self.get_files(File.collection_id.is_(None), offset=offset, limit=limit)
+
+    def files_without_collection(self, offset: int = 0, limit: int = -1) -> T.List[File]:
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('SELECT * FROM files WHERE id = ? AND project = ?',
-                           (identifier, self.identifier))
-            row = cursor.fetchone()
+        get a list of files without a collection
 
-            if row is not None:
-                return File(self.database, row)
+        :return: list of files
+        """
+        return self._files_without_collection(offset=offset, limit=limit).all()
 
-            return None
 
-    def add_file(self, uuid: str, file_type: str, name: str, extension: str, size: int,
-                 filename: str, frames: int = None, fps: float = None) -> Tuple[File, bool]:
+    def count_files_without_collection(self) -> int:
         """
-        add a file to this project
+        count files associated with this project but without a collection
 
-        :param uuid: unique identifier which is used for temporary files
-        :param file_type: file type (either image or video)
-        :param name: file name
-        :param extension: file extension
-        :param size: file size
-        :param filename: actual name in filesystem
-        :param frames: frame count
-        :param fps: frames per second
-        :return: file
+        :return: count
         """
-        created = int(time())
-        path = join(self.data_folder, filename + extension)
-
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO files (
-                    uuid, project, type, name, extension, size, created, path, frames, fps
-                )
-                VALUES (  
-                       ?,       ?,    ?,    ?,         ?,    ?,       ?,    ?,      ?,   ?
-                )
-                ON CONFLICT (project, path) DO
-                UPDATE SET type = ?, name = ?, extension = ?, size = ?, frames = ?, fps = ?
-            ''', (uuid, self.identifier, file_type, name, extension, size, created, path, frames,
-                  fps, file_type, name, extension, size, frames, fps))
-
-            # lastrowid is 0 if on conflict clause applies.
-            # If this is the case we do an extra query to receive the row id.
-            if cursor.lastrowid > 0:
-                row_id = cursor.lastrowid
-                insert = True
-            else:
-                cursor.execute('SELECT id FROM files WHERE project = ? AND path = ?',
-                               (self.identifier, path))
-                row_id = cursor.fetchone()[0]
-                insert = False
-
-        return self.file(row_id), insert
+        return self._files_without_collection().count()

pycs/database/Result.py

@@ -1,41 +1,63 @@
-from contextlib import closing
-
-from json import dumps, loads
-
-
-class Result:
-    """
-    database class for results
-    """
-
-    def __init__(self, database, row):
-        self.database = database
-
-        self.identifier = row[0]
-        self.file_id = row[1]
-        self.origin = row[2]
-        self.type = row[3]
-        self.label = row[4]
-        self.data = loads(row[5]) if row[5] is not None else None
-
-    def file(self):
+import json
+import typing as T
+
+from pycs import db
+from pycs.database.base import BaseModel
+from pycs.database.util import commit_on_return
+
+class Result(BaseModel):
+    """ DB Model for projects """
+
+    file_id = db.Column(
+        db.Integer,
+        db.ForeignKey("file.id", ondelete="CASCADE"),
+        nullable=False)
+
+    origin = db.Column(db.String, nullable=False)
+    type = db.Column(db.String, nullable=False)
+
+    label_id = db.Column(
+        db.Integer,
+        db.ForeignKey("label.id", ondelete="SET NULL"),
+        nullable=True)
+
+    data_encoded = db.Column(db.String)
+
+    serialize_only = BaseModel.serialize_only + (
+        "file_id",
+        "origin",
+        "type",
+        "label_id",
+        "data",
+    )
+
+    def serialize(self):
+        """ extends the default serialize with the decoded data attribute """
+        result = super().serialize()
+        result["data"] = self.data
+        return result
+
+    @property
+    def data(self) -> T.Optional[dict]:
+        """ getter for the decoded data attribute """
+        return None if self.data_encoded is None else json.loads(self.data_encoded)
+
+    @data.setter
+    def data(self, value):
         """
-        getter for the according file
-
-        :return: file object
+            setter for the decoded data attribute
+            The attribute is encoded property before assigned to the object.
         """
+        if isinstance(value, str) or value is None:
+            self.data_encoded = value
 
-        return self.database.file(self.file_id)
-
-    def remove(self):
-        """
-        remove this result from the database
+        elif isinstance(value, (dict, list)):
+            self.data_encoded = json.dumps(value)
 
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('DELETE FROM results WHERE id = ?', [self.identifier])
+        else:
+            raise ValueError(f"Not supported type: {type(value)}")
 
+    @commit_on_return
     def set_origin(self, origin: str):
         """
         set this results origin
@@ -43,33 +65,15 @@ class Result:
         :param origin: either 'user' or 'pipeline'
         :return:
         """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE results SET origin = ? WHERE id = ?', (origin, self.identifier))
-            self.origin = origin
+        self.origin = origin
 
-    def set_label(self, label: int):
-        """
-        set this results label
-
-        :param label: label id
-        :return:
-        """
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE results SET label = ? WHERE id = ?', (label, self.identifier))
-            self.label = label
 
-    def set_data(self, data: dict):
+    @commit_on_return
+    def set_label(self, label: int):
         """
-        set this results data object
+        set this results origin
 
-        :param data: data object
+        :param label: label ID
         :return:
         """
-        if data is None:
-            data_txt = None
-        else:
-            data_txt = dumps(data)
-
-        with closing(self.database.con.cursor()) as cursor:
-            cursor.execute('UPDATE results SET data = ? WHERE id = ?', (data_txt, self.identifier))
-            self.data = data
+        self.label_id = label

pycs/database/base.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import typing as T
+
+from flask import abort
+from sqlalchemy_serializer import SerializerMixin
+
+from pycs import db
+from pycs.database.util import commit_on_return
+
+class BaseModel(db.Model, SerializerMixin):
+    """ Base model class """
+    __abstract__ = True
+
+    # setup of the SerializerMixin
+    date_format = '%s'  # Unixtimestamp (seconds)
+    datetime_format = '%d. %b. %Y %H:%M:%S'
+    time_format = '%H:%M'
+
+
+    id = db.Column(db.Integer, primary_key=True)
+
+    serialize_only = ("id", "identifier")
+
+    def identifier(self):
+        """ alias for id attribute """
+        return self.id
+
+    def __repr__(self):
+        attrs = self.serialize()
+        content = ", ".join([f"{attr}={value}" for attr, value in attrs.items()])
+        return f"<{self.__class__.__name__}: {content}>"
+
+
+    def serialize(self) -> dict:
+        """ default model serialization method """
+        return self.to_dict()
+
+
+    @commit_on_return
+    def delete(self) -> dict:
+        """
+        delete this instance from the database
+
+        :return: serialized self
+        """
+        dump = self.serialize()
+        db.session.delete(self)
+
+        return dump
+
+
+    # do an alias
+    remove = delete
+
+
+    @classmethod
+    def new(cls, commit: bool = True, **kwargs):
+        """ creates a new object. optionally commits the created object. """
+        obj = cls(**kwargs)
+        db.session.add(obj)
+
+        if commit:
+            obj.commit()
+
+        return obj
+
+    @classmethod
+    def get_or_create(cls, **kwargs) -> T.Tuple[BaseModel, bool]:
+        """ get an object from the DB based on the kwargs, or create an object with these. """
+
+        is_new = False
+
+        obj = cls.query.filter_by(**kwargs).one_or_none()
+
+        if obj is None:
+            obj = cls.new(commit=False, **kwargs)
+            is_new = True
+
+        return obj, is_new
+
+
+    @classmethod
+    def get_or_404(cls, obj_id: int) -> BaseModel:
+        """ get an object for the given id or raise 404 error if the object is not present """
+        obj = cls.query.get(obj_id)
+
+        if obj is None:
+            abort(404, f"{cls.__name__} with ID {obj_id} could not be found!")
+
+        return obj
+
+
+    @staticmethod
+    def commit():
+        """ commit current session """
+        db.session.commit()
+
+    @staticmethod
+    def flush():
+        """ flush current session """
+        db.session.flush()
+
+
+class NamedBaseModel(BaseModel):
+    """ Extends the base model with a name attribute. """
+    __abstract__ = True
+
+    name = db.Column(db.String, nullable=False)
+
+    serialize_only = BaseModel.serialize_only + ("name",)
+
+    @commit_on_return
+    def set_name(self, name: str):
+        """ set the name attribute """
+        self.name = name

pycs/database/discovery/LabelProviderDiscovery.py

@@ -1,49 +0,0 @@
-import re
-from contextlib import closing
-from glob import glob
-from json import load
-from os import path, listdir
-
-
-def __find_files():
-    # list folders in labels/
-    for folder in glob('labels/*'):
-        # list files
-        for filename in listdir(folder):
-            file_path = path.join(folder, filename)
-
-            # filter configuration files
-            if not path.isfile(file_path):
-                continue
-
-            if not re.match(r'^configuration(\d+)?\.json$', filename):
-                continue
-
-            # yield element
-            yield folder, filename, file_path
-
-
-def discover(database):
-    """
-    find label providers in the corresponding folder and add them to the database
-
-    :param database:
-    :return:
-    """
-    with closing(database.cursor()) as cursor:
-        for folder, configuration_file, configuration_path in __find_files():
-            # load configuration file
-            with open(configuration_path, 'r') as file:
-                label = load(file)
-
-            # extract data
-            name = label['name']
-            description = label['description'] if 'description' in label else None
-
-            # save to database
-            cursor.execute('''
-                INSERT INTO label_providers (name, description, root_folder, configuration_file)
-                VALUES (?, ?, ?, ?)
-                ON CONFLICT (root_folder, configuration_file)
-                DO UPDATE SET name = ?, description = ?
-            ''', (name, description, folder, configuration_file, name, description))

pycs/database/discovery/ModelDiscovery.py

@@ -1,32 +0,0 @@
-from contextlib import closing
-from glob import glob
-from json import load, dumps
-from os import path
-
-
-def discover(database):
-    """
-    find models in the corresponding folder and add them to the database
-
-    :param database:
-    :return:
-    """
-    with closing(database.cursor()) as cursor:
-        # list folders in models/
-        for folder in glob('models/*'):
-            # load distribution.json
-            with open(path.join(folder, 'configuration.json'), 'r') as file:
-                model = load(file)
-
-            # extract data
-            name = model['name']
-            description = model['description'] if 'description' in model else None
-            supports = dumps(model['supports'])
-
-            # save to database
-            cursor.execute('''
-                INSERT INTO models (name, description, root_folder, supports)
-                VALUES (?, ?, ?, ?)
-                ON CONFLICT (root_folder)
-                DO UPDATE SET name = ?, description = ?, supports = ?
-            ''', (name, description, folder, supports, name, description, supports))

pycs/database/util/JSONEncoder.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from flask.json import JSONEncoder as Base
 
+from pycs.database.base import BaseModel
 
 class JSONEncoder(Base):
     """
@@ -9,6 +10,7 @@ class JSONEncoder(Base):
     """
 
     def default(self, o: Any) -> Any:
-        copy = o.__dict__.copy()
-        del copy['database']
-        return copy
+        if isinstance(o, BaseModel):
+            return o.serialize()
+
+        return o.__dict__.copy()

pycs/database/util/TreeNodeLabel.py

@@ -1,7 +0,0 @@
-from pycs.database.Label import Label
-
-
-class TreeNodeLabel(Label):
-    def __init__(self, database, row):
-        super().__init__(database, row)
-        self.children = []

pycs/database/util/__init__.py

@@ -0,0 +1,21 @@
+from functools import wraps
+
+from pycs import db
+
+def commit_on_return(method):
+    """
+        Decorator for model's methods. It provides an extra argument to the method: commit.
+        On return if commit is True, then the database session is commited.
+    """
+
+    @wraps(method)
+    def inner(self, *args, commit: bool = True, **kwargs):
+
+        res = method(self, *args, **kwargs)
+
+        if commit:
+            db.session.commit()
+
+        return res
+
+    return inner

pycs/frontend/WebServer.py

@@ -1,12 +1,15 @@
+import os
+import logging.config
+
 from glob import glob
-from os import path, getcwd
-from os.path import exists
 
 import eventlet
 import socketio
-from flask import Flask, send_from_directory
 
-from pycs.database.Database import Database
+from flask import send_from_directory
+
+from pycs.database.Model import Model
+from pycs.database.LabelProvider import LabelProvider
 from pycs.frontend.endpoints.ListJobs import ListJobs
 from pycs.frontend.endpoints.ListLabelProviders import ListLabelProviders
 from pycs.frontend.endpoints.ListModels import ListModels
@@ -34,8 +37,8 @@ from pycs.frontend.endpoints.projects.EditProjectName import EditProjectName
 from pycs.frontend.endpoints.projects.ExecuteExternalStorage import ExecuteExternalStorage
 from pycs.frontend.endpoints.projects.ExecuteLabelProvider import ExecuteLabelProvider
 from pycs.frontend.endpoints.projects.GetProjectModel import GetProjectModel
-from pycs.frontend.endpoints.projects.ListCollections import ListCollections
-from pycs.frontend.endpoints.projects.ListFiles import ListFiles
+from pycs.frontend.endpoints.projects.ListProjectCollections import ListProjectCollections
+from pycs.frontend.endpoints.projects.ListProjectFiles import ListProjectFiles
 from pycs.frontend.endpoints.projects.RemoveProject import RemoveProject
 from pycs.frontend.endpoints.results.ConfirmResult import ConfirmResult
 from pycs.frontend.endpoints.results.CreateResult import CreateResult
@@ -56,12 +59,14 @@ class WebServer:
     wrapper class for flask and socket.io which initializes most networking
     """
 
-    # pylint: disable=line-too-long
-    # pylint: disable=too-many-statements
-    def __init__(self, settings: dict, database: Database, jobs: JobRunner, pipelines: PipelineCache):
+    def __init__(self, app, settings: dict, discovery: bool = True):
+
+        logging.config.dictConfig(settings.logging)
+        is_production = os.path.exists('webui/index.html')
+
         # initialize web server
-        if exists('webui/index.html'):
-            print('production build')
+        if is_production:
+            app.logger.info('production build')
 
             # find static files and folders
             static_files = {}
@@ -82,219 +87,240 @@ class WebServer:
             else:
                 self.__sio = socketio.Server(async_mode='eventlet')
 
-            self.__flask = Flask(__name__)
-            self.__app = socketio.WSGIApp(self.__sio, self.__flask, static_files=static_files)
+            self.wsgi_app = socketio.WSGIApp(self.__sio, app, static_files=static_files)
 
             # overwrite root path to serve index.html
-            @self.__flask.route('/', methods=['GET'])
+            @app.route('/', methods=['GET'])
             def index():
                 # pylint: disable=unused-variable
-                return send_from_directory(path.join(getcwd(), 'webui'), 'index.html')
+                return send_from_directory(os.path.join(os.getcwd(), 'webui'), 'index.html')
 
         else:
-            print('development build')
+            app.logger.info('development build')
 
             # create service objects
             self.__sio = socketio.Server(cors_allowed_origins='*', async_mode='eventlet')
-            self.__flask = Flask(__name__)
-            self.__app = socketio.WSGIApp(self.__sio, self.__flask)
+            self.app = app
+            self.wsgi_app = socketio.WSGIApp(self.__sio, app)
 
             # set access control header to allow requests from Vue.js development server
-            @self.__flask.after_request
+            @self.app.after_request
             def after_request(response):
                 # pylint: disable=unused-variable
                 response.headers['Access-Control-Allow-Origin'] = '*'
                 return response
 
         # set json encoder so database objects are serialized correctly
-        self.__flask.json_encoder = JSONEncoder
+        self.app.json_encoder = JSONEncoder
+
+        self.host = settings.host
+        self.port = settings.port
 
         # create notification manager
-        notifications = NotificationManager(self.__sio)
+        self.jobs = JobRunner()
+        self.pipelines = PipelineCache(self.jobs, settings.get("pipeline_cache_time"))
+        self.notifications = NotificationManager(self.__sio)
+
+        self.jobs.on_create(self.notifications.create_job)
+        self.jobs.on_start(self.notifications.edit_job)
+        self.jobs.on_progress(self.notifications.edit_job)
+        self.jobs.on_finish(self.notifications.edit_job)
+        self.jobs.on_remove(self.notifications.remove_job)
 
-        jobs.on_create(notifications.create_job)
-        jobs.on_start(notifications.edit_job)
-        jobs.on_progress(notifications.edit_job)
-        jobs.on_finish(notifications.edit_job)
-        jobs.on_remove(notifications.remove_job)
+        self.define_routes()
+
+        if discovery:
+            Model.discover("models/")
+            LabelProvider.discover("labels/")
+
+
+    def define_routes(self):
+        """ defines app routes """
 
         # additional
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/folder',
             view_func=FolderInformation.as_view('folder_information')
         )
 
         # jobs
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/jobs',
-            view_func=ListJobs.as_view('list_jobs', jobs)
+            view_func=ListJobs.as_view('list_jobs', self.jobs)
         )
-        self.__flask.add_url_rule(
-            '/jobs/<identifier>/remove',
-            view_func=RemoveJob.as_view('remove_job', jobs)
+        self.app.add_url_rule(
+            '/jobs/<job_id>/remove',
+            view_func=RemoveJob.as_view('remove_job', self.jobs)
         )
 
         # models
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/models',
-            view_func=ListModels.as_view('list_models', database)
+            view_func=ListModels.as_view('list_models')
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/model',
-            view_func=GetProjectModel.as_view('get_project_model', database)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/model',
+            view_func=GetProjectModel.as_view('get_project_model')
         )
 
         # labels
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/label_providers',
-            view_func=ListLabelProviders.as_view('label_providers', database)
+            view_func=ListLabelProviders.as_view('label_providers')
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/labels',
-            view_func=ListLabels.as_view('list_labels', database)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/labels',
+            view_func=ListLabels.as_view('list_labels')
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/labels/tree',
-            view_func=ListLabelTree.as_view('list_label_tree', database)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/labels/tree',
+            view_func=ListLabelTree.as_view('list_label_tree')
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/labels',
-            view_func=CreateLabel.as_view('create_label', database, notifications)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/labels',
+            view_func=CreateLabel.as_view('create_label', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/labels/<int:label_id>/remove',
-            view_func=RemoveLabel.as_view('remove_label', database, notifications)
+            view_func=RemoveLabel.as_view('remove_label', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/labels/<int:label_id>/name',
-            view_func=EditLabelName.as_view('edit_label_name', database, notifications)
+            view_func=EditLabelName.as_view('edit_label_name', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/labels/<int:label_id>/parent',
-            view_func=EditLabelParent.as_view('edit_label_parent', database, notifications)
+            view_func=EditLabelParent.as_view('edit_label_parent', self.notifications)
         )
 
         # collections
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/collections',
-            view_func=ListCollections.as_view('list_collections', database)
+            view_func=ListProjectCollections.as_view('list_collections')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/data/<int:collection_id>/<int:start>/<int:length>',
-            view_func=ListFiles.as_view('list_collection_files', database)
+            view_func=ListProjectFiles.as_view('list_collection_files')
         )
 
         # data
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/data',
-            view_func=UploadFile.as_view('upload_file', database, notifications)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/data',
+            view_func=UploadFile.as_view('upload_file', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/data',
+            view_func=ListProjectFiles.as_view('list_all_files')
+        )
+        self.app.add_url_rule(
             '/projects/<int:project_id>/data/<int:start>/<int:length>',
-            view_func=ListFiles.as_view('list_files', database)
+            view_func=ListProjectFiles.as_view('list_files')
         )
-        self.__flask.add_url_rule(
-            '/data/<int:identifier>/remove',
-            view_func=RemoveFile.as_view('remove_file', database, notifications)
+        self.app.add_url_rule(
+            '/data/<int:file_id>/remove',
+            view_func=RemoveFile.as_view('remove_file', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>',
-            view_func=GetFile.as_view('get_file', database)
+            view_func=GetFile.as_view('get_file')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/<resolution>',
-            view_func=GetResizedFile.as_view('get_resized_file', database)
+            view_func=GetResizedFile.as_view('get_resized_file')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/<resolution>/<crop_box>',
-            view_func=GetCroppedFile.as_view('crop_result', database)
+            view_func=GetCroppedFile.as_view('get_cropped_file')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/previous_next',
-            view_func=GetPreviousAndNextFile.as_view('get_previous_and_next_file', database)
+            view_func=GetPreviousAndNextFile.as_view('get_previous_and_next_file')
         )
 
         # results
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/results',
-            view_func=GetProjectResults.as_view('get_project_results', database)
+            view_func=GetProjectResults.as_view('get_project_results')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/results',
-            view_func=GetResults.as_view('get_results', database)
+            view_func=GetResults.as_view('get_results')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/results',
-            view_func=CreateResult.as_view('create_result', database, notifications)
+            view_func=CreateResult.as_view('create_result', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/reset',
-            view_func=ResetResults.as_view('reset_results', database, notifications)
+            view_func=ResetResults.as_view('reset_results', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/results/<int:result_id>/remove',
-            view_func=RemoveResult.as_view('remove_result', database, notifications)
+            view_func=RemoveResult.as_view('remove_result', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/results/<int:result_id>/confirm',
-            view_func=ConfirmResult.as_view('confirm_result', database, notifications)
+            view_func=ConfirmResult.as_view('confirm_result', self.notifications)
         )
-        self.__flask.add_url_rule(
+
+        self.app.add_url_rule(
             '/results/<int:result_id>/label',
-            view_func=EditResultLabel.as_view('edit_result_label', database, notifications)
+            view_func=EditResultLabel.as_view('edit_result_label', self.notifications)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/results/<int:result_id>/data',
-            view_func=EditResultData.as_view('edit_result_data', database, notifications)
+            view_func=EditResultData.as_view('edit_result_data', self.notifications)
         )
 
         # projects
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects',
-            view_func=ListProjects.as_view('list_projects', database)
+            view_func=ListProjects.as_view('list_projects')
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects',
-            view_func=CreateProject.as_view('create_project', database, notifications, jobs)
+            view_func=CreateProject.as_view('create_project', self.notifications, self.jobs)
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/label_provider',
-            view_func=ExecuteLabelProvider.as_view('execute_label_provider', database,
-                                                   notifications, jobs)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/label_provider',
+            view_func=ExecuteLabelProvider.as_view('execute_label_provider',
+                                                   self.notifications, self.jobs)
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/external_storage',
-            view_func=ExecuteExternalStorage.as_view('execute_external_storage', database,
-                                                     notifications, jobs)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/external_storage',
+            view_func=ExecuteExternalStorage.as_view('execute_external_storage',
+                                                     self.notifications, self.jobs)
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/remove',
-            view_func=RemoveProject.as_view('remove_project', database, notifications)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/remove',
+            view_func=RemoveProject.as_view('remove_project', self.notifications)
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/name',
-            view_func=EditProjectName.as_view('edit_project_name', database, notifications)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/name',
+            view_func=EditProjectName.as_view('edit_project_name', self.notifications)
         )
-        self.__flask.add_url_rule(
-            '/projects/<int:identifier>/description',
-            view_func=EditProjectDescription.as_view('edit_project_description', database,
-                                                     notifications)
+        self.app.add_url_rule(
+            '/projects/<int:project_id>/description',
+            view_func=EditProjectDescription.as_view('edit_project_description', self.notifications)
         )
 
         # pipelines
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/pipelines/fit',
-            view_func=FitModel.as_view('fit_model', database, jobs, pipelines)
+            view_func=FitModel.as_view('fit_model', self.jobs, self.pipelines)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/projects/<int:project_id>/pipelines/predict',
-            view_func=PredictModel.as_view('predict_model', database, notifications, jobs,
-                                           pipelines)
+            view_func=PredictModel.as_view('predict_model', self.notifications, self.jobs,
+                                           self.pipelines)
         )
-        self.__flask.add_url_rule(
+        self.app.add_url_rule(
             '/data/<int:file_id>/predict',
-            view_func=PredictFile.as_view('predict_file', database, notifications, jobs, pipelines)
+            view_func=PredictFile.as_view('predict_file', self.notifications,
+                                          self.jobs, self.pipelines)
         )
 
-        # finally start web server
-        eventlet.wsgi.server(eventlet.listen((settings['host'], settings['port'])), self.__app)
+    def run(self):
+        """ start web server """
+        self.pipelines.start()
+        eventlet.wsgi.server(eventlet.listen((self.host, self.port)), self.wsgi_app)

pycs/frontend/endpoints/ListLabelProviders.py

@@ -1,7 +1,7 @@
 from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.LabelProvider import LabelProvider
 
 
 class ListLabelProviders(View):
@@ -11,10 +11,6 @@ class ListLabelProviders(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self):
-        label_providers = list(self.db.label_providers())
-        return jsonify(label_providers)
+        return jsonify(LabelProvider.query.all())

pycs/frontend/endpoints/ListModels.py

@@ -1,7 +1,7 @@
 from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Model import Model
 
 
 class ListModels(View):
@@ -11,10 +11,6 @@ class ListModels(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self):
-        models = list(self.db.models())
-        return jsonify(models)
+        return jsonify(Model.query.all())

pycs/frontend/endpoints/ListProjects.py

@@ -1,7 +1,7 @@
 from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 
 
 class ListProjects(View):
@@ -11,10 +11,6 @@ class ListProjects(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self):
-        projects = list(self.db.projects())
-        return jsonify(projects)
+        return jsonify(Project.query.all())

pycs/frontend/endpoints/additional/FolderInformation.py

@@ -1,6 +1,8 @@
-from os import path, listdir
+import os
 
-from flask import request, abort, jsonify
+from flask import abort
+from flask import jsonify
+from flask import request
 from flask.views import View
 
 
@@ -21,12 +23,12 @@ class FolderInformation(View):
 
         # check if directory exists
         result = {
-            'exists': path.exists(folder)
+            'exists': os.path.exists(folder)
         }
 
         # count files
         if result['exists']:
-            result['count'] = len(listdir(folder))
+            result['count'] = len(os.listdir(folder))
 
         # send result
         return jsonify(result)

pycs/frontend/endpoints/data/GetCroppedFile.py

@@ -6,7 +6,8 @@ from flask import abort
 from flask import send_from_directory
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.File import File
+from pycs.util.FileOperations import BoundingBox
 from pycs.util.FileOperations import crop_file
 
 
@@ -17,17 +18,12 @@ class GetCroppedFile(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self, file_id: int, resolution: str, crop_box: str):
         # get file from database
-        file = self.db.file(file_id)
-        if file is None:
-            return abort(404)
+        file = File.get_or_404(file_id)
 
-        project = file.project()
+        project = file.project
 
         if not os.path.exists(file.absolute_path):
             abort(404, "File not found!")
@@ -43,10 +39,10 @@ class GetCroppedFile(View):
         crop_w = float(crop_box[2]) if len(crop_box) > 2 else 1 - crop_x
         crop_h = float(crop_box[3]) if len(crop_box) > 3 else 1 - crop_y
 
+        box = BoundingBox(x=crop_x, y=crop_y, w=crop_w, h=crop_h)
         # crop file
         file_directory, file_name = tpool.execute(crop_file, file, project.root_folder,
-                                                  crop_x, crop_y, crop_w, crop_h,
-                                                  max_width, max_height)
+                                                  box, max_width, max_height)
 
         # send to client
         return send_from_directory(file_directory, file_name)

pycs/frontend/endpoints/data/GetFile.py

@@ -1,9 +1,9 @@
-from os import path, getcwd
+import os
 
-from flask import abort, send_from_directory
+from flask import send_from_directory
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.File import File
 
 
 class GetFile(View):
@@ -13,23 +13,11 @@ class GetFile(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
-
     def dispatch_request(self, file_id: int):
-        # get file from database
-        file = self.db.file(file_id)
-
-        if file is None:
-            return abort(404)
+        file = File.get_or_404(file_id)
 
-        # get absolute path
-        if path.isabs(file.path):
-            abs_file_path = file.path
-        else:
-            abs_file_path = path.join(getcwd(), file.path)
+        abs_file_path = file.absolute_path
 
         # return data
-        file_directory, file_name = path.split(abs_file_path)
+        file_directory, file_name = os.path.split(abs_file_path)
         return send_from_directory(file_directory, file_name)

pycs/frontend/endpoints/data/GetPreviousAndNextFile.py

@@ -1,7 +1,7 @@
-from flask import abort, jsonify
+from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.File import File
 
 
 class GetPreviousAndNextFile(View):
@@ -11,16 +11,10 @@ class GetPreviousAndNextFile(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self, file_id: int):
         # get file from database
-        file = self.db.file(file_id)
-
-        if file is None:
-            return abort(404)
+        file = File.get_or_404(file_id)
 
         # get previous and next
         result = {

pycs/frontend/endpoints/data/GetResizedFile.py

@@ -6,7 +6,7 @@ from flask import abort
 from flask import send_from_directory
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.File import File
 from pycs.util.FileOperations import resize_file
 
 
@@ -17,17 +17,12 @@ class GetResizedFile(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self, file_id: int, resolution: str):
         # get file from database
-        file = self.db.file(file_id)
-        if file is None:
-            return abort(404)
+        file = File.get_or_404(file_id)
 
-        project = file.project()
+        project = file.project
 
         if not os.path.exists(file.absolute_path):
             abort(404, "File not found!")

pycs/frontend/endpoints/data/RemoveFile.py

@@ -1,9 +1,8 @@
-from os import remove
-
 from flask import make_response, request, abort
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
+from pycs.database.File import File
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -14,38 +13,30 @@ class RemoveFile(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, file_id: int):
         # extract request data
         data = request.get_json(force=True)
 
-        if 'remove' not in data or data['remove'] is not True:
-            return abort(400)
+        if not data.get('remove', False):
+            abort(400, "remove flag is missing")
 
         # start transaction
-        with self.db:
+        with db.session.begin_nested():
             # find file
-            file = self.db.file(identifier)
-            if file is None:
-                return abort(400)
+            file = File.get_or_404(file_id)
 
             # check if project uses an external data directory
-            project = file.project()
-            if project.external_data:
-                return abort(400)
+            if file.project.external_data:
+                abort(400,
+                    "Cannot remove file, project is setup with external data!")
 
             # remove file from database
-            file.remove()
-
-            # remove file from folder
-            remove(file.path)
-
-            # TODO remove temp files
+            file_dump = file.delete()
 
         # send notification
-        self.nm.remove_file(file)
+        self.nm.remove_file(file_dump)
         return make_response()

pycs/frontend/endpoints/data/UploadFile.py

@@ -1,12 +1,12 @@
-from os import path
-from uuid import uuid1
+import os
+import uuid
 
 from eventlet import tpool
 from flask import make_response, request, abort
 from flask.views import View
 from werkzeug import formparser
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.util.FileOperations import file_info
 
@@ -18,23 +18,19 @@ class UploadFile(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
         self.data_folder = None
-        self.file_id = None
-        self.file_name = None
-        self.file_extension = None
-        self.file_size = None
+        self.uuid = None
+        self.name = None
+        self.extension = None
+        self.size = None
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id: int):
         # find project
-        project = self.db.project(identifier)
-
-        if project is None:
-            return abort(404, "Project not found")
+        project = Project.get_or_404(project_id)
 
         # abort if external storage is used
         if project.external_data:
@@ -42,11 +38,12 @@ class UploadFile(View):
 
         # get upload path and id
         self.data_folder = project.data_folder
-        self.file_id = str(uuid1())
+        self.uuid = str(uuid.uuid1())
 
         # parse upload data
         _, _, files = tpool.execute(formparser.parse_form_data,
-                                    request.environ, stream_factory=self.custom_stream_factory)
+                                    request.environ,
+                                    stream_factory=self.custom_stream_factory)
 
         # abort if there is no file entry in uploaded data
         if 'file' not in files.keys():
@@ -55,14 +52,21 @@ class UploadFile(View):
         # detect file type
         try:
             ftype, frames, fps = tpool.execute(file_info,
-                                               self.data_folder, self.file_id, self.file_extension)
+                                               self.data_folder,
+                                               self.uuid,
+                                               self.extension)
         except ValueError as exception:
             return abort(400, str(exception))
 
-        # add to project files
-        with self.db:
-            file, _ = project.add_file(self.file_id, ftype, self.file_name, self.file_extension,
-                                       self.file_size, self.file_id, frames, fps)
+        file, _ = project.add_file(
+            uuid=self.uuid,
+            file_type=ftype,
+            name=self.name,
+            extension=self.extension,
+            size=self.size,
+            filename=self.uuid,
+            frames=frames,
+            fps=fps)
 
         # send update
         self.nm.create_file(file)
@@ -83,13 +87,14 @@ class UploadFile(View):
         """
         # pylint: disable=unused-argument
         # set relevant properties
-        self.file_name, self.file_extension = path.splitext(filename)
+        self.name, self.extension = os.path.splitext(filename)
 
         if content_length is not None and content_length > 0:
-            self.file_size = content_length
+            self.size = content_length
         else:
-            self.file_size = total_content_length
+            self.size = total_content_length
 
         # open file handler
-        file_path = path.join(self.data_folder, f'{self.file_id}{self.file_extension}')
+        file_path = os.path.join(self.data_folder, f'{self.uuid}{self.extension}')
+        #pylint: disable=consider-using-with
         return open(file_path, 'wb')

pycs/frontend/endpoints/jobs/RemoveJob.py

@@ -1,4 +1,6 @@
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
 from pycs.jobs.JobRunner import JobRunner
@@ -15,15 +17,15 @@ class RemoveJob(View):
         # pylint: disable=invalid-name
         self.jobs = jobs
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, job_id):
         # extract request data
         data = request.get_json(force=True)
 
-        if 'remove' not in data or data['remove'] is not True:
-            abort(400)
+        if not data.get('remove', False):
+            abort(400, "remove flag is missing")
 
         # remove job
-        self.jobs.remove(identifier)
+        self.jobs.remove(job_id)
 
         # return success response
         return make_response()

pycs/frontend/endpoints/labels/CreateLabel.py

@@ -1,7 +1,10 @@
-from flask import request, abort, make_response
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,30 +15,35 @@ class CreateLabel(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
-    def dispatch_request(self, identifier):
+
+    def dispatch_request(self, project_id):
         # extract request data
         data = request.get_json(force=True)
+        name = data.get('name')
 
-        if 'name' not in data:
+        if name is None:
             abort(400)
 
         name = data['name']
-        parent = data['parent'] if 'parent' in data else None
+        reference = data.get('reference')
+        parent = data.get('parent')
 
         # find project
-        project = self.db.project(identifier)
-        if project is None:
-            abort(404)
-
-        # start transaction
-        with self.db:
-            # insert label
-            label, _ = project.create_label(name, parent=parent)
+        project = Project.get_or_404(project_id)
+
+        # insert label
+        with db.session.begin_nested():
+            label, is_new = project.create_label(
+                name=name,
+                reference=reference,
+                parent=parent,
+                commit=False)
+            if not is_new:
+                abort(400, f"Label already exists: {label}")
 
         # send notification
         self.nm.create_label(label)

pycs/frontend/endpoints/labels/EditLabelName.py

@@ -1,7 +1,9 @@
-from flask import request, abort, make_response
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Label import Label
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,32 +14,28 @@ class EditLabelName(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
+
     def dispatch_request(self, project_id: int, label_id: int):
         # extract request data
         data = request.get_json(force=True)
+        name = data.get('name')
 
-        if 'name' not in data:
-            abort(400)
-
-        # find project
-        project = self.db.project(project_id)
-        if project is None:
-            abort(404)
+        if name is None:
+            abort(400, 'name argument is missing')
 
         # find label
-        label = project.label(label_id)
+        label = Label.query.filter(
+            Label.project_id == project_id,
+            Label.id == label_id).one_or_none()
+
         if label is None:
             abort(404)
 
-        # start transaction
-        with self.db:
-            # change name
-            label.set_name(data['name'])
+        label.set_name(name)
 
         # send notification
         self.nm.edit_label(label)

pycs/frontend/endpoints/labels/EditLabelParent.py

@@ -1,7 +1,9 @@
-from flask import request, abort, make_response
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Label import Label
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,32 +14,28 @@ class EditLabelParent(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
+
     def dispatch_request(self, project_id: int, label_id: int):
         # extract request data
         data = request.get_json(force=True)
+        parent = data.get('parent')
 
-        if 'parent' not in data:
-            abort(400)
-
-        # find project
-        project = self.db.project(project_id)
-        if project is None:
-            abort(404)
+        if parent is None:
+            abort(400, 'parent argument is missing')
 
         # find label
-        label = project.label(label_id)
+        label = Label.query.filter(
+            Label.project_id == project_id,
+            Label.id == label_id).one_or_none()
+
         if label is None:
             abort(404)
 
-        # start transaction
-        with self.db:
-            # change parent
-            label.set_parent(data['parent'])
+        label.set_parent(parent)
 
         # send notification
         self.nm.edit_label(label)

pycs/frontend/endpoints/labels/ListLabelTree.py

@@ -1,7 +1,7 @@
-from flask import abort, jsonify
+from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 
 
 class ListLabelTree(View):
@@ -11,18 +11,24 @@ class ListLabelTree(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id):
         # find project
-        project = self.db.project(identifier)
-        if project is None:
-            abort(404)
+        project = Project.get_or_404(project_id)
 
-        # get labels
-        labels = project.label_tree()
+        labels = [lab.serialize() for lab in project.labels.all()]
 
-        # return labels
-        return jsonify(labels)
+        lookup = {}
+        for label in labels:
+            label["children"] = []
+            lookup[label["id"]] = label
+
+        tree = []
+        for label in labels:
+            parent_id = label["parent_id"]
+            if parent_id is None:
+                tree.append(label)
+            else:
+                lookup[parent_id]["children"].append(label)
+
+        return jsonify(tree)

pycs/frontend/endpoints/labels/ListLabels.py

@@ -1,7 +1,7 @@
-from flask import abort, jsonify
+from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 
 
 class ListLabels(View):
@@ -11,18 +11,10 @@ class ListLabels(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id):
         # find project
-        project = self.db.project(identifier)
-        if project is None:
-            abort(404)
-
-        # get labels
-        labels = project.labels()
+        project = Project.get_or_404(project_id)
 
         # return labels
-        return jsonify(labels)
+        return jsonify(project.labels.all())

pycs/frontend/endpoints/labels/RemoveLabel.py

@@ -1,7 +1,10 @@
-from flask import request, abort, make_response
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
+from pycs.database.Label import Label
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,41 +15,39 @@ class RemoveLabel(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
     def dispatch_request(self, project_id: int, label_id: int):
         # extract request data
         data = request.get_json(force=True)
 
-        if 'remove' not in data or data['remove'] is not True:
-            abort(400)
-
-        # find project
-        project = self.db.project(project_id)
-        if project is None:
-            abort(404)
+        if not data.get('remove', False):
+            abort(400, "remove flag is missing")
 
         # find label
-        label = project.label(label_id)
+        label = Label.query.filter(
+            Label.project_id == project_id,
+            Label.id == label_id).one_or_none()
+
         if label is None:
             abort(404)
 
-        # find children
-        children = label.children()
-
         # start transaction
-        with self.db:
-            # remove children's parent entry
+        with db.session.begin_nested():
+
+            children = label.children
+            # remove label. because of cascade, parent_id is set ot NULL
+            label_dump = label.delete(commit=False)
+            label.flush()
+
+            # notify about changes and reset the parent
             for child in children:
-                child.set_parent(None)
+                child.parent_id = label.parent_id
                 self.nm.edit_label(child)
 
-            # remove label
-            label.remove()
-            self.nm.remove_label(label)
+            self.nm.remove_label(label_dump)
 
         # return success response
         return make_response()

pycs/frontend/endpoints/pipelines/FitModel.py

@@ -1,7 +1,9 @@
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.interfaces.MediaStorage import MediaStorage
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
 from pycs.jobs.JobRunner import JobRunner
@@ -15,23 +17,20 @@ class FitModel(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, jobs: JobRunner, pipelines: PipelineCache):
+    def __init__(self, jobs: JobRunner, pipelines: PipelineCache):
         # pylint: disable=invalid-name
-        self.db = db
         self.jobs = jobs
         self.pipelines = pipelines
 
     def dispatch_request(self, project_id):
+        project = Project.get_or_404(project_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'fit' not in data or data['fit'] is not True:
-            return abort(400)
+        if not data.get('fit', False):
+            abort(400, "fit flag is missing")
 
-        # find project
-        project = self.db.project(project_id)
-        if project is None:
-            return abort(404)
 
         # create job
         try:
@@ -39,40 +38,36 @@ class FitModel(View):
                           'Model Interaction',
                           f'{project.name} (fit model with new data)',
                           f'{project.name}/model-interaction',
-                          self.load_and_fit, self.db, project.identifier)
+                          FitModel.load_and_fit,
+                          self.pipelines,
+                          project.id)
+
         except JobGroupBusyException:
-            return abort(400)
+            return abort(400, "Model fitting already running")
 
         return make_response()
 
     @staticmethod
-    def load_and_fit(database: Database, pipelines: PipelineCache, project_id: int):
+    def load_and_fit(pipelines: PipelineCache, project_id: int):
         """
         load the pipeline and call the fit function
 
-        :param database: database object
         :param pipelines: pipeline cache
         :param project_id: project id
         """
-        database_copy = None
         pipeline = None
 
         # create new database instance
-        try:
-            database_copy = database.copy()
-            project = database_copy.project(project_id)
-            model = project.model()
-            storage = MediaStorage(database_copy, project_id)
+        project = Project.query.get(project_id)
+        model = project.model
+        storage = MediaStorage(project_id)
 
-            # load pipeline
-            try:
-                pipeline = pipelines.load_from_root_folder(project, model.root_folder)
-                yield from pipeline.fit(storage)
-            except TypeError:
-                pass
-            finally:
-                if pipeline is not None:
-                    pipelines.free_instance(model.root_folder)
+        # load pipeline
+        try:
+            pipeline = pipelines.load_from_root_folder(project_id, model.root_folder)
+            yield from pipeline.fit(storage)
+        except TypeError:
+            pass
         finally:
-            if database_copy is not None:
-                database_copy.close()
+            if pipeline is not None:
+                pipelines.free_instance(model.root_folder)

pycs/frontend/endpoints/pipelines/PredictFile.py

@@ -1,7 +1,9 @@
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.File import File
 from pycs.frontend.endpoints.pipelines.PredictModel import PredictModel as Predict
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.frontend.notifications.NotificationManager import NotificationManager
@@ -17,28 +19,24 @@ class PredictFile(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self,
-                 db: Database, nm: NotificationManager, jobs: JobRunner, pipelines: PipelineCache):
+    def __init__(self, nm: NotificationManager, jobs: JobRunner, pipelines: PipelineCache):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
         self.jobs = jobs
         self.pipelines = pipelines
 
     def dispatch_request(self, file_id):
+        # find file
+        file = File.get_or_404(file_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'predict' not in data or data['predict'] is not True:
-            return abort(400)
-
-        # find file
-        file = self.db.file(file_id)
-        if file is None:
-            return abort(404)
+        if not data.get('predict', False):
+            abort(400, "predict flag is missing")
 
         # get project and model
-        project = file.project()
+        project = file.project
 
         # create job
         try:
@@ -47,11 +45,13 @@ class PredictFile(View):
             self.jobs.run(project,
                           'Model Interaction',
                           f'{project.name} (create predictions)',
-                          f'{project.name}/model-interaction',
+                          f'{project.id}/model-interaction',
                           Predict.load_and_predict,
-                          self.db, self.pipelines, notifications, project.identifier, [file],
+                          self.pipelines, notifications,
+                          project.id, [file.id],
                           progress=Predict.progress)
+
         except JobGroupBusyException:
-            return abort(400)
+            abort(400, "File prediction is already running")
 
         return make_response()

pycs/frontend/endpoints/pipelines/PredictModel.py

@@ -1,10 +1,13 @@
-from typing import Union, List
+from typing import List
+from typing import Union
 
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
-from pycs.database.File import File
+from pycs import db
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.interfaces.MediaFile import MediaFile
@@ -21,25 +24,25 @@ class PredictModel(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self,
-                 db: Database, nm: NotificationManager, jobs: JobRunner, pipelines: PipelineCache):
+    def __init__(self, nm: NotificationManager, jobs: JobRunner, pipelines: PipelineCache):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
         self.jobs = jobs
         self.pipelines = pipelines
 
     def dispatch_request(self, project_id):
+        project = Project.get_or_404(project_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'predict' not in data or data['predict'] not in ['all', 'new']:
-            return abort(400)
+        predict = data.get('predict')
+
+        if predict is None:
+            abort(400, "predict argument is missing")
 
-        # find project
-        project = self.db.project(project_id)
-        if project is None:
-            return abort(404)
+        if predict not in ['all', 'new']:
+            abort(400, "predict must be either 'all' or 'new'")
 
         # create job
         try:
@@ -48,20 +51,21 @@ class PredictModel(View):
             self.jobs.run(project,
                           'Model Interaction',
                           f'{project.name} (create predictions)',
-                          f'{project.name}/model-interaction',
-                          self.load_and_predict,
-                          self.db, self.pipelines, notifications,
-                          project.identifier, data['predict'],
+                          f'{project.id}/model-interaction',
+                          PredictModel.load_and_predict,
+                          self.pipelines, notifications,
+                          project.id, predict,
                           progress=self.progress)
+
         except JobGroupBusyException:
-            return abort(400)
+            abort(400, "Model prediction is already running")
 
         return make_response()
 
     @staticmethod
-    def load_and_predict(database: Database, pipelines: PipelineCache,
+    def load_and_predict(pipelines: PipelineCache,
                          notifications: NotificationList,
-                         project_id: int, file_filter: Union[str, List[File]]):
+                         project_id: int, file_filter: Union[str, List[int]]):
         """
         load the pipeline and call the execute function
 
@@ -69,58 +73,55 @@ class PredictModel(View):
         :param pipelines: pipeline cache
         :param notifications: notification object
         :param project_id: project id
-        :param file_filter: list of files or 'new' / 'all'
+        :param file_filter: list of file ids or 'new' / 'all'
         :return:
         """
-        database_copy = None
         pipeline = None
 
         # create new database instance
-        try:
-            database_copy = database.copy()
-            project = database_copy.project(project_id)
-            model = project.model()
-            storage = MediaStorage(database_copy, project_id, notifications)
-
-            # create a list of MediaFile
-            if isinstance(file_filter, str):
-                if file_filter == 'new':
-                    length = project.count_files_without_results()
-                    files = map(lambda f: MediaFile(f, notifications),
-                                project.files_without_results())
-                else:
-                    length = project.count_files()
-                    files = map(lambda f: MediaFile(f, notifications),
-                                project.files())
+        project = Project.query.get(project_id)
+        model_root = project.model.root_folder
+        storage = MediaStorage(project_id, notifications)
+
+        # create a list of MediaFile
+        if isinstance(file_filter, str):
+            if file_filter == 'new':
+                files = project.files_without_results()
+                length = project.count_files_without_results()
+
             else:
-                files = map(lambda f: MediaFile(project.file(f.identifier), notifications),
-                            file_filter)
-                length = len(file_filter)
-
-            # load pipeline
-            try:
-                pipeline = pipelines.load_from_root_folder(project, model.root_folder)
-
-                # iterate over files
-                index = 0
-                for file in files:
-                    # remove old predictions
-                    file.remove_predictions()
-
-                    # create new predictions
-                    pipeline.execute(storage, file)
-
-                    # commit changes and yield progress
-                    database_copy.commit()
-                    yield index / length, notifications
-
-                    index += 1
-            finally:
-                if pipeline is not None:
-                    pipelines.free_instance(model.root_folder)
+                files = project.files.all()
+                length = project.files.count()
+
+        else:
+            files = [project.file(identifier) for identifier in file_filter]
+            length = len(files)
+
+
+        media_files = map(lambda f: MediaFile(f, notifications), files)
+        # load pipeline
+        try:
+            pipeline = pipelines.load_from_root_folder(project_id, model_root)
+
+            # iterate over media files
+            index = 0
+            for file in media_files:
+                # remove old predictions
+                file.remove_predictions()
+
+                # create new predictions
+                pipeline.execute(storage, file)
+
+                # commit changes and yield progress
+                db.session.commit()
+                yield index / length, notifications
+
+                index += 1
+
         finally:
-            if database_copy is not None:
-                database_copy.close()
+            if pipeline is not None:
+                pipelines.free_instance(model_root)
+
 
     @staticmethod
     def progress(progress: float, notifications: NotificationList):

pycs/frontend/endpoints/projects/CreateProject.py

@@ -1,13 +1,19 @@
+import shutil
+import uuid
+
 from contextlib import closing
-from os import mkdir
-from os import path
-from shutil import copytree
-from uuid import uuid1
+from pathlib import Path
 
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
+from pycs import settings
+from pycs.database.LabelProvider import LabelProvider
+from pycs.database.Model import Model
+from pycs.database.Project import Project
 from pycs.frontend.endpoints.projects.ExecuteExternalStorage import ExecuteExternalStorage
 from pycs.frontend.endpoints.projects.ExecuteLabelProvider import ExecuteLabelProvider
 from pycs.frontend.notifications.NotificationManager import NotificationManager
@@ -22,9 +28,8 @@ class CreateProject(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager, jobs: JobRunner):
+    def __init__(self, nm: NotificationManager, jobs: JobRunner):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
         self.jobs = jobs
 
@@ -32,65 +37,71 @@ class CreateProject(View):
         # extract request data
         data = request.get_json(force=True)
 
-        if 'name' not in data or 'description' not in data:
-            return abort(400)
+        name = data.get('name')
+        description = data.get('description')
+
+        if name is None:
+            abort(400, "name argument is missing!")
 
-        name = data['name']
-        description = data['description']
+        if description is None:
+            abort(400, "description argument is missing!")
 
-        # start transaction
-        with self.db:
-            # find model
-            model_id = int(data['model'])
-            model = self.db.model(model_id)
 
-            if model is None:
-                return abort(404)
+        model_id = int(data['model'])
+        model = Model.get_or_404(model_id)
 
-            # find label provider
-            if data['label'] is None:
-                label_provider = None
-            else:
-                label_provider_id = int(data['label'])
-                label_provider = self.db.label_provider(label_provider_id)
+        label_provider_id = data.get('label')
+        label_provider = None
 
-                if label_provider is None:
-                    return abort(404)
+        if label_provider_id is not None:
+            label_provider = LabelProvider.get_or_404(label_provider_id)
 
-            # create project folder
-            project_folder = path.join('projects', str(uuid1()))
-            mkdir(project_folder)
+        # create project folder
+        project_folder = Path(settings.projects_folder, str(uuid.uuid1()))
+        project_folder.mkdir()
 
-            temp_folder = path.join(project_folder, 'temp')
-            mkdir(temp_folder)
+        temp_folder = project_folder / 'temp'
+        temp_folder.mkdir()
 
-            # check project data directory
-            if data['external'] is None:
-                external_data = False
-                data_folder = path.join(project_folder, 'data')
+        # check project data directory
+        if data['external'] is None:
+            external_data = False
+            data_folder = project_folder / 'data'
+            data_folder.mkdir()
 
-                mkdir(data_folder)
-            else:
-                external_data = True
-                data_folder = data['external']
+        else:
+            external_data = True
+            data_folder = Path(data['external'])
 
-                # check if exists
-                if not path.exists(data_folder):
-                    return abort(400)
+            # check if exists
+            if not data_folder.exists():
+                return abort(400, f"External folder does not exist: {data_folder}")
 
-            # copy model to project folder
-            model_folder = path.join(project_folder, 'model')
-            copytree(model.root_folder, model_folder)
+        # copy model to project folder
+        model_folder = project_folder / 'model'
+        shutil.copytree(model.root_folder, str(model_folder))
 
-            model, _ = model.copy_to(f'{model.name} ({name})', model_folder)
+        with db.session.begin_nested():
+            model, is_new = model.copy_to(
+                name=f'{model.name} ({name})',
+                root_folder=str(model_folder),
+                commit=False)
+            model.flush()
 
-            # create entry in database
-            created = self.db.create_project(name, description, model, label_provider,
-                                             project_folder, external_data, data_folder)
+            if not is_new:
+                abort(400, # pragma: no cover
+                    f"Could not copy model! Model in \"{model_folder}\" already exists!")
+            project = Project.new(name=name,
+                                  description=description,
+                                  model_id=model.id,
+                                  label_provider_id=label_provider_id,
+                                  root_folder=str(project_folder),
+                                  external_data=external_data,
+                                  data_folder=str(data_folder))
 
         # execute label provider and add labels to project
         if label_provider is not None:
-            ExecuteLabelProvider.execute_label_provider(self.db, self.nm, self.jobs, created,
+            ExecuteLabelProvider.execute_label_provider(self.nm, self.jobs, project,
                                                         label_provider)
 
         # load model and add collections to the project
@@ -99,28 +110,26 @@ class CreateProject(View):
                 return pipeline.collections()
 
         def add_collections_to_project(provided_collections):
-            with self.db:
-                for position, collection in enumerate(provided_collections):
-                    created.create_collection(collection['reference'],
-                                              collection['name'],
-                                              collection['description'],
-                                              position + 1,
-                                              collection['autoselect'])
-
-        self.jobs.run(created,
+            with db.session.begin_nested():
+                for position, collection in enumerate(provided_collections, 1):
+                    project.create_collection(commit=False,
+                                              position=position,
+                                              **collection)
+
+        self.jobs.run(project,
                       'Media Collections',
-                      f'{created.name}',
-                      f'{created.identifier}/media-collections',
+                      f'{project.name}',
+                      f'{project.id}/media-collections',
                       executable=load_model_and_get_collections,
                       result=add_collections_to_project)
 
         # find media files
         if external_data:
-            ExecuteExternalStorage.find_media_files(self.db, self.nm, self.jobs, created)
+            ExecuteExternalStorage.find_media_files(self.nm, self.jobs, project)
 
         # fire event
         self.nm.create_model(model)
-        self.nm.create_project(created)
+        self.nm.create_project(project)
 
         # return success response
         return make_response()

pycs/frontend/endpoints/projects/EditProjectDescription.py

@@ -1,7 +1,9 @@
-from flask import make_response, abort
-from flask.views import View, request
+from flask import abort
+from flask import make_response
+from flask.views import View
+from flask.views import request
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,27 +14,24 @@ class EditProjectDescription(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
-    def dispatch_request(self, identifier):
+
+    def dispatch_request(self, project_id: int):
         # extract request data
         data = request.get_json(force=True)
+        description = data.get('description')
+
+        if description is None:
+            abort(400, 'description argument is missing')
 
-        if 'description' not in data or not data['description']:
-            return abort(400)
+        project = Project.get_or_404(project_id)
 
-        # start transaction
-        with self.db:
-            # find project
-            project = self.db.project(identifier)
-            if project is None:
-                return abort(404)
+        project.description = description
+        project.commit()
 
-            # set description
-            project.set_description(data['description'])
-            self.nm.edit_project(project)
+        self.nm.edit_project(project)
 
         return make_response()

pycs/frontend/endpoints/projects/EditProjectName.py

@@ -1,7 +1,9 @@
-from flask import make_response, abort
-from flask.views import View, request
+from flask import abort
+from flask import make_response
+from flask.views import View
+from flask.views import request
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,27 +14,24 @@ class EditProjectName(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
-    def dispatch_request(self, identifier):
+
+    def dispatch_request(self, project_id: int):
         # extract request data
         data = request.get_json(force=True)
+        name = data.get('name')
+
+        if name is None:
+            abort(400, 'name argument is missing')
 
-        if 'name' not in data or not data['name']:
-            return abort(400)
+        project = Project.get_or_404(project_id)
 
-        # start transaction
-        with self.db:
-            # find project
-            project = self.db.project(identifier)
-            if project is None:
-                return abort(404)
+        project.name = name
+        project.commit()
 
-            # set name
-            project.set_name(data['name'])
-            self.nm.edit_project(project)
+        self.nm.edit_project(project)
 
-            return make_response()
+        return make_response()

pycs/frontend/endpoints/projects/ExecuteExternalStorage.py

@@ -1,12 +1,12 @@
-from os import listdir
-from os import path
-from os.path import isfile
-from uuid import uuid1
+import os
+import uuid
 
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
 from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
@@ -21,42 +21,38 @@ class ExecuteExternalStorage(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager, jobs: JobRunner):
+    def __init__(self, nm: NotificationManager, jobs: JobRunner):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
         self.jobs = jobs
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id: int):
         # extract request data
         data = request.get_json(force=True)
 
-        if 'execute' not in data or data['execute'] is not True:
+        if not data.get('execute', False):
             return abort(400)
 
         # find project
-        project = self.db.project(identifier)
-        if project is None:
-            return abort(404)
+        project = Project.get_or_404(project_id)
 
         if not project.external_data:
-            return abort(400)
+            return abort(400, "External data is not set!")
 
         # execute label provider and add labels to project
         try:
-            self.find_media_files(self.db, self.nm, self.jobs, project)
+            ExecuteExternalStorage.find_media_files(self.nm, self.jobs, project)
         except JobGroupBusyException:
-            return abort(400)
+            return abort(400, "Job is already running!")
 
         return make_response()
 
     @staticmethod
-    def find_media_files(db: Database, nm: NotificationManager, jobs: JobRunner, project: Project):
+    def find_media_files(nm: NotificationManager, jobs: JobRunner, project: Project):
         """
         start a job that finds media files in the projects data_folder and adds them to the
         database afterwards
 
-        :param db: database object
         :param nm: notification manager object
         :param jobs: job runner object
         :param project: project
@@ -65,27 +61,36 @@ class ExecuteExternalStorage(View):
 
         # pylint: disable=invalid-name
         # find lists the given data folder and prepares item dictionaries
-        def find():
-            files = listdir(project.data_folder)
+        def find(data_folder):
+            files = os.listdir(data_folder)
             length = len(files)
 
             elements = []
             current = 0
 
             for file_name in files:
-                file_path = path.join(project.data_folder, file_name)
-                if not isfile(file_path):
+                file_path = os.path.join(data_folder, file_name)
+                if not os.path.isfile(file_path):
                     continue
 
-                file_name, file_extension = path.splitext(file_name)
-                file_size = path.getsize(file_path)
+                file_name, file_extension = os.path.splitext(file_name)
+                file_size = os.path.getsize(file_path)
 
                 try:
-                    ftype, frames, fps = file_info(project.data_folder, file_name, file_extension)
+                    ftype, frames, fps = file_info(data_folder, file_name, file_extension)
                 except ValueError:
                     continue
 
-                elements.append((ftype, file_name, file_extension, file_size, frames, fps))
+                file_attrs = dict(
+                    uuid=str(uuid.uuid1()),
+                    file_type=ftype,
+                    name=file_name,
+                    extension=file_extension,
+                    size=file_size,
+                    frames=frames,
+                    fps=fps)
+
+                elements.append(file_attrs)
                 current += 1
 
                 if len(elements) >= 200:
@@ -97,13 +102,11 @@ class ExecuteExternalStorage(View):
 
         # progress inserts elements into the database and fires events
         def progress(elements, current, length):
-            with db:
-                for ftype, file_name, file_extension, file_size, frames, fps in elements:
-                    uuid = str(uuid1())
-                    file, insert = project.add_file(uuid, ftype, file_name, file_extension,
-                                                    file_size, file_name, frames, fps)
+            with db.session.begin_nested():
+                for file_attrs in elements:
+                    file, is_new = project.add_file(commit=False, **file_attrs)
 
-                    if insert:
+                    if is_new:
                         nm.create_file(file)
 
             return current / length
@@ -112,6 +115,6 @@ class ExecuteExternalStorage(View):
         jobs.run(project,
                  'Find Media Files',
                  project.name,
-                 f'{project.identifier}/find-files',
-                 find,
+                 f'{project.id}/find-files',
+                 find, project.data_folder,
                  progress=progress)

pycs/frontend/endpoints/projects/ExecuteLabelProvider.py

@@ -1,9 +1,11 @@
 from contextlib import closing
 
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
 from pycs.database.LabelProvider import LabelProvider
 from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
@@ -18,45 +20,40 @@ class ExecuteLabelProvider(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager, jobs: JobRunner):
+    def __init__(self, nm: NotificationManager, jobs: JobRunner):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
         self.jobs = jobs
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id: int):
+        project = Project.get_or_404(project_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'execute' not in data or data['execute'] is not True:
-            return abort(400)
-
-        # find project
-        project = self.db.project(identifier)
-        if project is None:
-            return abort(404)
+        if not data.get('execute', False):
+            abort(400, "execute flag is missing")
 
         # get label provider
-        label_provider = project.label_provider()
+        label_provider = project.label_provider
         if label_provider is None:
-            return abort(400)
+            abort(400, "This project does not have a label provider.")
 
         # execute label provider and add labels to project
         try:
-            self.execute_label_provider(self.db, self.nm, self.jobs, project, label_provider)
+            self.execute_label_provider(self.nm, self.jobs, project, label_provider)
         except JobGroupBusyException:
-            return abort(400)
+            abort(400, "Label provider already running.")
 
         return make_response()
 
     @staticmethod
-    def execute_label_provider(db: Database, nm: NotificationManager, jobs: JobRunner,
+    def execute_label_provider(nm: NotificationManager, jobs: JobRunner,
                                project: Project, label_provider: LabelProvider):
         """
         start a job that loads and executes a label provider and saves its results to the
         database afterwards
 
-        :param db: database object
         :param nm: notification manager object
         :param jobs: job runner object
         :param project: project
@@ -68,26 +65,30 @@ class ExecuteLabelProvider(View):
         # receive loads and executes the given label provider
         def receive():
             with closing(label_provider.load()) as label_provider_impl:
-                provided_labels = label_provider_impl.get_labels()
-                return provided_labels
+                return label_provider_impl.get_labels()
 
+        project_id = project.id
         # result adds the received labels to the database and fires events
         def result(provided_labels):
-            with db:
-                for label in provided_labels:
-                    created_label, insert = project.create_label(
-                        label['name'], label['reference'], label['parent'], label['hierarchy_level']
-                    )
+            nonlocal project_id
+
+            def _inner(labels, project_id):
+                project = Project.query.get(project_id)
+                labels = project.bulk_create_labels(labels)
+
+                for label in labels:
+                    nm.create_label(label)
 
-                    if insert:
-                        nm.create_label(created_label)
-                    else:
-                        nm.edit_label(created_label)
+            if db.session().in_transaction():
+                _inner(provided_labels, project_id)
+            else:
+                with db.session.begin():
+                    _inner(provided_labels, project_id)
 
         # run job with given functions
         jobs.run(project,
                  'Label Provider',
                  f'{project.name} ({label_provider.name})',
-                 f'{project.identifier}/label-provider',
+                 f'{project.id}/label-provider',
                  receive,
                  result=result)

pycs/frontend/endpoints/projects/GetProjectModel.py

@@ -1,7 +1,7 @@
-from flask import abort, jsonify
+from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 
 
 class GetProjectModel(View):
@@ -11,18 +11,10 @@ class GetProjectModel(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id: int):
         # find project
-        project = self.db.project(identifier)
-        if project is None:
-            abort(404)
-
-        # get model
-        model = project.model()
+        project = Project.get_or_404(project_id)
 
         # return model
-        return jsonify(model)
+        return jsonify(project.model)

pycs/frontend/endpoints/projects/ListCollections.py

@@ -1,40 +0,0 @@
-from flask import abort, jsonify
-from flask.views import View
-
-from pycs.database.Database import Database
-
-
-class ListCollections(View):
-    """
-    return a list of collections for a given project
-    """
-    # pylint: disable=arguments-differ
-    methods = ['GET']
-
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
-
-    def dispatch_request(self, project_id: int):
-        # find project
-        project = self.db.project(project_id)
-
-        if project is None:
-            return abort(404)
-
-        # get collection list
-        collections = project.collections()
-
-        # disable autoselect if there are no elements in the collection
-        found = False
-
-        for collection in collections:
-            if collection.autoselect:
-                if found:
-                    collection.autoselect = False
-                elif collection.count_files() == 0:
-                    collection.autoselect = False
-                    found = True
-
-        # return files
-        return jsonify(collections)

pycs/frontend/endpoints/projects/ListFiles.py

@@ -1,44 +0,0 @@
-from flask import abort, jsonify
-from flask.views import View
-
-from pycs.database.Database import Database
-
-
-class ListFiles(View):
-    """
-    return a list of files for a given project
-    """
-    # pylint: disable=arguments-differ
-    methods = ['GET']
-
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
-
-    def dispatch_request(self, project_id: int, start: int, length: int, collection_id: int = None):
-        # find project
-        project = self.db.project(project_id)
-        if project is None:
-            return abort(404)
-
-        # get count and files
-        if collection_id is not None:
-            if collection_id == 0:
-                count = project.count_files_without_collection()
-                files = list(project.files_without_collection(start, length))
-            else:
-                collection = project.collection(collection_id)
-                if collection is None:
-                    return abort(404)
-
-                count = collection.count_files()
-                files = list(collection.files(start, length))
-        else:
-            count = project.count_files()
-            files = list(project.files(start, length))
-
-        # return files
-        return jsonify({
-            'count': count,
-            'files': files
-        })

pycs/frontend/endpoints/projects/ListProjectCollections.py

@@ -0,0 +1,24 @@
+from flask import jsonify
+from flask.views import View
+
+from pycs.database.Collection import Collection
+from pycs.database.Project import Project
+
+
+class ListProjectCollections(View):
+    """
+    return a list of collections for a given project
+    """
+    # pylint: disable=arguments-differ
+    methods = ['GET']
+
+
+    def dispatch_request(self, project_id: int):
+        # find project
+        project = Project.get_or_404(project_id)
+
+        # get collection list
+        collections = Collection.update_autoselect(project.collections)
+
+        # return files
+        return jsonify(collections.all())

pycs/frontend/endpoints/projects/ListProjectFiles.py

@@ -0,0 +1,47 @@
+from flask import abort
+from flask import jsonify
+from flask.views import View
+
+from pycs.database.Project import Project
+
+
+class ListProjectFiles(View):
+    """
+    return a list of files for a given project
+    """
+    # pylint: disable=arguments-differ
+    methods = ['GET']
+
+
+    def dispatch_request(self,
+                         project_id: int,
+                         start: int = 0,
+                         length: int = -1,
+                         collection_id: int = None):
+        # find project
+
+        project = Project.get_or_404(project_id)
+
+        # get count and files
+        if collection_id is not None:
+            if collection_id == 0:
+                count = project.count_files_without_collection()
+                files = project.files_without_collection(offset=start, limit=length)
+
+            else:
+                collection = project.collection(collection_id)
+                if collection is None:
+                    abort(404)
+
+                count = collection.files.count()
+                files = collection.get_files(offset=start, limit=length).all()
+
+        else:
+            count = project.files.count()
+            files = project.get_files(offset=start, limit=length).all()
+
+        # return files
+        return jsonify({
+            'count': count,
+            'files': files
+        })

pycs/frontend/endpoints/projects/RemoveProject.py

@@ -1,9 +1,9 @@
-from shutil import rmtree
-
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -14,37 +14,25 @@ class RemoveProject(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self,nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
-    def dispatch_request(self, identifier):
+    def dispatch_request(self, project_id: int):
         # extract request data
         data = request.get_json(force=True)
 
-        if 'remove' not in data or data['remove'] is not True:
-            abort(400)
-
-        # start transaction
-        with self.db:
-            # find project
-            project = self.db.project(identifier)
-            if project is None:
-                abort(404)
-
-            # remove from database
-            project.remove()
+        if not data.get('remove', False):
+            abort(400, "remove flag is missing")
 
-            # remove model from database
-            model = project.model()
-            model.remove()
+        # find project
+        project = Project.get_or_404(project_id)
 
-            # remove from file system
-            rmtree(project.root_folder)
+        # remove from database
+        project_dump, model_dump = project.delete()
 
-            # send update
-            self.nm.remove_model(model)
-            self.nm.remove_project(project)
+        # send update
+        self.nm.remove_model(model_dump)
+        self.nm.remove_project(project_dump)
 
-            return make_response()
+        return make_response()

pycs/frontend/endpoints/results/ConfirmResult.py

@@ -1,7 +1,7 @@
 from flask import make_response, request, abort
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Result import Result
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,26 +12,23 @@ class ConfirmResult(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
     def dispatch_request(self, result_id: int):
+        # find result
+        result = Result.get_or_404(result_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'confirm' not in data or data['confirm'] is not True:
-            return abort(400)
 
-        # find result
-        result = self.db.result(result_id)
-        if result is None:
-            return abort(404)
+        if not data.get('confirm', False):
+            return abort(400, "confirm flag is missing")
+
 
-        # start transaction
-        with self.db:
-            result.set_origin('user')
+        result.set_origin('user')
 
         self.nm.edit_result(result)
         return make_response()

pycs/frontend/endpoints/results/CreateResult.py

@@ -1,7 +1,10 @@
-from flask import request, abort, jsonify
+from flask import abort
+from flask import jsonify
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs import db
+from pycs.database.File import File
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,51 +15,63 @@ class CreateResult(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
-    def dispatch_request(self, file_id: int):
-        # extract request data
+    @staticmethod
+    def extract_request_data():
+        """ get the data from the request and validate it """
         request_data = request.get_json(force=True)
 
         if 'type' not in request_data:
-            return abort(400)
-        if request_data['type'] not in ['labeled-image', 'bounding-box']:
-            return abort(400)
-
-        rtype = request_data['type']
-
-        if 'label' in request_data and request_data['label']:
-            label = request_data['label']
-        elif request_data['type'] == 'labeled-image':
-            return abort(400)
-        else:
-            label = None
-
-        if 'data' in request_data and request_data['data']:
-            data = request_data['data']
-        elif request_data['type'] == 'bounding-box':
-            return abort(400)
-        else:
-            data = {}
-
-        # find file
-        file = self.db.file(file_id)
-        if file is None:
-            return abort(404)
+            abort(400, "result type argument is missing")
+
+        result_type = request_data.get('type')
+        if result_type not in ['labeled-image', 'bounding-box']:
+            abort(400, "result type must be either 'labeled-image' or 'bounding-box'")
+
+        label = None
+        data = None
+
+        if result_type == 'labeled-image':
+            label = request_data.get('label')
+            if label is None:
+                abort(400, f"Could not find label argument ({result_type=})")
+
+        elif result_type == 'bounding-box':
+            data = request_data.get('data')
+            if data is None:
+                abort(400, f"Could not find data argument ({result_type=})")
+
+        return result_type, label, data
+
+
+    def dispatch_request(self, file_id: int):
+
+        file = File.get_or_404(file_id)
+
+        result_type, label, data = self.extract_request_data()
+
 
         # start transaction
-        with self.db:
+        with db.session.begin_nested():
             # find full-image labels and remove them
-            for result in file.results():
-                if result.type == 'labeled-image':
-                    result.remove()
-                    self.nm.remove_result(result)
+            image_results = file.results.filter_by(type='labeled-image')
+
+            removed = [result.serialize() for result in image_results.all()]
+            image_results.delete()
+
+            for result in removed:
+                self.nm.remove_result(result)
 
             # insert into database
-            result = file.create_result('user', rtype, label, data)
+            result = file.create_result(
+                origin='user',
+                result_type=result_type,
+                label=label,
+                data=data)
+
             self.nm.create_result(result)
 
         return jsonify(result)

pycs/frontend/endpoints/results/EditResultData.py

@@ -1,7 +1,9 @@
-from flask import make_response, abort
-from flask.views import View, request
+from flask import abort
+from flask import make_response
+from flask.views import View
+from flask.views import request
 
-from pycs.database.Database import Database
+from pycs.database.Result import Result
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,27 +14,23 @@ class EditResultData(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
     def dispatch_request(self, result_id: int):
+        # find result
+        result = Result.get_or_404(result_id)
+
         # extract request data
-        data = request.get_json(force=True)
+        request_data = request.get_json(force=True)
+        data = request_data.get('data')
 
-        if 'data' not in data:
-            return abort(400)
+        if data is None:
+            abort(400, "Could not find data argument!")
 
-        # find result
-        result = self.db.result(result_id)
-        if result is None:
-            return abort(404)
-
-        # start transaction and set label
-        with self.db:
-            result.set_data(data['data'])
-            result.set_origin('user')
+        result.data = data
+        result.set_origin('user', commit=True)
 
         self.nm.edit_result(result)
         return make_response()

pycs/frontend/endpoints/results/EditResultLabel.py

@@ -1,7 +1,9 @@
-from flask import make_response, abort
-from flask.views import View, request
+from flask import abort
+from flask import make_response
+from flask.views import View
+from flask.views import request
 
-from pycs.database.Database import Database
+from pycs.database.Result import Result
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,31 +14,28 @@ class EditResultLabel(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
     def dispatch_request(self, result_id: int):
+        # find result
+        result = Result.get_or_404(result_id)
+
         # extract request data
-        data = request.get_json(force=True)
+        request_data = request.get_json(force=True)
 
-        if 'label' not in data:
-            return abort(400)
+        if 'label' not in request_data:
+            abort(400, "Could not find label argument!")
 
-        # find result
-        result = self.db.result(result_id)
-        if result is None:
-            return abort(404)
+        label = request_data.get('label')
 
         # abort if label is empty for labeled-images
-        if result.type == 'labeled-image' and not data['label']:
-            return abort(400)
+        if result.type == 'labeled-image' and label is None:
+            abort(400, "Label is required for 'labeled-images' results")
 
-        # start transaction and set label
-        with self.db:
-            result.set_label(data['label'])
-            result.set_origin('user')
+        result.label_id = label
+        result.set_origin('user', commit=True)
 
         self.nm.edit_result(result)
         return make_response()

pycs/frontend/endpoints/results/GetProjectResults.py

@@ -1,7 +1,7 @@
-from flask import abort, jsonify
+from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.interfaces.MediaStorage import MediaStorage
 
 
@@ -12,18 +12,13 @@ class GetProjectResults(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self, project_id: int):
         # get project from database
-        project = self.db.project(project_id)
-        if project is None:
-            return abort(404)
+        project = Project.get_or_404(project_id)
 
         # map media files to a dict
-        storage = MediaStorage(self.db, project.identifier, None)
+        storage = MediaStorage(project.id, None)
         files = list(map(lambda f: f.serialize(), storage.files().iter()))
 
         # return result

pycs/frontend/endpoints/results/GetResults.py

@@ -1,7 +1,7 @@
-from flask import abort, jsonify
+from flask import jsonify
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.File import File
 
 
 class GetResults(View):
@@ -11,18 +11,10 @@ class GetResults(View):
     # pylint: disable=arguments-differ
     methods = ['GET']
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
 
     def dispatch_request(self, file_id: int):
         # get file from database
-        file = self.db.file(file_id)
-        if file is None:
-            return abort(404)
-
-        # get results
-        results = file.results()
+        file = File.get_or_404(file_id)
 
         # return result
-        return jsonify(results)
+        return jsonify(file.results.all())

pycs/frontend/endpoints/results/RemoveResult.py

@@ -1,7 +1,9 @@
-from flask import make_response, request, abort
+from flask import abort
+from flask import make_response
+from flask import request
 from flask.views import View
 
-from pycs.database.Database import Database
+from pycs.database.Result import Result
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,26 +14,21 @@ class RemoveResult(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
     def dispatch_request(self, result_id: int):
+        result = Result.get_or_404(result_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'remove' not in data or data['remove'] is not True:
-            abort(400)
+        if not data.get('remove', False):
+            abort(400, "remove flag is missing")
 
-        # find result
-        result = self.db.result(result_id)
-        if result is None:
-            return abort(404)
 
-        # start transaction
-        with self.db:
-            result.remove()
+        dump = result.delete()
 
-        self.nm.remove_result(result)
+        self.nm.remove_result(dump)
         return make_response()

pycs/frontend/endpoints/results/ResetResults.py

@@ -1,7 +1,9 @@
-from flask import make_response, abort
-from flask.views import View, request
+from flask import abort
+from flask import make_response
+from flask.views import View
+from flask.views import request
 
-from pycs.database.Database import Database
+from pycs.database.File import File
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
 
@@ -12,32 +14,28 @@ class ResetResults(View):
     # pylint: disable=arguments-differ
     methods = ['POST']
 
-    def __init__(self, db: Database, nm: NotificationManager):
+    def __init__(self, nm: NotificationManager):
         # pylint: disable=invalid-name
-        self.db = db
         self.nm = nm
 
     def dispatch_request(self, file_id: int):
+        file = File.get_or_404(file_id)
+
         # extract request data
         data = request.get_json(force=True)
 
-        if 'reset' not in data or data['reset'] is not True:
+        if not data.get('reset', False):
             abort(400)
 
-        # find file
-        file = self.db.file(file_id)
-        if file is None:
-            return abort(404)
 
-        # get results
-        results = file.results()
+        removed = []
+
+        for result in file.results.all():
+            removed.append(result.serialize())
 
-        # start transaction
-        with self.db:
-            for result in results:
-                result.remove()
+        file.results.delete()
 
-        for result in results:
+        for result in removed:
             self.nm.remove_result(result)
 
         return make_response()

pycs/frontend/notifications/NotificationManager.py

@@ -1,5 +1,6 @@
 from socketio import Server
 
+from pycs import app
 from pycs.database.File import File
 from pycs.database.Label import Label
 from pycs.database.Model import Model
@@ -29,7 +30,7 @@ class NotificationManager:
         :param created_job:
         :return:
         """
-        print('create_job', created_job)
+        app.logger.debug(created_job)
         self.__emit('create-job', created_job)
 
     def edit_job(self, edited_job: Job):
@@ -39,7 +40,7 @@ class NotificationManager:
         :param edited_job:
         :return:
         """
-        print('edit_job', edited_job)
+        app.logger.debug(edited_job)
         self.__emit('edit-job', edited_job)
 
     def remove_job(self, removed_job: Job):
@@ -49,7 +50,7 @@ class NotificationManager:
         :param removed_job:
         :return:
         """
-        print('remove_job', removed_job)
+        app.logger.debug(removed_job)
         self.__emit('remove-job', removed_job)
 
     def create_model(self, created_model: Model):
@@ -59,7 +60,7 @@ class NotificationManager:
         :param created_model:
         :return:
         """
-        print('create_model', created_model)
+        app.logger.debug(created_model)
         self.__emit('create-model', created_model)
 
     def remove_model(self, removed_model: Model):
@@ -69,7 +70,7 @@ class NotificationManager:
         :param removed_model:
         :return:
         """
-        print('remove_model', removed_model)
+        app.logger.debug(removed_model)
         self.__emit('remove-model', removed_model)
 
     def create_project(self, created_project: Project):
@@ -79,7 +80,7 @@ class NotificationManager:
         :param created_project:
         :return:
         """
-        print('create_project', created_project)
+        app.logger.debug(created_project)
         self.__emit('create-project', created_project)
 
     def remove_project(self, removed_project: Project):
@@ -89,7 +90,7 @@ class NotificationManager:
         :param removed_project:
         :return:
         """
-        print('remove_project', removed_project)
+        app.logger.debug(removed_project)
         self.__emit('remove-project', removed_project)
 
     def edit_project(self, edited_project: Project):
@@ -99,7 +100,7 @@ class NotificationManager:
         :param edited_project:
         :return:
         """
-        print('edit_project', edited_project)
+        app.logger.debug(edited_project)
         self.__emit('edit-project', edited_project)
 
     def create_label(self, created_label: Label):
@@ -109,7 +110,7 @@ class NotificationManager:
         :param created_label:
         :return:
         """
-        print('create_label', created_label)
+        app.logger.debug(created_label)
         self.__emit('create-label', created_label)
 
     def edit_label(self, edited_label: Label):
@@ -119,7 +120,7 @@ class NotificationManager:
         :param edited_label:
         :return:
         """
-        print('edit_label', edited_label)
+        app.logger.debug(edited_label)
         self.__emit('edit-label', edited_label)
 
     def remove_label(self, removed_label: Label):
@@ -129,7 +130,7 @@ class NotificationManager:
         :param removed_label:
         :return:
         """
-        print('remove_label', removed_label)
+        app.logger.debug(removed_label)
         self.__emit('remove-label', removed_label)
 
     def create_file(self, created_file: File):
@@ -139,7 +140,7 @@ class NotificationManager:
         :param created_file:
         :return:
         """
-        print('create_file', created_file)
+        app.logger.debug(created_file)
         self.__emit('create-file', created_file)
 
     def edit_file(self, edited_file: File):
@@ -149,7 +150,7 @@ class NotificationManager:
         :param edited_file:
         :return:
         """
-        print('edit_file', edited_file)
+        app.logger.debug(edited_file)
         self.__emit('edit-file', edited_file)
 
     def remove_file(self, removed_file: File):
@@ -159,7 +160,7 @@ class NotificationManager:
         :param removed_file:
         :return:
         """
-        print('remove_file', removed_file)
+        app.logger.debug(removed_file)
         self.__emit('remove-file', removed_file)
 
     def create_result(self, created_result: Result):
@@ -169,7 +170,7 @@ class NotificationManager:
         :param created_result:
         :return:
         """
-        print('create_result', created_result)
+        app.logger.debug(created_result)
         self.__emit('create-result', created_result)
 
     def edit_result(self, edited_result: Result):
@@ -179,7 +180,7 @@ class NotificationManager:
         :param edited_result:
         :return:
         """
-        print('edit_result', edited_result)
+        app.logger.debug(edited_result)
         self.__emit('edit-result', edited_result)
 
     def remove_result(self, removed_result: Result):
@@ -189,5 +190,5 @@ class NotificationManager:
         :param removed_result:
         :return:
         """
-        print('remove_result', removed_result)
+        app.logger.debug(removed_result)
         self.__emit('remove-result', removed_result)

pycs/frontend/util/JSONEncoder.py

@@ -1,22 +1,31 @@
+import datetime
+
 from typing import Any
 
 from flask.json import JSONEncoder as Base
 
-from pycs.database.util.JSONEncoder import JSONEncoder as Database
-from pycs.jobs.util.JSONEncoder import JSONEncoder as Jobs
+from pycs.database.util.JSONEncoder import JSONEncoder as DatabaseEncoder
+from pycs.jobs.util.JSONEncoder import JSONEncoder as JobsEncoder
 
 
 class JSONEncoder(Base):
     """
-    prepares job objects to be json encoded
+    prepares job and DB objects to be json encoded
     """
 
     def default(self, o: Any) -> Any:
         module = o.__class__.__module__
 
         if module.startswith('pycs.database'):
-            return Database().default(o)
+            return DatabaseEncoder().default(o)
+
         if module.startswith('pycs.jobs'):
-            return Jobs().default(o)
+            return JobsEncoder().default(o)
+
+        if isinstance(o, datetime.datetime):
+            return str(o)
+
+        if isinstance(o, dict):
+            return o
 
         return o.__dict__

pycs/interfaces/MediaBoundingBox.py

@@ -1,6 +1,7 @@
 from pycs.database.Result import Result
 
 
+#pylint: disable=too-few-public-methods
 class MediaBoundingBox:
     """
     A bounding box defined by it's upper left corner coordinates plus width and height. All those

pycs/interfaces/MediaFile.py

@@ -89,15 +89,12 @@ class MediaFile:
             self.__notifications.add(self.__notifications.notifications.remove_result, result)
 
     def __get_results(self, origin: str) -> List[Union[MediaImageLabel, MediaBoundingBox]]:
-        def map_r(result: Result) -> Union[MediaImageLabel, MediaBoundingBox]:
-            if result.type == 'labeled-image':
-                return MediaImageLabel(result)
 
-            return MediaBoundingBox(result)
+        def result_to_media(result: Result) -> Union[MediaImageLabel, MediaBoundingBox]:
+            cls = MediaImageLabel if result.type == 'labeled-image' else MediaBoundingBox
+            return cls(result)
 
-        return list(map(map_r,
-                        filter(lambda r: r.origin == origin,
-                               self.__file.results())))
+        return [result_to_media(r) for r in self.__file.results.filter_by(origin=origin).all()]
 
     def results(self) -> List[Union[MediaImageLabel, MediaBoundingBox]]:
         """
@@ -127,7 +124,7 @@ class MediaFile:
             'frames': self.frames,
             'fps': self.fps,
             'path': self.path,
-            'filename': self.__file.name + self.__file.extension,
+            'filename': self.__file.filename,
             'results': list(map(lambda r: r.serialize(), self.results())),
             'predictions': list(map(lambda r: r.serialize(), self.predictions())),
         }

pycs/interfaces/MediaFileList.py

@@ -49,11 +49,11 @@ class MediaFileList:
             source = self.__project
 
         if self.__label is None:
-            for file in source.files():
+            for file in source.files.all():
                 yield MediaFile(file, self.__notifications)
         else:
-            for file in source.files():
-                for result in file.results():
+            for file in source.files.all():
+                for result in file.results.all():
                     if result.label == self.__label:
                         yield MediaFile(file, self.__notifications)
                         break

pycs/interfaces/MediaImageLabel.py

@@ -1,6 +1,7 @@
 from pycs.database.Result import Result
 
 
+#pylint: disable=too-few-public-methods
 class MediaImageLabel:
     """
     An image label with an optional frame index for videos.

pycs/interfaces/MediaLabel.py

@@ -1,13 +1,14 @@
 from pycs.database.Label import Label
 
 
+#pylint: disable=too-few-public-methods
 class MediaLabel:
     """
     a label
     """
 
     def __init__(self, label: Label):
-        self.identifier = label.identifier
+        self.identifier = label.id
         self.parent = None
         self.children = []
         self.reference = label.reference

pycs/interfaces/MediaStorage.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.interfaces.MediaFileList import MediaFileList
 from pycs.interfaces.MediaLabel import MediaLabel
@@ -11,13 +11,13 @@ class MediaStorage:
     helper class for pipelines to interact with database entities
     """
 
-    def __init__(self, db: Database, project_id: int, notifications: NotificationList = None):
-        self.__db = db
+    def __init__(self, project_id: int, notifications: NotificationList = None):
         self.__project_id = project_id
         self.__notifications = notifications
 
-        self.__project = self.__db.project(self.__project_id)
-        self.__collections = self.__project.collections()
+        self.__project = Project.query.get(self.__project_id)
+        # this one is not used anywhere
+        # self.__collections = self.__project.collections.all()
 
     def labels(self) -> List[MediaLabel]:
         """
@@ -25,12 +25,12 @@ class MediaStorage:
 
         :return: list of labels
         """
-        label_list = self.__project.labels()
-        label_dict = {la.identifier: MediaLabel(la) for la in label_list}
+        label_list = self.__project.labels.all()
+        label_dict = {la.id: MediaLabel(la) for la in label_list}
         result = []
 
         for label in label_list:
-            medial_label = label_dict[label.identifier]
+            medial_label = label_dict[label.id]
 
             if label.parent_id is not None:
                 medial_label.parent = label_dict[label.parent_id]
@@ -46,10 +46,7 @@ class MediaStorage:
 
         :return: list of root-level labels (parent is None)
         """
-        return list(filter(
-            lambda ml: ml.parent is None,
-            self.labels()
-        ))
+        return [label for label in self.labels() if label.parent is None]
 
     def files(self) -> MediaFileList:
         """

pycs/interfaces/Pipeline.py

@@ -9,6 +9,7 @@ class Pipeline:
     pipeline interface that should be implemented by model developers
     """
 
+    #pylint: disable=unnecessary-pass
     def __init__(self, root_folder: str, distribution: dict):
         """
         prepare everything needed to run jobs later
@@ -18,6 +19,7 @@ class Pipeline:
         """
         pass
 
+    #pylint: disable=unnecessary-pass
     def close(self):
         """
         is called everytime a pipeline is not needed anymore and should be used
@@ -27,6 +29,7 @@ class Pipeline:
         """
         pass
 
+    #pylint: disable=no-self-use
     def collections(self) -> List[dict]:
         """
         is called while initializing a pipeline to receive available

pycs/jobs/Job.py

@@ -11,8 +11,8 @@ class Job:
 
     # pylint: disable=too-few-public-methods
     def __init__(self, project: Project, job_type: str, name: str):
-        self.identifier = str(uuid1())
-        self.project_id = project.identifier
+        self.uuid = self.identifier = str(uuid1())
+        self.project_id = project.id
         self.type = job_type
         self.name = name
         self.exception = None

pycs/jobs/JobRunner.py

@@ -1,9 +1,16 @@
+import traceback
+
 from concurrent.futures import ThreadPoolExecutor
 from time import time
 from types import GeneratorType
-from typing import Callable, List, Generator, Optional, Any
-
-from eventlet import spawn_n, tpool
+from typing import Any
+from typing import Callable
+from typing import Generator
+from typing import List
+from typing import Optional
+
+from eventlet import spawn_n
+from eventlet import tpool
 from eventlet.event import Event
 from eventlet.queue import Queue
 
@@ -219,6 +226,7 @@ class JobRunner:
 
             # save exceptions to show in ui
             except Exception as exception:
+                traceback.print_exc()
                 job.exception = f'{type(exception).__name__} ({str(exception)})'
 
             # remove from group dict

pycs/util/FileOperations.py

@@ -1,7 +1,10 @@
 import os
-from typing import Tuple
+import typing as T
+
+from collections import namedtuple
 
 import cv2
+
 from PIL import Image
 
 from pycs.database.File import File
@@ -9,6 +12,8 @@ from pycs.database.File import File
 DEFAULT_JPEG_QUALITY = 80
 
 
+BoundingBox = namedtuple("BoundingBox", "x y w h")
+
 def file_info(data_folder: str, file_name: str, file_ext: str):
     """
     Receive file type, frame count and frames per second.
@@ -46,7 +51,10 @@ def file_info(data_folder: str, file_name: str, file_ext: str):
     return file_type, frames, fps
 
 
-def resize_file(file: File, project_root: str, max_width: int, max_height: int) -> Tuple[str, str]:
+def resize_file(file: File,
+                project_root: str,
+                max_width: int,
+                max_height: int) -> T.Tuple[str, str]:
     """
     If file type equals video this function extracts a thumbnail first. It calls resize_image
     to resize and returns the resized files directory and name.
@@ -71,18 +79,17 @@ def resize_file(file: File, project_root: str, max_width: int, max_height: int)
                                    project_root,
                                    'temp',
                                    f'{file.uuid}_{max_width}_{max_height}.jpg')
-    result = resize_image(abs_file_path, abs_target_path, max_width, max_height)
+    resized = resize_image(abs_file_path, abs_target_path, max_width, max_height)
 
     # return path
-    if result is not None:
+    if resized:
         return os.path.split(abs_target_path)
 
     return os.path.split(abs_file_path)
 
 
-def crop_file(file: File, project_root: str,
-              x: float, y: float, w: float, h: float,
-              max_width: int, max_height: int) -> Tuple[str, str]:
+def crop_file(file: File, project_root: str, box: BoundingBox,
+              max_width: int, max_height: int) -> T.Tuple[str, str]:
     """
     gets a file for the given file_id, crops the according image to the
     bounding box and saves the crops in the temp folder of the project.
@@ -91,10 +98,8 @@ def crop_file(file: File, project_root: str,
     :param project_root: project root folder path
     :param max_width: maximum image or thumbnail width
     :param max_height: maximum image or thumbnail height
-    :param x: relative x-coordinate of the top left corner
-    :param y: relative y-coordinate of the top left corner
-    :param w: relative width of the bounding box
-    :param h: relative height of the bounding box
+    :param box: BoundingBox with relative x, y coordinates and
+        the relative height and width of the bounding box
 
     :return: directory and file name of the cropped patch
     """
@@ -107,24 +112,26 @@ def crop_file(file: File, project_root: str,
 
         abs_file_path = abs_target_path
 
+    bounding_box_suffix = f"{box.x}_{box.y}_{box.w}_{box.h}"
     # crop image file
     abs_target_path = os.path.join(os.getcwd(),
                                    project_root,
                                    'temp',
-                                   f'{file.uuid}_{x}_{y}_{w}_{h}.jpg')
-    result = crop_image(abs_file_path, abs_target_path, x, y, w, h)
+                                   f'{file.uuid}_{bounding_box_suffix}.jpg')
+    cropped = crop_image(abs_file_path, abs_target_path, box)
 
-    if result:
+    if cropped:
         abs_file_path = abs_target_path
 
     # resize image
+    target_file_name = f'{file.uuid}_{max_width}_{max_height}_{bounding_box_suffix}.jpg'
     abs_target_path = os.path.join(os.getcwd(),
                                    project_root,
                                    'temp',
-                                   f'{file.uuid}_{max_width}_{max_height}_{x}_{y}_{w}_{h}.jpg')
-    result = resize_image(abs_file_path, abs_target_path, max_width, max_height)
+                                   target_file_name)
+    resized = resize_image(abs_file_path, abs_target_path, max_width, max_height)
 
-    if result:
+    if resized:
         abs_file_path = abs_target_path
 
     # return image
@@ -193,17 +200,15 @@ def resize_image(file_path: str, target_path: str, max_width: int, max_height: i
     return True
 
 
-def crop_image(file_path: str, target_path: str, x: float, y: float, w: float, h: float) -> bool:
+def crop_image(file_path: str, target_path: str, box: BoundingBox) -> bool:
     """
     Crop an image with the given coordinates, width and height.
     If however no crop is applied no new file is stored.
 
     :param file_path: path to source file
     :param target_path: path to target file
-    :param x: crop x position (normalized)
-    :param y: crop y position (normalized)
-    :param w: crop width (normalized)
-    :param h: crop height (normalized)
+    :param box: BoundingBox with relative x, y coordinates and
+        the relative height and width of the bounding box
     :return: `True` if a crop operation was performed or the target file already exists
     """
     # return if file exists
@@ -211,7 +216,7 @@ def crop_image(file_path: str, target_path: str, x: float, y: float, w: float, h
         return True
 
     # abort if no crop would be applied
-    if x <= 0 and y <= 0 and w >= 1 and h >= 1:
+    if box.x <= 0 and box.y <= 0 and box.w >= 1 and box.h >= 1:
         return False
 
     # load full size image
@@ -219,13 +224,12 @@ def crop_image(file_path: str, target_path: str, x: float, y: float, w: float, h
     img_width, img_height = image.size
 
     # calculate absolute crop position
-    crop_x1 = int(img_width * x)
-    crop_y1 = int(img_height * y)
-    crop_x2 = min(int(img_width * w) + crop_x1, img_width)
-    crop_y2 = min(int(img_height * h) + crop_y1, img_height)
+    crop_x1 = int(img_width * box.x)
+    crop_y1 = int(img_height * box.y)
+    crop_x2 = min(int(img_width * box.w) + crop_x1, img_width)
+    crop_y2 = min(int(img_height * box.h) + crop_y1, img_height)
 
     # crop image
-    print(crop_x1, crop_y1, crop_x2, crop_y2)
     cropped_image = image.crop((crop_x1, crop_y1, crop_x2, crop_y2))
 
     # save to file

pycs/util/PipelineCache.py

@@ -1,9 +1,16 @@
-from queue import Queue
+import queue
+import warnings
+
 from threading import Lock
-from time import time, sleep
+from time import sleep
+from time import time
+
+import eventlet
 
-from eventlet import tpool, spawn_n
+from eventlet import spawn_n
+from eventlet import tpool
 
+from pycs import app
 from pycs.database.Project import Project
 from pycs.interfaces.Pipeline import Pipeline
 from pycs.jobs.JobRunner import JobRunner
@@ -17,20 +24,40 @@ class PipelineCache:
     """
     CLOSE_TIMER = 120
 
-    def __init__(self, jobs: JobRunner):
+    def __init__(self, jobs: JobRunner, cache_time: float = None):
         self.__jobs = jobs
 
         self.__pipelines = {}
-        self.__queue = Queue()
+        self.__is_running = False
+        self.__queue = queue.Queue()
         self.__lock = Lock()
 
+        self._cache_time = cache_time or self.CLOSE_TIMER
+        msg = ("Initialized Pipeline cache "
+            f"(pipelines are closed after {self._cache_time:.3f} sec)")
+        app.logger.info(msg)
+
+    def start(self):
+        """ starts the main worker method """
+        if self.__is_running:
+            warnings.warn("Pipeline cache is already started")
+            return
         spawn_n(self.__run)
 
-    def load_from_root_folder(self, project: Project, root_folder: str) -> Pipeline:
+    @property
+    def is_empty(self):
+        """ checks whether the pipeline cache is empty """
+        return len(self.__pipelines) == 0 and self.__queue.empty()
+
+    def shutdown(self):
+        """ puts None in the queue to signal the worker to stop """
+        self.__queue.put(None)
+
+    def load_from_root_folder(self, project_id: int, root_folder: str) -> Pipeline:
         """
         load configuration.json and create an instance from the included code object
 
-        :param project: associated project
+        :param project_id: associated project ID
         :param root_folder: path to model root folder
         :return: Pipeline instance
         """
@@ -50,7 +77,9 @@ class PipelineCache:
 
         # save instance to cache
         with self.__lock:
-            self.__pipelines[root_folder] = [1, pipeline, project]
+            if not self.__is_running:
+                warnings.warn("[save instance] pipeline cache was not started yet!")
+            self.__pipelines[root_folder] = [1, pipeline, project_id]
 
         # return
         return pipeline
@@ -58,7 +87,7 @@ class PipelineCache:
     def free_instance(self, root_folder: str):
         """
         Change an instance's status to unused and start the timer to call it's `close` function
-        after `CLOSE_TIMER` seconds. The next call to `load_from_root_folder` in this interval
+        after `_cache_time` seconds. The next call to `load_from_root_folder` in this interval
         will disable this timer.
 
         :param root_folder: path to model root folder
@@ -72,19 +101,34 @@ class PipelineCache:
         timestamp = time()
         self.__queue.put((root_folder, timestamp))
 
+        if not self.__is_running:
+            warnings.warn("[free instance] pipeline cache was not started yet!")
+
     def __get(self):
         while True:
             # get element from queue
-            root_folder, timestamp = self.__queue.get()
+            while True:
+                try:
+                    entry = self.__queue.get(block=False)
+                    break
+                except queue.Empty:
+                    eventlet.sleep(0.2)
+
+            if entry is None:
+                # closing pipeline cache
+                return None
+            root_folder, timestamp = entry
 
             # sleep if needed
-            delay = int(timestamp + self.CLOSE_TIMER - time())
+            delay = int(timestamp + self._cache_time - time())
 
             if delay > 0:
+                app.logger.info(f"Cache sleeps for {delay:.3f} sec")
                 sleep(delay)
 
             # lock and access __pipelines
             with self.__lock:
+                app.logger.info("Removing pipeline from cache")
                 instance = self.__pipelines[root_folder]
 
                 # reference counter greater than 1
@@ -100,8 +144,16 @@ class PipelineCache:
 
     def __run(self):
         while True:
+            self.__is_running = True
             # get pipeline
-            pipeline, project = tpool.execute(self.__get)
+            result = tpool.execute(self.__get)
+            if result is None:
+                self.__is_running = False
+                return
+
+            pipeline, project_id = result
+
+            project = Project.query.get(project_id)
 
             # create job to close pipeline
             self.__jobs.run(project,

requirements.dev.txt

@@ -0,0 +1,4 @@
+coverage
+pylint
+pylint-flask-sqlalchemy
+pylint-flask

requirements.txt

@@ -4,6 +4,10 @@ Pillow
 scipy
 eventlet
 flask
+flask-socketio
+flask-sqlalchemy
+sqlalchemy_serializer
+flask-migrate
 python-socketio
 munch
 scikit-image

settings.json

@@ -1,5 +1,36 @@
 {
   "host": "",
   "port": 5000,
-  "allowedOrigins": []
+  "allowedOrigins": [],
+  "projects_folder": "projects",
+  "database": "data2.sqlite3",
+  "pipeline_cache_time": 120,
+
+  "logging": {
+    "version": 1,
+    "formatters": {
+      "default": {
+        "format": "{levelname: ^7s} - [{asctime}] {filename}:L{lineno} [{funcName}]: {message}",
+        "style": "{"
+      }
+    },
+    "handlers": {
+      "console": {
+        "class": "logging.StreamHandler",
+        "stream": "ext://flask.logging.wsgi_errors_stream",
+        "formatter": "default",
+        "level": "INFO"
+      }
+    },
+    "root": {
+      "level": "INFO",
+      "handlers": ["console"]
+    },
+    "loggers": {
+      "sqlalchemy.engine": {
+        "level": "CRITICAL",
+        "handlers": ["console"]
+      }
+    }
+  }
 }

test/test_database.py

@@ -1,128 +0,0 @@
-import unittest
-from contextlib import closing
-
-from pycs.database.Database import Database
-
-
-class TestDatabase(unittest.TestCase):
-    def setUp(self) -> None:
-        # create database
-        self.database = Database(discovery=False)
-
-        # insert default models and label_providers
-        with self.database:
-            with closing(self.database.con.cursor()) as cursor:
-                # models
-                cursor.execute('''
-                    INSERT INTO models (name, description, root_folder, supports)
-                    VALUES 
-                        ('Model 1', 'Description for Model 1', 'modeldir1', '["labeled-image", "fit"]'),
-                        ('Model 2', 'Description for Model 2', 'modeldir2', '["labeled-bounding-boxes"]'),
-                        ('Model 3', 'Description for Model 3', 'modeldir3', '["labeled-bounding-boxes"]')
-                ''')
-
-                # label providers
-                cursor.execute('''
-                    INSERT INTO label_providers (name, description, root_folder, configuration_file)
-                    VALUES
-                        ('Label Provider 1', 'Description for Label Provider 1', 'labeldir1', 'file1'),
-                        ('Label Provider 2', 'Description for Label Provider 2', 'labeldir2', 'file2')
-                ''')
-
-                # projects
-                models = list(self.database.models())
-                label_providers = list(self.database.label_providers())
-
-                for i in range(3):
-                    self.database.create_project(
-                        f'Project {i + 1}', f'Project Description {i + 1}',
-                        models[i],
-                        label_providers[i] if i < 2 else None,
-                        f'projectdir{i + 1}', i == 1, f'datadir{i + 1}'
-                    )
-
-    def tearDown(self) -> None:
-        self.database.close()
-
-    def test_models(self):
-        models = list(self.database.models())
-
-        # test length
-        self.assertEqual(len(models), 3)
-
-        # test insert
-        for i in range(2):
-            self.assertEqual(models[i].identifier, i + 1)
-            self.assertEqual(models[i].name, f'Model {i + 1}')
-            self.assertEqual(models[i].description, f'Description for Model {i + 1}')
-            self.assertEqual(models[i].root_folder, f'modeldir{i + 1}')
-
-        self.assertEqual(models[0].supports, ['labeled-image', 'fit'])
-        self.assertEqual(models[1].supports, ['labeled-bounding-boxes'])
-
-        # test copy
-        copy, _ = models[0].copy_to('Copied Model', 'modeldir3')
-        self.assertEqual(copy.identifier, 3)
-        self.assertEqual(copy.name, 'Copied Model')
-        self.assertEqual(copy.description, 'Description for Model 1')
-        self.assertEqual(copy.root_folder, 'modeldir3')
-        self.assertEqual(copy.supports, ['labeled-image', 'fit'])
-
-    def test_label_providers(self):
-        label_providers = list(self.database.label_providers())
-
-        # test length
-        self.assertEqual(len(label_providers), 2)
-
-        for i in range(2):
-            self.assertEqual(label_providers[i].identifier, i + 1)
-            self.assertEqual(label_providers[i].name, f'Label Provider {i + 1}')
-            self.assertEqual(label_providers[i].description, f'Description for Label Provider {i + 1}')
-            self.assertEqual(label_providers[i].root_folder, f'labeldir{i + 1}')
-            self.assertEqual(label_providers[i].configuration_file, f'file{i + 1}')
-
-    def test_projects(self):
-        models = list(self.database.models())
-        label_providers = list(self.database.label_providers())
-        projects = list(self.database.projects())
-
-        # create projects
-        for i in range(3):
-            project = projects[i]
-
-            self.assertEqual(project.identifier, i + 1)
-            self.assertEqual(project.name, f'Project {i + 1}')
-            self.assertEqual(project.description, f'Project Description {i + 1}')
-            self.assertEqual(project.model_id, i + 1)
-            self.assertEqual(project.model().__dict__, models[i].__dict__)
-            self.assertEqual(project.label_provider_id, label_providers[i].identifier if i < 2 else None)
-            self.assertEqual(
-                project.label_provider().__dict__ if project.label_provider() is not None else None,
-                label_providers[i].__dict__ if i < 2 else None
-            )
-            self.assertEqual(project.root_folder, f'projectdir{i + 1}')
-            self.assertEqual(project.external_data, i == 1)
-            self.assertEqual(project.data_folder, f'datadir{i + 1}')
-
-        # get projects
-        self.assertEqual(len(list(self.database.projects())), 3)
-
-        # remove a project
-        list(self.database.projects())[0].remove()
-        projects = list(self.database.projects())
-
-        self.assertEqual(len(projects), 2)
-        self.assertEqual(projects[0].name, 'Project 2')
-
-        # set properties
-        project = list(self.database.projects())[0]
-
-        project.set_name('Project 0')
-        self.assertEqual(list(self.database.projects())[0].name, 'Project 0')
-
-        project.set_description('Description 0')
-        self.assertEqual(list(self.database.projects())[0].description, 'Description 0')
-
-
-if __name__ == '__main__':
-    unittest.main()

tests/__init__.py

@@ -0,0 +1,2 @@
+from tests.client import *
+from tests.test_database import *

tests/base.py

@@ -0,0 +1,165 @@
+import eventlet
+import logging
+import os
+import shutil
+import typing as T
+import unittest
+
+from pathlib import Path
+from unittest import mock
+
+from pycs import app
+from pycs import db
+from pycs import settings
+from pycs.database.LabelProvider import LabelProvider
+from pycs.database.Model import Model
+from pycs.frontend.WebServer import WebServer
+from pycs.util.PipelineCache import PipelineCache
+
+server = None
+
+def pаtch_tpool_execute(test_func):
+
+    def call_func(func, *args, **kwargs):
+        return func(*args, **kwargs)
+
+    decorator = mock.patch("eventlet.tpool.execute",
+        side_effect=call_func)
+
+    return decorator(test_func)
+
+class BaseTestCase(unittest.TestCase):
+    _sleep_time = 0.2
+    server = None
+
+    DB_FILE = Path.cwd() / "test.sqlite"
+
+
+    @classmethod
+    def setUpClass(cls, discovery: bool = False):
+        global server
+        app.logger.setLevel(logging.CRITICAL)
+        app.config["TESTING"] = True
+        app.config["WTF_CSRF_ENABLED"] = False
+        app.config["DEBUG"] = False
+        app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{cls.DB_FILE}"
+
+        if server is None:
+            settings["pipeline_cache_time"] = 2
+            server = WebServer(app, settings, discovery)
+
+        if cls.server is None:
+            cls.server = server
+        db.create_all()
+
+        # # run discovery modules manually
+        # Model.discover("models/")
+        # LabelProvider.discover("labels/")
+        cls.server.pipelines.start()
+
+
+    def wait_for_bg_jobs(self, raise_errors=True):
+
+        # wait for JobRunner jobs to finish
+        while True:
+            ready = True
+            for job in self.server.jobs.list():
+                if job.finished is None:
+                    app.logger.debug(f"{job} is not finished!")
+                    ready = False
+                    break
+
+                if raise_errors:
+                    self.assertTrue(job.exception is None,
+                        job.exception)
+
+            if ready:
+                break
+
+            self.wait_for_coroutines()
+
+        # wait for PipelineCache to finish
+
+        while not self.server.pipelines.is_empty:
+            self.wait_for_coroutines()
+
+    def wait_for_coroutines(self):
+        eventlet.sleep(self._sleep_time)
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        cls.server.pipelines.shutdown()
+
+        while not cls.server.pipelines.is_empty:
+            eventlet.sleep(cls._sleep_time)
+
+
+
+    def setUp(self):
+        self.projects_dir = app.config["TEST_PROJECTS_DIR"] = "test_projects"
+
+        db.create_all()
+
+        self.client = app.test_client()
+        self.context = app.test_request_context()
+        self.context.push()
+
+        self.setupModels()
+
+    def setupModels(self):
+        pass
+
+    def tearDown(self):
+        self.wait_for_bg_jobs(raise_errors=False)
+
+        self.context.pop()
+
+        if os.path.exists(self.projects_dir):
+            shutil.rmtree(self.projects_dir)
+
+        db.drop_all()
+
+
+    def _do_request(self, request_func, *args,  status_code=200, **kwargs):
+        response = request_func(*args, follow_redirects=True, **kwargs)
+        try:
+            content = response.get_data().decode()
+        except UnicodeDecodeError:
+            content = response.get_data()
+
+        self.assertEqual(response.status_code, status_code, content)
+        return response
+
+    def post(self,
+             url: str,
+             *,
+             status_code: int = 200,
+             data: T.Optional[dict] = None,
+             json: T.Optional[dict] = None,
+             **kwargs):
+
+        return self._do_request(
+            self.client.post,
+            url,
+            status_code=status_code,
+            json=json,
+            data=data,
+            **kwargs
+        )
+
+    def get(self,
+             url: str,
+             *,
+             status_code: int = 200,
+             data: T.Optional[dict] = None,
+             json: T.Optional[dict] = None):
+
+        return self._do_request(
+            self.client.get,
+            url,
+            status_code=status_code,
+            json=json,
+            data=data,
+        )
+

tests/client/__init__.py

@@ -0,0 +1,113 @@
+import tempfile
+
+from flask import url_for
+
+from pycs.database.LabelProvider import LabelProvider
+from pycs.database.Model import Model
+
+from tests.base import BaseTestCase
+from tests.client.file_tests import *
+from tests.client.label_tests import *
+from tests.client.pipeline_tests import *
+from tests.client.project_tests import *
+from tests.client.result_tests import *
+
+
+class FolderInformationTest(BaseTestCase):
+
+    def _check(self, url, folder, content_should):
+        response = self.post(url, json=dict(folder=folder))
+        self.assertTrue(response.is_json)
+        self.assertDictEqual(content_should, response.json)
+
+
+    def test_folder_information(self):
+
+        url = url_for("folder_information")
+        self.post(url, json=dict(), status_code=400)
+
+        with tempfile.TemporaryDirectory() as folder:
+
+            self._check(url, "/not_existing/folder",
+                dict(exists=False))
+
+            for i in range(10):
+                self._check(url, folder,
+                    dict(exists=True, count=i))
+
+                f = tempfile.NamedTemporaryFile(dir=folder, delete=False)
+
+
+class ListModelsAndLabelProviders(BaseTestCase):
+
+    def test_list_models(self):
+        self.assertEqual(0, Model.query.count())
+        url = url_for("list_models")
+
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+
+        self.assertEqual([], response.json)
+
+        models = {}
+        n = 5
+        for i, _ in enumerate(range(n), 1):
+
+            model = Model.new(
+                commit=False,
+                name=f"TestModel{i}",
+                description="Model for a test case #{i}",
+                root_folder=f"models/fixed_model{i}",
+            )
+            model.supports = ["labeled-image"]
+            model.flush()
+
+            models[model.id] = model
+        model.commit()
+
+        self.assertEqual(n, Model.query.count())
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+        content = response.json
+        self.assertEqual(n, len(response.json))
+
+        for entry in content:
+            model = models[entry["id"]]
+            self.assertDictEqual(model.serialize(), entry)
+
+
+    def test_list_label_providers(self):
+        self.assertEqual(0, LabelProvider.query.count())
+        url = url_for("label_providers")
+
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+
+        self.assertEqual([], response.json)
+
+        providers = {}
+        n = 5
+        for i, _ in enumerate(range(n), 1):
+
+            provider = LabelProvider.new(
+                commit=False,
+                name=f"Testprovider{i}",
+                description="LabelProvider for a test case #{i}",
+                root_folder=f"providers/fixed_provider{i}",
+                configuration_file=f"providers/fixed_provider{i}/config.json",
+            )
+            provider.supports = ["labeled-image"]
+            provider.flush()
+
+            providers[provider.id] = provider
+        provider.commit()
+
+        self.assertEqual(n, LabelProvider.query.count())
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+        content = response.json
+        self.assertEqual(n, len(response.json))
+
+        for entry in content:
+            provider = providers[entry["id"]]
+            self.assertDictEqual(provider.serialize(), entry)

tests/client/file_tests.py

@@ -0,0 +1,414 @@
+import cv2
+import io
+import numpy as np
+import os
+import uuid
+
+from PIL import Image
+from flask import url_for
+from pathlib import Path
+
+from pycs.database.File import File
+from pycs.util.FileOperations import BoundingBox
+
+from tests.base import pаtch_tpool_execute
+from tests.client.label_tests import _BaseLabelTests
+
+
+class _BaseFileTests(_BaseLabelTests):
+
+    def setupModels(self):
+        super().setupModels()
+        root = Path(self.project.root_folder)
+        data_root = Path(self.project.data_folder)
+
+        for folder in [data_root, root / "temp"]:
+            folder.mkdir(exist_ok=True, parents=True)
+
+
+class FileCreationTests(_BaseFileTests):
+
+    @pаtch_tpool_execute
+    def test_file_upload_project_with_external_data(self, mocked_execute=None):
+
+        file_content = b"some content+1"
+        url = url_for("upload_file", project_id=self.project.id)
+
+        self.assertEqual(0, File.query.count())
+
+        self.project.external_data = True
+        self.project.commit()
+
+        self.post(url,
+            data=dict(file=(io.BytesIO(file_content), "image.jpg")),
+            content_type="multipart/form-data",
+            status_code=400,
+        )
+
+        self.assertEqual(0, File.query.count())
+
+    @pаtch_tpool_execute
+    def test_file_upload(self, mocked_execute=None):
+
+        url = url_for("upload_file", project_id=4242)
+        self.post(url, data=dict(), status_code=404)
+
+        file_content = b"some content+1"
+        url = url_for("upload_file", project_id=self.project.id)
+
+        self.assertEqual(0, File.query.count())
+
+        self.post(url, data=dict(),
+            status_code=400)
+        self.assertEqual(0, File.query.count())
+
+        self.post(url,
+            data=dict(file=(io.BytesIO(file_content), "image.jpg")),
+            content_type="multipart/form-data",
+        )
+
+        self.assertEqual(1, File.query.count())
+
+        # this does not work, if we do not set the CONTENT_LENGTH by ourself
+        # file = File.query.first()
+        # self.assertEqual(len(file_content), file.size)
+
+
+class FileDeletionTests(_BaseFileTests):
+
+    def test_file_removal(self):
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".jpg",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+
+        self.assertEqual(1, self.project.files.count())
+
+        with open(file.absolute_path, "w"):
+            pass
+
+        self.assertTrue(os.path.exists(file.absolute_path))
+
+        url = url_for("remove_file", file_id=file.id)
+        self.post(url, json=dict(), status_code=400)
+        self.post(url, json=dict(remove=False), status_code=400)
+        self.post(url, json=dict(remove=True))
+        self.assertEqual(0, self.project.files.count())
+        self.assertFalse(os.path.exists(file.absolute_path))
+
+        url = url_for("remove_file", file_id=4242)
+        self.post(url, json=dict(remove=True), status_code=404)
+
+    def test_file_removal_from_project_with_external_data(self):
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".jpg",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+
+        with open(file.absolute_path, "w"):
+            pass
+
+        self.project.external_data = True
+        self.assertTrue(os.path.exists(file.absolute_path))
+        url = url_for("remove_file", file_id=file.id)
+
+        self.assertEqual(1, self.project.files.count())
+        self.post(url, json=dict(remove=True), status_code=400)
+        self.assertEqual(1, self.project.files.count())
+
+
+class FileGettingTests(_BaseFileTests):
+
+
+    def test_get_file_getting(self):
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".jpg",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+        self.assertEqual(1, self.project.files.count())
+
+        url = url_for("get_file", file_id=file.id)
+
+        # without an actual file, this GET request returns 404
+        self.get(url, status_code=404)
+
+        content = b"some text"
+        with open(file.absolute_path, "wb") as f:
+            f.write(content)
+
+        response = self.get(url)
+
+        self.assertFalse(response.is_json)
+        self.assertEqual(content, response.data)
+
+    def test_get_prev_next_file(self):
+
+        for i in range(1, 6):
+            file_uuid = str(uuid.uuid1())
+            file, is_new = self.project.add_file(
+                uuid=file_uuid,
+                file_type="image",
+                name=f"name_{i}",
+                filename=f"image_{i}",
+                extension=".jpg",
+                size=32*1024,
+            )
+
+            self.assertTrue(is_new)
+            with open(file.absolute_path, "wb") as f:
+                f.write(b"some content")
+
+        self.assertEqual(5, self.project.files.count())
+        files = self.project.files.all()
+
+        url = url_for("get_previous_and_next_file", file_id=4542)
+        self.get(url, status_code=404)
+
+        for i, file in enumerate(files):
+            p_file, n_file = None, None
+
+            if i != 0:
+                p_file = files[i-1].serialize()
+
+            if i < len(files)-1:
+                n_file = files[i+1].serialize()
+
+
+            url = url_for("get_previous_and_next_file", file_id=file.id)
+
+            response = self.get(url)
+            self.assertTrue(response.is_json)
+
+            content_should = dict(
+                next=n_file,
+                nextInCollection=n_file,
+                previous=p_file,
+                previousInCollection=p_file,
+            )
+
+            self.assertDictEqual(content_should, response.json)
+
+        files[1].delete()
+        file = files[2]
+        p_file, n_file = files[0], files[3]
+        url = url_for("get_previous_and_next_file", file_id=file.id)
+
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+
+        content_should = dict(
+            next=n_file.serialize(),
+            nextInCollection=n_file.serialize(),
+            previous=p_file.serialize(),
+            previousInCollection=p_file.serialize(),
+        )
+
+        self.assertDictEqual(content_should, response.json)
+
+        files[3].delete()
+        file = files[2]
+        p_file, n_file = files[0], files[4]
+        url = url_for("get_previous_and_next_file", file_id=file.id)
+
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+
+        content_should = dict(
+            next=n_file.serialize(),
+            nextInCollection=n_file.serialize(),
+            previous=p_file.serialize(),
+            previousInCollection=p_file.serialize(),
+        )
+
+        self.assertDictEqual(content_should, response.json)
+
+
+
+class FileResizingTests(_BaseFileTests):
+
+    def _add_image(self, shape, file: File):
+        image = np.random.randint(0, 256, shape).astype(np.uint8)
+
+        im = Image.fromarray(image)
+
+        im.save(file.absolute_path)
+        self.assertTrue(os.path.exists(file.absolute_path))
+        return image
+
+    def _compare_images(self, im0, im1, threshold=1e-3):
+        im0, im1 = im0 / 255, im1 / 255
+        mse = np.mean((im0 - im1)**2)
+        self.assertLess(mse, threshold)
+
+    @pаtch_tpool_execute
+    def test_resize_image(self, mocked_execute):
+
+        self.get(url_for("get_resized_file", file_id=4242, resolution=300), status_code=404)
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".png",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+
+        image = self._add_image((300, 300), file)
+
+        for upscale in [300, 1200, 500, 320]:
+            url = url_for("get_resized_file", file_id=file.id, resolution=upscale)
+            response = self.get(url)
+
+            self.assertFalse(response.is_json)
+
+            returned_im = _im_from_bytes(response.data)
+
+            self.assertEqual(image.shape, returned_im.shape)
+            self._compare_images(image, returned_im)
+
+
+        # repeat the last scale two times to get the cached resized image
+        for downscale in [299, 200, 150, 32, 32]:
+            sm_image = _resize(image, downscale)
+
+            url = url_for("get_resized_file", file_id=file.id, resolution=downscale)
+            response = self.get(url)
+
+            self.assertFalse(response.is_json)
+
+            returned_im = _im_from_bytes(response.data)
+
+            self.assertEqual(sm_image.shape, returned_im.shape)
+            self._compare_images(sm_image, returned_im)
+
+            del sm_image
+
+    @pаtch_tpool_execute
+    def test_resize_image_not_found(self, mocked_execute):
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".png",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+
+        image = self._add_image((300, 300), file)
+
+        save = file.path
+        file.path = "/some/nonexisting/path"
+        file.commit()
+        url = url_for("get_resized_file", file_id=file.id, resolution=300)
+        response = self.get(url, status_code=404)
+
+        file.path = save
+        file.commit()
+
+    @pаtch_tpool_execute
+    def test_crop_image_not_found(self, mocked_execute):
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".png",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+
+        image = self._add_image((300, 300), file)
+
+        save = file.path
+        file.path = "/some/nonexisting/path"
+        file.commit()
+        url = url_for("get_cropped_file", file_id=file.id,
+            resolution=300, crop_box="0x0x1x1")
+        response = self.get(url, status_code=404)
+
+        file.path = save
+        file.commit()
+
+
+    @pаtch_tpool_execute
+    def test_crop_image(self, mocked_execute):
+
+        file_uuid = str(uuid.uuid1())
+        file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name=f"name",
+            filename=f"image",
+            extension=".png",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+
+        image = self._add_image((300, 300), file)
+
+        for box in [(0,0,1,1), (0,0,1/2,1/2), (1/2,1/2, 1, 1), (1/3,1/2,3/4, 1), ]:
+            url = url_for("get_cropped_file", file_id=file.id,
+                resolution=300, crop_box="x".join(map(str, box)))
+            response = self.get(url)
+
+            self.assertFalse(response.is_json)
+
+            returned_im = _im_from_bytes(response.data)
+
+            crop = _crop(image, BoundingBox(*box))
+            self.assertEqual(crop.shape, returned_im.shape)
+            self._compare_images(crop, returned_im)
+
+
+def _im_from_bytes(data: bytes) -> np.ndarray:
+    return np.asarray(Image.open(io.BytesIO(data)))
+
+
+def _resize(image: np.ndarray, size: int) -> np.ndarray:
+    return np.asarray(Image.fromarray(image).resize((size, size)))
+
+
+def _crop(image: np.ndarray, box: BoundingBox) -> np.ndarray:
+    h, w, *c = image.shape
+
+    x0, y0 = int(w * box.x), int(h * box.y)
+    crop_w, crop_h = int(w * box.w), int(h * box.h)
+    x1, y1 = x0 + crop_w, y0 + crop_h
+
+    return image[y0:y1, x0:x1]

tests/client/label_tests.py

@@ -0,0 +1,340 @@
+from flask import url_for
+
+
+from pycs import db
+from pycs.database.Label import Label
+from pycs.database.Model import Model
+from pycs.database.Project import Project
+
+from tests.client.project_tests import _BaseProjectTests
+
+
+class _BaseLabelTests(_BaseProjectTests):
+
+    def setupModels(self):
+        super().setupModels()
+
+        self.project = Project.new(
+            name="test_project",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+        )
+
+    def tearDown(self):
+        self.project.delete()
+        super().tearDown()
+
+class LabelCreationTests(_BaseLabelTests):
+
+    def setUp(self):
+        super().setUp()
+        self.url = url_for("create_label", project_id=self.project.id)
+
+    def test_create_labels(self):
+        self.assertEqual(0, self.project.labels.count())
+
+        self.post(self.url, json=dict(), status_code=400)
+        self.assertEqual(0, self.project.labels.count())
+
+        for i in range(1, 11):
+            self.post(self.url, json=dict(name=f"Label{i}"))
+            self.assertEqual(self.project.labels.count(), i)
+            last_label = Label.query.order_by(Label.id.desc()).first()
+            self.assertEqual(last_label.name, f"Label{i}")
+
+    def test_create_label_with_same_name(self):
+        self.assertEqual(0, self.project.labels.count())
+
+        name = "SameName"
+        for i in range(1, 11):
+            self.post(self.url, json=dict(name=name))
+            self.assertEqual(self.project.labels.count(), i)
+            last_label = Label.query.order_by(Label.id.desc()).first()
+            self.assertEqual(last_label.name, name)
+
+    def test_create_labels_with_reference(self):
+        self.assertEqual(0, self.project.labels.count())
+
+        for i in range(1, 11):
+            self.post(self.url, json=dict(name=f"Label{i}", reference=f"label_ref{i}"))
+            self.assertEqual(self.project.labels.count(), i)
+            last_label = Label.query.order_by(Label.id.desc()).first()
+            self.assertEqual(last_label.name, f"Label{i}")
+            self.assertEqual(last_label.reference, f"label_ref{i}")
+
+
+    def test_create_labels_with_same_reference(self):
+        self.assertEqual(0, self.project.labels.count())
+
+        ref = "same_ref"
+        self.post(self.url, json=dict(name=f"Label", reference=ref))
+        last_label = Label.query.order_by(Label.id.desc()).first()
+        self.assertEqual(last_label.name, f"Label")
+        self.assertEqual(last_label.reference, ref)
+
+        for i in range(2, 11):
+            self.post(self.url, status_code=400, json=dict(name=f"Label{i}", reference=ref))
+            self.assertEqual(self.project.labels.count(), 1)
+            last_label = Label.query.order_by(Label.id.desc()).first()
+            self.assertEqual(last_label.name, f"Label")
+            self.assertEqual(last_label.reference, ref)
+
+
+class LabelListTests(_BaseLabelTests):
+
+    def setupModels(self):
+        super().setupModels()
+
+        for i in range(1, 11):
+            parent, is_new = self.project.create_label(
+                name=f"Label{i}",
+                reference=f"label{i}",
+                hierarchy_level="Level1",
+            )
+            self.assertTrue(is_new)
+
+            for j in range(1, 4):
+                label, is_new = self.project.create_label(
+                    name=f"Label{i}_{j}",
+                    reference=f"label{i}_{j}",
+                    hierarchy_level="Level2",
+                    parent=parent)
+                self.assertTrue(is_new)
+
+                for k in range(1, 4):
+                    final_label, is_new = self.project.create_label(
+                        name=f"Label{i}_{j}_{k}",
+                        reference=f"label{i}_{j}_{k}",
+                        parent=label)
+                    self.assertTrue(is_new)
+
+    def test_list_labels(self):
+        self.get(url_for("list_labels", project_id=4242), status_code=404)
+
+        response = self.get(url_for("list_labels", project_id=self.project.id))
+        self.assertTrue(response.is_json)
+        content = response.json
+
+        labels = {l.id: l for l in self.project.labels.all()}
+
+        self.assertEqual(len(labels), len(content))
+
+        for entry in content:
+            label = labels[entry["id"]]
+            self.assertDictEqual(label.serialize(), entry)
+
+    def test_list_label_tree(self):
+        self.get(url_for("list_label_tree", project_id=4242), status_code=404)
+
+        response = self.get(url_for("list_label_tree", project_id=self.project.id))
+        self.assertTrue(response.is_json)
+        content = response.json
+
+        root_labels = {l.id: l for l in Label.query.filter(
+            Label.project_id==self.project.id, Label.parent_id==None).all()}
+
+        self.assertEqual(len(root_labels), len(content))
+
+        Label.serialize_only += ("children",)
+        for entry in content:
+            label = root_labels[entry["id"]]
+            self.assertDictEqual(label.serialize(), entry)
+
+
+
+class LabelRemovalTests(_BaseLabelTests):
+
+    def url(self, label_id):
+        return url_for("remove_label",
+            project_id=self.project.id, label_id=label_id)
+
+    def setupModels(self):
+        super().setupModels()
+        self.N = 10
+        self.labels = []
+        with db.session.begin_nested():
+            for i in range(1, self.N+1):
+                label, is_new = self.project.create_label(
+                    commit=False,
+                    name=f"Label{i}",
+                    reference=f"Label{i}",
+                )
+                assert is_new, "labels should be new"
+                label.flush()
+                self.labels.append(label)
+
+    def test_remove_label_without_flag(self):
+        self.assertEqual(self.N, self.project.labels.count())
+
+        self.post(self.url(self.labels[0].id), status_code=400)
+        self.assertEqual(self.N, self.project.labels.count())
+
+        self.post(self.url(self.labels[0].id), status_code=400, json=dict(remove=False))
+        self.assertEqual(self.N, self.project.labels.count())
+
+
+    def test_remove_label(self):
+        self.assertEqual(self.N, self.project.labels.count())
+
+        self.post(self.url(self.labels[0].id), json=dict(remove=True))
+        self.assertEqual(self.N-1, self.project.labels.count())
+
+    def test_remove_non_existing_label(self):
+        self.assertEqual(self.N, self.project.labels.count())
+
+        self.post(self.url(424242), json=dict(remove=True), status_code=404)
+        self.assertEqual(self.N, self.project.labels.count())
+
+
+class LabelTreeRemovalTests(_BaseLabelTests):
+
+    def url(self, label_id):
+        return url_for("remove_label",
+            project_id=self.project.id, label_id=label_id)
+
+    def setupModels(self):
+        super().setupModels()
+        self.n_roots = 5
+        self.leafs_per_root = 3
+        self.n_leafs = self.n_roots * self.leafs_per_root
+        self.N = self.n_roots + self.n_leafs
+
+        with db.session.begin_nested():
+            for i in range(1, self.n_roots+1):
+                root_label, is_new = self.project.create_label(
+                    commit=False,
+                    name=f"Label{i}",
+                    reference=f"Label{i}",
+                )
+                root_label.flush()
+
+                for j in range(1, self.leafs_per_root+1):
+                    label, is_new = self.project.create_label(
+                        commit=False,
+                        name=f"Label{i}_{j}",
+                        reference=f"Label{i}_{j}",
+                        parent=root_label
+                    )
+                    label.flush()
+
+
+    @property
+    def root_labels(self):
+        return self.project.labels.filter(Label.parent_id == None)
+
+    @property
+    def leaf_labels(self):
+        return self.project.labels.filter(Label.parent_id != None)
+
+    def test_remove_root_label(self):
+        self.assertEqual(self.N, self.project.labels.count())
+        self.assertEqual(self.n_roots, self.root_labels.count())
+        self.assertEqual(self.n_leafs, self.leaf_labels.count())
+
+        label = self.root_labels.first()
+
+        self.post(self.url(label.id), json=dict(remove=True))
+
+        self.assertEqual(self.N-1, self.project.labels.count())
+        self.assertEqual(self.n_roots-1+self.leafs_per_root, self.root_labels.count())
+        self.assertEqual(self.n_leafs-self.leafs_per_root, self.leaf_labels.count())
+
+    def test_remove_leaf_label(self):
+        self.assertEqual(self.N, self.project.labels.count())
+        self.assertEqual(self.n_roots, self.root_labels.count())
+        self.assertEqual(self.n_leafs, self.leaf_labels.count())
+
+        label = self.leaf_labels.first()
+        self.post(self.url(label.id), json=dict(remove=True))
+
+        self.assertEqual(self.N-1, self.project.labels.count())
+        self.assertEqual(self.n_roots, self.root_labels.count())
+        self.assertEqual(self.n_leafs-1, self.leaf_labels.count())
+
+    def test_remove_intermediate_label(self):
+        self.assertEqual(self.N, self.project.labels.count())
+        self.assertEqual(self.n_roots, self.root_labels.count())
+        self.assertEqual(self.n_leafs, self.leaf_labels.count())
+
+        label = self.leaf_labels.first()
+        label_parent = label.parent
+
+        new_label, is_new = self.project.create_label(
+            commit=False,
+            name="New Label",
+            reference="new_ref",
+            parent=label,
+        )
+
+        self.assertTrue(is_new)
+        self.assertEqual(label.id, new_label.parent_id)
+        self.assertEqual(self.n_roots, self.root_labels.count())
+        self.assertEqual(self.n_leafs + 1, self.leaf_labels.count())
+
+        self.post(self.url(label.id), json=dict(remove=True))
+
+        self.assertEqual(label_parent.id, new_label.parent_id)
+        self.assertEqual(self.n_roots, self.root_labels.count())
+        self.assertEqual(self.n_leafs, self.leaf_labels.count())
+
+class LabelEditTests(_BaseLabelTests):
+
+
+    def test_edit_name(self):
+        self.assertEqual(0, self.project.labels.count())
+        label = self.project.create_label(name="Label", reference="label")
+        self.assertEqual(1, self.project.labels.count())
+        label = self.project.labels.first()
+        self.assertEqual("Label", label.name)
+
+        self.post(self.url(label.id), json=dict(), status_code=400)
+
+        name = "Another name"
+        self.post(self.url(label.id), json=dict(name=name))
+
+        label = self.project.labels.first()
+        self.assertEqual(name, label.name)
+
+    def test_edit_name(self):
+        self.assertEqual(0, self.project.labels.count())
+        label = self.project.create_label(name="Label", reference="label")
+        self.assertEqual(1, self.project.labels.count())
+        label = self.project.labels.first()
+        self.assertEqual("Label", label.name)
+
+
+        url = lambda label_id: url_for("edit_label_name",
+            project_id=self.project.id, label_id=label_id)
+
+        self.post(url(label.id), json=dict(), status_code=400)
+
+        name = "Another name"
+        self.post(url(4242), json=dict(name=name), status_code=404)
+        self.post(url(label.id), json=dict(name=name))
+
+        label = self.project.labels.first()
+        self.assertEqual(name, label.name)
+
+    def test_edit_parent(self):
+        self.assertEqual(0, self.project.labels.count())
+        label, _ = self.project.create_label(name="Label", reference="label")
+        parent, _ = self.project.create_label(name="Label2", reference="label2")
+        self.assertEqual(2, self.project.labels.count())
+
+        label = Label.query.get(label.id)
+        self.assertIsNone(label.parent_id)
+
+
+        url = lambda label_id: url_for("edit_label_parent",
+            project_id=self.project.id, label_id=label_id)
+
+        self.post(url(label.id), json=dict(), status_code=400)
+
+        self.post(url(4242), json=dict(parent=parent.id), status_code=404)
+        self.post(url(label.id), json=dict(parent=parent.id))
+
+        label = Label.query.get(label.id)
+        self.assertEqual(parent.id, label.parent_id)

tests/client/pipeline_tests.py

@@ -0,0 +1,212 @@
+import uuid
+
+from flask import url_for
+from pathlib import Path
+
+from pycs.database.Model import Model
+from pycs.database.LabelProvider import LabelProvider
+from pycs.database.Project import Project
+
+from tests.base import BaseTestCase
+from tests.base import pаtch_tpool_execute
+
+
+class _BasePipelineTests(BaseTestCase):
+
+    def setupModels(self):
+        super().setupModels()
+
+        Model.discover("tests/client/test_models")
+
+        self.model = Model.query.one()
+
+        self.project = self.new_project()
+
+        root = Path(self.project.root_folder)
+        data_root = Path(self.project.data_folder)
+
+        for folder in [data_root, root / "temp"]:
+            folder.mkdir(exist_ok=True, parents=True)
+
+        file_uuid = str(uuid.uuid1())
+        self.file, is_new = self.project.add_file(
+            uuid=file_uuid,
+            file_type="image",
+            name="name",
+            filename="image",
+            extension=".jpg",
+            size=32*1024,
+        )
+
+        self.assertTrue(is_new)
+        with open(self.file.absolute_path, "wb") as f:
+            f.write(b"some content")
+
+    def new_project(self, **kwargs):
+
+        return Project.new(
+            name="test_project",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+            **kwargs
+        )
+
+    def tearDown(self):
+        self.wait_for_bg_jobs(raise_errors=False)
+        self.project.delete()
+        super().tearDown()
+
+class ModelPipelineTests(_BasePipelineTests):
+
+    _sleep_time: float = .2
+
+    def test_predict_file_busy(self):
+        url = url_for("predict_file", file_id=self.file.id)
+
+        self.post(url, json=dict(predict=True))
+        self.post(url, json=dict(predict=True), status_code=400)
+
+    def test_predict_file_errors(self):
+        self.post(url_for("predict_file", file_id=4242),
+            status_code=404)
+
+        url = url_for("predict_file", file_id=self.file.id)
+
+        for data in [None, dict(), dict(predict=False)]:
+            self.post(url, status_code=400, json=data)
+
+    def test_predict_file(self):
+        url = url_for("predict_file", file_id=self.file.id)
+        self.assertEqual(0, self.file.results.count())
+        self.post(url, json=dict(predict=True))
+        self.wait_for_bg_jobs()
+        self.assertEqual(1, self.file.results.count())
+
+    def test_predict_file_multiple_times(self):
+        url = url_for("predict_file", file_id=self.file.id)
+
+        self.assertEqual(0, self.file.results.count())
+
+        self.post(url, json=dict(predict=True))
+        self.wait_for_bg_jobs()
+        self.assertEqual(1, self.file.results.count())
+
+        self.post(url, json=dict(predict=True))
+        self.wait_for_bg_jobs()
+        self.assertEqual(1, self.file.results.count())
+
+    def test_predict_model_errors(self):
+        self.post(url_for("predict_model", project_id=4242),
+            status_code=404)
+
+        url = url_for("predict_model", project_id=self.project.id)
+
+        for data in [None, dict(), dict(predict=False), dict(predict=True), dict(predict="not new or all")]:
+            self.post(url, status_code=400, json=data)
+
+    def test_predict_model_busy(self):
+        url = url_for("predict_model", project_id=self.project.id)
+        self.post(url, json=dict(predict="new"))
+        self.post(url, json=dict(predict="new"), status_code=400)
+
+
+    def test_predict_model_for_new(self):
+
+        url = url_for("predict_model", project_id=self.project.id)
+        self.post(url, json=dict(predict="new"))
+
+
+    def test_predict_model_for_all(self):
+        url = url_for("predict_model", project_id=self.project.id)
+        self.post(url, json=dict(predict="all"))
+
+
+
+    def test_model_fit_errors(self):
+
+        self.post(url_for("fit_model", project_id=4242),
+            status_code=404)
+
+        url = url_for("fit_model", project_id=self.project.id)
+
+        for data in [None, dict(), dict(fit=False)]:
+            self.post(url, status_code=400, json=data)
+
+    def test_model_fit_busy(self):
+        url = url_for("fit_model", project_id=self.project.id)
+        self.post(url, json=dict(fit=True))
+        self.post(url, json=dict(fit=True), status_code=400)
+
+    def test_model_fit(self):
+        url = url_for("fit_model", project_id=self.project.id)
+        self.post(url, json=dict(fit=True))
+
+
+class LabelProviderPipelineTests:
+
+    def new_project(self):
+        LabelProvider.discover("tests/client/test_labels")
+        return super().new_project(label_provider=self.label_provider)
+
+    @property
+    def url(self):
+        return url_for("execute_label_provider", project_id=self.project.id)
+
+    def test_label_provider_errors(self):
+        url = url_for("execute_label_provider", project_id=4242)
+        self.post(url, status_code=404)
+
+        for data in [None, dict(), dict(execute=False)]:
+            self.post(self.url, json=data, status_code=400)
+
+        self.project.label_provider = None
+        self.project.commit()
+        self.post(self.url, json=dict(execute=True), status_code=400)
+
+    def test_label_provider_busy(self):
+        self.post(self.url, json=dict(execute=True))
+        self.post(self.url, json=dict(execute=True), status_code=400)
+
+    def test_label_loading(self):
+        self.post(self.url, json=dict(execute=True))
+        self.wait_for_bg_jobs()
+
+        self.assertEqual(self.n_labels, self.project.labels.count())
+
+    def test_label_loading_multiple(self):
+
+        for i in range(3):
+            self.post(self.url, json=dict(execute=True))
+            self.wait_for_bg_jobs()
+
+            self.assertEqual(self.n_labels, self.project.labels.count())
+
+class SimpleLabelProviderPipelineTests(LabelProviderPipelineTests, _BasePipelineTests):
+
+    @property
+    def n_labels(self):
+        return 10
+
+    @property
+    def label_provider(self):
+        name_filter = LabelProvider.name.contains("Simple")
+        return LabelProvider.query.filter(name_filter).one()
+
+
+class HierarchicalLabelProviderPipelineTests(LabelProviderPipelineTests, _BasePipelineTests):
+
+    @property
+    def n_labels(self):
+        leafs = 10 * 3 * 3
+        intermediate = 10 * 3
+        roots = 10
+
+        return roots + intermediate + leafs
+
+    @property
+    def label_provider(self):
+        name_filter = LabelProvider.name.contains("Hierarchical")
+        return LabelProvider.query.filter(name_filter).one()

tests/client/project_tests.py

@@ -0,0 +1,519 @@
+import uuid
+
+from flask import url_for
+
+from pycs.database.Collection import Collection
+from pycs.database.File import File
+from pycs.database.Label import Label
+from pycs.database.Model import Model
+from pycs.database.Project import Project
+from pycs.database.Result import Result
+from pycs.interfaces.MediaFile import MediaFile
+
+from tests.base import BaseTestCase
+
+
+class _BaseProjectTests(BaseTestCase):
+
+    def setupModels(self):
+
+        model = Model.new(
+            commit=False,
+            name="TestModel",
+            description="Model for a test case",
+            root_folder="models/fixed_model",
+        )
+        model.supports = ["labeled-image"]
+        model.flush()
+
+        self.model = model
+
+    def tearDown(self):
+        self.model.delete()
+        super().tearDown()
+
+class ProjectCreationTests(_BaseProjectTests):
+
+    def setUp(self):
+        super().setUp()
+        self.url = url_for("create_project")
+
+    def test_project_creation_without_name(self):
+        self.assertEqual(0, Project.query.count())
+
+        self.post(self.url, json=dict(
+                # name="Some Project",
+                description="Some description",
+                model=self.model.id,
+                label=None,
+                external=None,
+            ),
+            status_code=400,
+        )
+
+        self.assertEqual(0, Project.query.count())
+
+    def test_project_creation_without_description(self):
+        self.assertEqual(0, Project.query.count())
+
+        self.post(self.url, json=dict(
+                name="Some Project",
+                # description="Some description",
+                model=self.model.id,
+                label=None,
+                external=None,
+            ),
+            status_code=400,
+        )
+
+        self.assertEqual(0, Project.query.count())
+
+    def test_project_creation(self):
+        self.assertEqual(0, Project.query.count())
+
+        self.post(self.url, json=dict(
+                name="Some Project",
+                description="Some description",
+                model=self.model.id,
+                label=None,
+                external=None,
+            )
+        )
+        self.assertEqual(1, Project.query.count())
+
+        project = Project.query.first()
+
+        self.assertIsNotNone(project)
+        self.assertIsNotNone(project.model)
+        self.assertNotEqual(project.model.id, self.model.id,
+            "The model shoud be copied")
+        self.assertIsNone(project.label_provider)
+
+
+class ProjectDeletionTests(_BaseProjectTests):
+
+    def setupModels(self):
+        super().setupModels()
+        self.project = Project.new(
+            name="test_project",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",)
+
+    @property
+    def url(self):
+        return url_for("remove_project",
+                       project_id=self.project.id)
+
+
+    def test_project_deletion(self):
+
+        self.assertEqual(1, Project.query.count())
+
+        self.post(self.url, json=dict(remove=True))
+        self.assertEqual(0, Project.query.count())
+
+    def test_project_deletion_without_flag(self):
+
+        self.assertEqual(1, Project.query.count())
+
+        self.post(self.url, json=dict(), status_code=400)
+        self.assertEqual(1, Project.query.count())
+
+        self.post(self.url, json=dict(remove=False), status_code=400)
+        self.assertEqual(1, Project.query.count())
+
+    def test_project_deletion_with_labels(self):
+
+        self.assertEqual(1, Project.query.count())
+        self.assertEqual(0, Label.query.count())
+
+        for i in range(1, 11):
+            self.project.create_label(name=f"Label_{i}")
+
+        self.assertEqual(10, Label.query.count())
+
+        self.post(self.url, json=dict(remove=True))
+
+        self.assertEqual(0, Project.query.count())
+        self.assertEqual(0, Label.query.count())
+
+
+class ProjectListTests(_BaseProjectTests):
+
+
+    def test_list_projects(self):
+        self.assertEqual(0, Project.query.count())
+
+        for i in range(1, 11):
+            Project.new(
+                name=f"TestProject{i}",
+                description=f"Project for a test case #{i}",
+                model=self.model,
+                root_folder=f"project_folder{i}",
+                external_data=False,
+                data_folder=f"project_folder{i}/data",
+            )
+
+        self.assertEqual(10, Project.query.count())
+
+        response = self.get(url_for("list_projects"))
+
+        self.assertTrue(response.is_json)
+        content = response.json
+
+        self.assertEqual(10, len(content))
+
+        for entry in content:
+            project = Project.query.get(entry["id"])
+            self.assertIsNotNone(project)
+            self.assertDictEqual(entry, project.serialize())
+
+    def test_list_project_collections(self):
+        project = Project.new(
+            name="TestProject",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+        )
+
+
+        self.assertEqual(0, Collection.query.count())
+        for i in range(1, 11):
+            project.create_collection(
+                reference=f"collection_{i}",
+                name=f"Some collection {i}",
+                description=f"A description {i}",
+                position=i,
+                autoselect=i == 1
+            )
+        self.assertEqual(10, Collection.query.count())
+
+        response = self.get(url_for("list_collections",
+            project_id=project.id))
+
+        self.assertTrue(response.is_json)
+        content = response.json
+
+        self.assertEqual(10, len(content))
+
+        for entry in content:
+            collection = Collection.query.get(entry["id"])
+            self.assertIsNotNone(collection)
+            self.assertDictEqual(entry, collection.serialize())
+
+    def test_list_all_files(self):
+        project = Project.new(
+            name="TestProject",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+        )
+
+
+        self.assertEqual(0, File.query.count())
+        files = []
+        for i in range(1, 11):
+            file_uuid = str(uuid.uuid1())
+            file, is_new = project.add_file(
+                uuid=file_uuid,
+                file_type="image",
+                name=f"name{i}",
+                filename=f"image_{i:03d}",
+                extension=".jpg",
+                size=32*1024,
+            )
+            self.assertTrue(is_new)
+            files.append(file)
+
+        self.assertEqual(10, File.query.count())
+
+        response = self.get(url_for("list_all_files",
+            project_id=project.id))
+
+        self.assertTrue(response.is_json)
+        _content = response.json
+        count = _content["count"]
+        content = _content["files"]
+
+        self.assertEqual(10, count)
+        self.assertEqual(10, len(content))
+
+        for file, entry in zip(files, content):
+            self.assertDictEqual(entry, file.serialize())
+
+    def test_list_some_files(self):
+        project = Project.new(
+            name="TestProject",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+        )
+
+
+        self.assertEqual(0, File.query.count())
+        files = []
+        for i in range(1, 11):
+            file_uuid = str(uuid.uuid1())
+            file, is_new = project.add_file(
+                uuid=file_uuid,
+                file_type="image",
+                name=f"name{i}",
+                filename=f"image_{i:03d}",
+                extension=".jpg",
+                size=32*1024,
+            )
+            self.assertTrue(is_new)
+            files.append(file)
+
+        self.assertEqual(10, File.query.count())
+
+        for start, length in [(0, 5), (0, 15), (5, 3), (5, 8)]:
+            response = self.get(url_for("list_files",
+                project_id=project.id,
+                start=start, length=length))
+
+            self.assertTrue(response.is_json)
+            _content = response.json
+            count = _content["count"]
+            content = _content["files"]
+
+            self.assertEqual(len(files), count)
+            self.assertEqual(min(len(files), start+length) - start, len(content))
+
+            for file, entry in zip(files[start:start+length], content):
+                self.assertDictEqual(entry, file.serialize())
+
+
+    def test_list_collection_files_of_non_existing_collection(self):
+
+        project = Project.new(
+            name="TestProject",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+        )
+
+        url = url_for("list_collection_files",
+                      project_id=project.id, collection_id=42,
+                      start=0, length=30)
+        self.get(url, status_code=404)
+
+
+    def test_list_collection_files(self):
+        project = Project.new(
+            name="TestProject",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",
+        )
+
+        self.assertEqual(1, Project.query.count())
+
+        collections = {}
+        for i in range(1, 3):
+            collection, is_new = project.create_collection(
+                reference=f"collection_{i}",
+                name=f"Some collection {i}",
+                description=f"A description {i}",
+                position=i,
+                autoselect=i == 1
+            )
+
+            self.assertTrue(is_new)
+
+            collection_files = []
+
+            for j in range(1, 4):
+                file_uuid = str(uuid.uuid1())
+                file, is_new = collection.add_file(
+                    uuid=file_uuid,
+                    file_type="image",
+                    name=f"col_{i}_name{j}",
+                    filename=f"col_{i}_image_{j:03d}",
+                    extension=".jpg",
+                    size=32*1024,
+                )
+                self.assertTrue(is_new)
+                collection_files.append(file)
+
+            collections[collection.id] = collection_files
+
+        files = []
+        for j in range(1, 4):
+            file_uuid = str(uuid.uuid1())
+            file, is_new = project.add_file(
+                uuid=file_uuid,
+                file_type="image",
+                name=f"name{j}",
+                filename=f"image_{j:03d}",
+                extension=".jpg",
+                size=32*1024,
+            )
+            self.assertTrue(is_new)
+            files.append(file)
+
+        collections[0] = files
+
+        self.assertEqual(2, Collection.query.filter(Collection.project_id==project.id).count())
+
+        self.assertEqual(6, File.query.filter(
+            File.project_id == project.id,
+            File.collection_id != None,
+        ).count())
+
+        self.assertEqual(3, File.query.filter(
+            File.project_id == project.id,
+            File.collection_id == None,
+        ).count())
+
+
+        for collection_id, files in collections.items():
+            for start, length in [(0, 5), (0, 15), (1, 3), (1, 8)]:
+                response = self.get(url_for("list_collection_files",
+                    project_id=project.id, collection_id=collection_id,
+                    start=start, length=length))
+
+                self.assertTrue(response.is_json)
+                _content = response.json
+                count = _content["count"]
+                content = _content["files"]
+
+                self.assertEqual(len(files), count)
+                self.assertEqual(min(len(files), start+length) - start, len(content))
+
+                for file, entry in zip(files[start:start+length], content):
+                    self.assertDictEqual(entry, file.serialize())
+
+
+    def test_list_project_results(self):
+        project = Project.new(
+            name="test_project",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",)
+
+        url = url_for("get_project_results", project_id=42)
+        self.get(url, status_code=404)
+
+        files = []
+        for i in range(1, 4):
+            file_uuid = str(uuid.uuid1())
+            file, is_new = project.add_file(
+                uuid=file_uuid,
+                file_type="image",
+                name=f"name{i}",
+                filename=f"image_{i:03d}",
+                extension=".jpg",
+                size=32*1024,
+            )
+            self.assertTrue(is_new)
+            files.append(file)
+
+            file.create_result(
+                origin="user",
+                result_type="bounding-box",
+                label=None,
+                data=dict(x=0, y=0, w=1, h=1)
+            )
+
+        self.assertEqual(3, File.query.count())
+        self.assertEqual(3, Result.query.count())
+
+
+        url = url_for("get_project_results", project_id=project.id)
+        response = self.get(url)
+
+        self.assertTrue(response.is_json)
+        content = response.json
+
+        self.assertTrue(project.files.count(), len(content))
+
+        media_files = {f.filename: MediaFile(f, None) for f in project.files.all()}
+
+        for entry in content:
+            media_file = media_files[entry["filename"]]
+
+            self.assertDictEqual(media_file.serialize(), entry)
+
+    def test_get_project_model(self):
+
+        project = Project.new(
+            name="test_project",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",)
+
+        url = url_for("get_project_model", project_id=42)
+        self.get(url, status_code=404)
+
+        url = url_for("get_project_model", project_id=project.id)
+        response = self.get(url)
+        self.assertTrue(response.is_json)
+
+        content = response.json
+        self.assertDictEqual(project.model.serialize(), content)
+
+
+class ProjectEditTests(_BaseProjectTests):
+
+    def setupModels(self):
+        super().setupModels()
+        self.project = Project.new(
+            name="test_project",
+            description="Project for a test case",
+            model=self.model,
+            root_folder="project_folder",
+            external_data=False,
+            data_folder="project_folder/data",)
+
+
+    def test_name_edit(self):
+
+        name = "new_name"
+        url = url_for("edit_project_name", project_id=self.project.id)
+        self.post(url, json=dict(name=name))
+
+        self.assertEqual(name, self.project.name)
+
+
+    def test_name_edit_missing_argument(self):
+
+        url = url_for("edit_project_name", project_id=self.project.id)
+        self.post(url, json=dict(), status_code=400)
+
+        self.assertEqual("test_project", self.project.name)
+
+
+    def test_description_edit(self):
+
+        description = "New description for the project"
+        url = url_for("edit_project_description", project_id=self.project.id)
+        self.post(url, json=dict(description=description))
+
+        self.assertEqual(description, self.project.description)
+
+
+    def test_description_edit_missing_argument(self):
+
+        url = url_for("edit_project_description", project_id=self.project.id)
+        self.post(url, json=dict(), status_code=400)
+
+        self.assertEqual("Project for a test case", self.project.description)
+