6
0
Răsfoiți Sursa

updated docker image creation and the docker-compose configuration

Dimitri Korsch 3 ani în urmă
părinte
comite
a035f99629

+ 4 - 0
.dockerignore

@@ -2,6 +2,10 @@ env/
 venv/
 venv/
 webui/node_modules/
 webui/node_modules/
 projects/
 projects/
+htmlcov/
+labels/
+models/
+tests/
 
 
 *.sqlite
 *.sqlite
 *.sqlite-journal
 *.sqlite-journal

+ 2 - 0
.gitignore

@@ -35,6 +35,8 @@ htmlcov/
 
 
 # projects and models
 # projects and models
 /projects/
 /projects/
+/db/
+/external_data/
 /models/
 /models/
 /labels/
 /labels/
 dist/
 dist/

+ 31 - 27
Dockerfile

@@ -4,46 +4,50 @@
 FROM node:14 AS builder
 FROM node:14 AS builder
 
 
 # copy files
 # copy files
-COPY models/        /pycs/models/
+COPY       webui/       /pycs/webui_dev/
-COPY labels/        /pycs/labels/
+WORKDIR    /pycs/webui_dev
-COPY pycs/          /pycs/pycs/
-COPY webui/         /pycs/webui_dev/
-COPY app.py         /pycs/
-COPY settings.json  /pycs/
 
 
 # install packages
 # install packages
-RUN cd /pycs/webui_dev/  \
+RUN npx browserslist@latest --update-db
- && npm install
+RUN npm install
 
 
 # build webui
 # build webui
-RUN cd /pycs/webui_dev/  \
+RUN npm run build
- && npm run build
-
-# move dist folder to webui
-RUN cd /pycs/                  \
- && mv webui_dev/dist/ webui/  \
- && rm -rf webui_dev/
-
 
 
 ########################################
 ########################################
 # build pycs image                     #
 # build pycs image                     #
 ########################################
 ########################################
-FROM python:3.9
+FROM python:3.9 AS pycs
+
+# get the user and group. if not defined, fallback to root
+ARG UID=root
+ARG GID=root
 
 
-# copy from builder
+# copy backend files
-COPY --from=builder /pycs/ /pycs/
+COPY pycs/              /pycs/pycs/
+COPY migrations/        /pycs/migrations/
+COPY app.py             /pycs/
+COPY requirements.txt   /pycs/
+
+WORKDIR    /pycs/
+
+RUN mkdir projects db
+RUN chown ${UID}:${GID} projects db
+
+# copy UI from builder
+COPY --from=builder /pycs/webui_dev/dist /pycs/webui
 
 
 # install required libraries for opencv
 # install required libraries for opencv
 RUN apt-get update \
 RUN apt-get update \
  && apt-get install -y libgl1-mesa-glx ffmpeg libsm6 libxext6
  && apt-get install -y libgl1-mesa-glx ffmpeg libsm6 libxext6
 
 
-# create venv and install dependencies
+# install dependencies
-RUN cd /pycs/                                                 \
+RUN python -m pip install --upgrade pip
-  && python -m venv venv                                      \
+RUN pip install numpy
-  && ./venv/bin/pip install numpy opencv-python Pillow scipy  \
+RUN pip install -r /pycs/requirements.txt
-  && ./venv/bin/pip install eventlet flask python-socketio
 
 
 # configure start
 # configure start
-EXPOSE     5000
+USER      ${UID}:${GID}
-WORKDIR    /pycs/
+EXPOSE    5000
-ENTRYPOINT ["./venv/bin/python", "app.py"]
+
+#ENTRYPOINT ["python", "app.py"]

+ 17 - 0
Makefile

@@ -1,3 +1,10 @@
+
+UID := $(shell id -u)
+GID := $(shell id -g)
+
+export UID
+export GID
+
 run:
 run:
 	python app.py
 	python app.py
 
 
@@ -18,3 +25,13 @@ run_coverage:
 
 
 run_pylint:
 run_pylint:
 	pylint --rcfile=.pylintrc app.py pycs
 	pylint --rcfile=.pylintrc app.py pycs
+
+build_docker:
+	docker build . \
+		--tag pycs:latest \
+		--build-arg UID=${UID} \
+		--build-arg GID=${GID}
+
+run_docker: build_docker
+	@mkdir -p projects db
+	docker-compose up -d

+ 28 - 3
docker-compose.yml

@@ -2,7 +2,32 @@ version: '3'
 
 
 services:
 services:
   web:
   web:
-    build: .
+    user: ${UID:-0}:${GID:-0}
-    command: venv/bin/python app.py
+    image: pycs:latest
+    container_name: pycs
+
+    command: python app.py
+
     ports:
     ports:
-      - "5000:5000"
+      - ${PORT:-5000}:5000
+
+    volumes:
+      - ./models:/pycs/models
+      - ./labels:/pycs/labels
+      - ./projects:/pycs/projects
+      - ./db:/pycs/db
+      - ./settings.json:/pycs/settings.json
+      - ./external_data:/data
+
+    depends_on:
+      - migration
+
+  migration:
+    user: ${UID:-0}:${GID:-0}
+    image: pycs:latest
+    container_name: pycs_migrator
+    command: flask db upgrade
+
+    volumes:
+      - ./db:/pycs/db
+      - ./settings.json:/pycs/settings.json

+ 6 - 6
models/moth_scanner/scanner/classifier.py

@@ -18,15 +18,17 @@ class Classifier(object):
 
 
         model_type = config.model_type
         model_type = config.model_type
         n_classes = config.n_classes
         n_classes = config.n_classes
-        weights = Path(root, config.weights).resolve()
+        weights = Path(root, config.weights)
+
+        assert weights.exists(), \
+            f"could not find classifier weights: {weights}"
 
 
         self.input_size = config.input_size
         self.input_size = config.input_size
         self.backbone = ModelFactory.new(model_type)
         self.backbone = ModelFactory.new(model_type)
-        self.backbone.load_for_inference(weights,
+        self.backbone.load_for_inference(weights.resolve(),
                                          n_classes=n_classes,
                                          n_classes=n_classes,
                                          path="model/",
                                          path="model/",
-                                         strict=True,
+                                         strict=True)
-                                        )
 
 
         with open(Path(root, config.mapping)) as f:
         with open(Path(root, config.mapping)) as f:
             mapping = json.load(f)
             mapping = json.load(f)
@@ -43,7 +45,6 @@ class Classifier(object):
         # print(f"{'CenterCrop:': <14s} {im.shape=}")
         # print(f"{'CenterCrop:': <14s} {im.shape=}")
         return im
         return im
 
 
-
     def __call__(self, im: np.ndarray) -> int:
     def __call__(self, im: np.ndarray) -> int:
         assert im.ndim in (3, 4), \
         assert im.ndim in (3, 4), \
             "Classifier accepts only RGB images (3D input) or a batch of images (4D input)!"
             "Classifier accepts only RGB images (3D input) or a batch of images (4D input)!"
@@ -53,7 +54,6 @@ class Classifier(object):
             # HxWxC -> 1xHxWxC
             # HxWxC -> 1xHxWxC
             im = im[None]
             im = im[None]
 
 
-
         im = [self._transform(_im) for _im in im]
         im = [self._transform(_im) for _im in im]
         x = self.backbone.xp.array(im)
         x = self.backbone.xp.array(im)
         with chainer.using_config("train", False), chainer.no_backprop_mode():
         with chainer.using_config("train", False), chainer.no_backprop_mode():

+ 3 - 4
pycs/database/LabelProvider.py

@@ -1,5 +1,6 @@
 import json
 import json
 import re
 import re
+import typing as T
 
 
 from pathlib import Path
 from pathlib import Path
 
 
@@ -32,7 +33,7 @@ class LabelProvider(NamedBaseModel):
     )
     )
 
 
     @classmethod
     @classmethod
-    def discover(cls, root: Path):
+    def discover(cls, root: T.Union[Path, str]):
         """
         """
             searches for label providers under the given path
             searches for label providers under the given path
             and stores them in the database
             and stores them in the database
@@ -60,13 +61,11 @@ class LabelProvider(NamedBaseModel):
         """ returns the root folder as Path object """
         """ returns the root folder as Path object """
         return Path(self.root_folder)
         return Path(self.root_folder)
 
 
-
     @property
     @property
     def configuration_file_path(self) -> str:
     def configuration_file_path(self) -> str:
         """ return the configuration file as Path object """
         """ return the configuration file as Path object """
         return str(self.root / self.configuration_file)
         return str(self.root / self.configuration_file)
 
 
-
     def load(self) -> LabelProviderInterface:
     def load(self) -> LabelProviderInterface:
         """
         """
         load configuration.json and create an instance from the included code object
         load configuration.json and create an instance from the included code object
@@ -89,7 +88,7 @@ class LabelProvider(NamedBaseModel):
         return class_attr(self.root_folder, configuration)
         return class_attr(self.root_folder, configuration)
 
 
 
 
-def _find_files(root: str, config_regex=re.compile(r'^configuration(\d+)?\.json$')):
+def _find_files(root: T.Union[Path, str], config_regex=re.compile(r'^configuration(\d+)?\.json$')):
     """ generator for config files found under the given path """
     """ generator for config files found under the given path """
 
 
     for folder in Path(root).glob('*'):
     for folder in Path(root).glob('*'):

+ 4 - 3
pycs/database/Model.py

@@ -1,4 +1,5 @@
 import json
 import json
+import typing as T
 
 
 from pathlib import Path
 from pathlib import Path
 
 
@@ -6,6 +7,7 @@ from pycs import db
 from pycs.database.base import NamedBaseModel
 from pycs.database.base import NamedBaseModel
 from pycs.database.util import commit_on_return
 from pycs.database.util import commit_on_return
 
 
+
 class Model(NamedBaseModel):
 class Model(NamedBaseModel):
     """
     """
     DB model for ML models
     DB model for ML models
@@ -18,19 +20,18 @@ class Model(NamedBaseModel):
     # relationships to other models
     # relationships to other models
     projects = db.relationship("Project", backref="model", lazy="dynamic")
     projects = db.relationship("Project", backref="model", lazy="dynamic")
 
 
-    serialize_only = NamedBaseModel.serialize_only + (
+    serialize_only: tuple = NamedBaseModel.serialize_only + (
         "description",
         "description",
         "root_folder",
         "root_folder",
     )
     )
 
 
-
     def serialize(self):
     def serialize(self):
         result = super().serialize()
         result = super().serialize()
         result["supports"] = self.supports
         result["supports"] = self.supports
         return result
         return result
 
 
     @classmethod
     @classmethod
-    def discover(cls, root: Path, config_name: str = "configuration.json"):
+    def discover(cls, root: T.Union[Path, str], config_name: str = "configuration.json"):
         """
         """
             searches for models under the given path
             searches for models under the given path
             and stores them in the database
             and stores them in the database

+ 1 - 18
pycs/database/Project.py

@@ -20,7 +20,7 @@ class Project(NamedBaseModel):
     description = db.Column(db.String)
     description = db.Column(db.String)
 
 
     created = db.Column(db.DateTime, default=datetime.utcnow,
     created = db.Column(db.DateTime, default=datetime.utcnow,
-        index=True, nullable=False)
+                        index=True, nullable=False)
 
 
     model_id = db.Column(
     model_id = db.Column(
         db.Integer,
         db.Integer,
@@ -61,7 +61,6 @@ class Project(NamedBaseModel):
         passive_deletes=True,
         passive_deletes=True,
     )
     )
 
 
-
     serialize_only = NamedBaseModel.serialize_only + (
     serialize_only = NamedBaseModel.serialize_only + (
         "created",
         "created",
         "description",
         "description",
@@ -88,7 +87,6 @@ class Project(NamedBaseModel):
 
 
         return dump, model_dump
         return dump, model_dump
 
 
-
     def label(self, identifier: int) -> T.Optional[Label]:
     def label(self, identifier: int) -> T.Optional[Label]:
         """
         """
         get a label using its unique identifier
         get a label using its unique identifier
@@ -98,7 +96,6 @@ class Project(NamedBaseModel):
         """
         """
         return self.labels.filter(Label.id == identifier).one_or_none()
         return self.labels.filter(Label.id == identifier).one_or_none()
 
 
-
     def label_by_reference(self, reference: str) -> T.Optional[Label]:
     def label_by_reference(self, reference: str) -> T.Optional[Label]:
         """
         """
         get a label using its reference string
         get a label using its reference string
@@ -108,7 +105,6 @@ class Project(NamedBaseModel):
         """
         """
         return self.labels.filter(Label.reference == reference).one_or_none()
         return self.labels.filter(Label.reference == reference).one_or_none()
 
 
-
     def file(self, identifier: int) -> T.Optional[Label]:
     def file(self, identifier: int) -> T.Optional[Label]:
         """
         """
         get a file using its unique identifier
         get a file using its unique identifier
@@ -118,7 +114,6 @@ class Project(NamedBaseModel):
         """
         """
         return self.files.filter(File.id == identifier).one_or_none()
         return self.files.filter(File.id == identifier).one_or_none()
 
 
-
     def collection(self, identifier: int) -> T.Optional[Collection]:
     def collection(self, identifier: int) -> T.Optional[Collection]:
         """
         """
         get a collection using its unique identifier
         get a collection using its unique identifier
@@ -128,7 +123,6 @@ class Project(NamedBaseModel):
         """
         """
         return self.collections.filter(Collection.id == identifier).one_or_none()
         return self.collections.filter(Collection.id == identifier).one_or_none()
 
 
-
     def collection_by_reference(self, reference: str) -> T.Optional[Collection]:
     def collection_by_reference(self, reference: str) -> T.Optional[Collection]:
         """
         """
         get a collection using its unique identifier
         get a collection using its unique identifier
@@ -138,7 +132,6 @@ class Project(NamedBaseModel):
         """
         """
         return self.collections.filter(Collection.reference == reference).one_or_none()
         return self.collections.filter(Collection.reference == reference).one_or_none()
 
 
-
     @commit_on_return
     @commit_on_return
     def create_label(self, name: str,
     def create_label(self, name: str,
                      reference: str = None,
                      reference: str = None,
@@ -220,8 +213,6 @@ class Project(NamedBaseModel):
 
 
             unique_keys[key] = label
             unique_keys[key] = label
 
 
-
-
     # pylint: disable=too-many-arguments
     # pylint: disable=too-many-arguments
     @commit_on_return
     @commit_on_return
     def create_collection(self,
     def create_collection(self,
@@ -242,7 +233,6 @@ class Project(NamedBaseModel):
         :return: collection object, insert
         :return: collection object, insert
         """
         """
 
 
-
         collection, is_new = Collection.get_or_create(
         collection, is_new = Collection.get_or_create(
             project_id=self.id, reference=reference)
             project_id=self.id, reference=reference)
 
 
@@ -253,7 +243,6 @@ class Project(NamedBaseModel):
 
 
         return collection, is_new
         return collection, is_new
 
 
-
     # pylint: disable=too-many-arguments
     # pylint: disable=too-many-arguments
     @commit_on_return
     @commit_on_return
     def add_file(self,
     def add_file(self,
@@ -293,7 +282,6 @@ class Project(NamedBaseModel):
 
 
         return file, is_new
         return file, is_new
 
 
-
     def get_files(self, *filters, offset: int = 0, limit: int = -1) -> T.List[File]:
     def get_files(self, *filters, offset: int = 0, limit: int = -1) -> T.List[File]:
         """
         """
         get an iterator of files associated with this project
         get an iterator of files associated with this project
@@ -305,7 +293,6 @@ class Project(NamedBaseModel):
 
 
         return self.files.filter(*filters).order_by(File.id).offset(offset).limit(limit)
         return self.files.filter(*filters).order_by(File.id).offset(offset).limit(limit)
 
 
-
     def _files_without_results(self):
     def _files_without_results(self):
         """
         """
         get files without any results
         get files without any results
@@ -315,7 +302,6 @@ class Project(NamedBaseModel):
         # pylint: disable=no-member
         # pylint: disable=no-member
         return self.files.filter(~File.results.any())
         return self.files.filter(~File.results.any())
 
 
-
     def count_files_without_results(self) -> int:
     def count_files_without_results(self) -> int:
         """
         """
         count files without associated results
         count files without associated results
@@ -325,7 +311,6 @@ class Project(NamedBaseModel):
 
 
         return self._files_without_results().count()
         return self._files_without_results().count()
 
 
-
     def files_without_results(self) -> T.List[File]:
     def files_without_results(self) -> T.List[File]:
         """
         """
         get a list of files without associated results
         get a list of files without associated results
@@ -334,7 +319,6 @@ class Project(NamedBaseModel):
         """
         """
         return self._files_without_results().all()
         return self._files_without_results().all()
 
 
-
     def _files_without_collection(self, offset: int = 0, limit: int = -1):
     def _files_without_collection(self, offset: int = 0, limit: int = -1):
         """
         """
         get files without a collection
         get files without a collection
@@ -352,7 +336,6 @@ class Project(NamedBaseModel):
         """
         """
         return self._files_without_collection(offset=offset, limit=limit).all()
         return self._files_without_collection(offset=offset, limit=limit).all()
 
 
-
     def count_files_without_collection(self) -> int:
     def count_files_without_collection(self) -> int:
         """
         """
         count files associated with this project but without a collection
         count files associated with this project but without a collection

+ 3 - 9
pycs/database/base.py

@@ -8,6 +8,7 @@ from sqlalchemy_serializer import SerializerMixin
 from pycs import db
 from pycs import db
 from pycs.database.util import commit_on_return
 from pycs.database.util import commit_on_return
 
 
+
 class BaseModel(db.Model, SerializerMixin):
 class BaseModel(db.Model, SerializerMixin):
     """ Base model class """
     """ Base model class """
     __abstract__ = True
     __abstract__ = True
@@ -17,10 +18,9 @@ class BaseModel(db.Model, SerializerMixin):
     datetime_format = '%d. %b. %Y %H:%M:%S'
     datetime_format = '%d. %b. %Y %H:%M:%S'
     time_format = '%H:%M'
     time_format = '%H:%M'
 
 
-
     id = db.Column(db.Integer, primary_key=True)
     id = db.Column(db.Integer, primary_key=True)
 
 
-    serialize_only = ("id", "identifier")
+    serialize_only: tuple = ("id", "identifier")
 
 
     def identifier(self):
     def identifier(self):
         """ alias for id attribute """
         """ alias for id attribute """
@@ -31,12 +31,10 @@ class BaseModel(db.Model, SerializerMixin):
         content = ", ".join([f"{attr}={value}" for attr, value in attrs.items()])
         content = ", ".join([f"{attr}={value}" for attr, value in attrs.items()])
         return f"<{self.__class__.__name__}: {content}>"
         return f"<{self.__class__.__name__}: {content}>"
 
 
-
     def serialize(self) -> dict:
     def serialize(self) -> dict:
         """ default model serialization method """
         """ default model serialization method """
         return self.to_dict()
         return self.to_dict()
 
 
-
     @commit_on_return
     @commit_on_return
     def delete(self) -> dict:
     def delete(self) -> dict:
         """
         """
@@ -49,11 +47,9 @@ class BaseModel(db.Model, SerializerMixin):
 
 
         return dump
         return dump
 
 
-
     # do an alias
     # do an alias
     remove = delete
     remove = delete
 
 
-
     @classmethod
     @classmethod
     def new(cls, commit: bool = True, **kwargs):
     def new(cls, commit: bool = True, **kwargs):
         """ creates a new object. optionally commits the created object. """
         """ creates a new object. optionally commits the created object. """
@@ -79,7 +75,6 @@ class BaseModel(db.Model, SerializerMixin):
 
 
         return obj, is_new
         return obj, is_new
 
 
-
     @classmethod
     @classmethod
     def get_or_404(cls, obj_id: int) -> BaseModel:
     def get_or_404(cls, obj_id: int) -> BaseModel:
         """ get an object for the given id or raise 404 error if the object is not present """
         """ get an object for the given id or raise 404 error if the object is not present """
@@ -90,7 +85,6 @@ class BaseModel(db.Model, SerializerMixin):
 
 
         return obj
         return obj
 
 
-
     @staticmethod
     @staticmethod
     def commit():
     def commit():
         """ commit current session """
         """ commit current session """
@@ -108,7 +102,7 @@ class NamedBaseModel(BaseModel):
 
 
     name = db.Column(db.String, nullable=False)
     name = db.Column(db.String, nullable=False)
 
 
-    serialize_only = BaseModel.serialize_only + ("name",)
+    serialize_only: tuple = BaseModel.serialize_only + ("name",)
 
 
     @commit_on_return
     @commit_on_return
     def set_name(self, name: str):
     def set_name(self, name: str):

+ 55 - 35
pycs/frontend/WebServer.py

@@ -1,9 +1,11 @@
-import os
 import logging.config
 import logging.config
+import typing as T
 
 
 from glob import glob
 from glob import glob
+from pathlib import Path
 
 
 import eventlet
 import eventlet
+import munch
 import socketio
 import socketio
 
 
 from flask import send_from_directory
 from flask import send_from_directory
@@ -59,50 +61,28 @@ class WebServer:
     wrapper class for flask and socket.io which initializes most networking
     wrapper class for flask and socket.io which initializes most networking
     """
     """
 
 
-    def __init__(self, app, settings: dict, discovery: bool = True):
+    index: Path = Path.cwd() / 'webui' / 'index.html'
+
+    def __init__(self, app, settings: munch.Munch, discovery: bool = True):
 
 
         logging.config.dictConfig(settings.logging)
         logging.config.dictConfig(settings.logging)
-        is_production = os.path.exists('webui/index.html')
+        self.app = app
+        # set json encoder so database objects are serialized correctly
+        self.app.json_encoder = JSONEncoder
 
 
         # initialize web server
         # initialize web server
-        if is_production:
+        if self.is_production:
             app.logger.info('production build')
             app.logger.info('production build')
 
 
-            # find static files and folders
-            static_files = {}
-
-            for file_path in glob('webui/*'):
-                file_path = file_path.replace('\\', '/')
-                static_files[file_path[5:]] = file_path
-
-            # separately add svg files and set their correct mime type
-            for svg_path in glob('webui/img/*.svg'):
-                svg_path = svg_path.replace('\\', '/')
-                static_files[svg_path[5:]] = {'content_type': 'image/svg+xml', 'filename': svg_path}
-
-            # create service objects
-            if len(settings['allowedOrigins']) > 0:
-                origins = settings['allowedOrigins']
-                self.__sio = socketio.Server(cors_allowed_origins=origins, async_mode='eventlet')
-            else:
-                self.__sio = socketio.Server(async_mode='eventlet')
-
-            self.wsgi_app = socketio.WSGIApp(self.__sio, app, static_files=static_files)
-
             # overwrite root path to serve index.html
             # overwrite root path to serve index.html
-            @app.route('/', methods=['GET'])
+            @self.app.route('/', methods=['GET'])
             def index():
             def index():
                 # pylint: disable=unused-variable
                 # pylint: disable=unused-variable
-                return send_from_directory(os.path.join(os.getcwd(), 'webui'), 'index.html')
+                return send_from_directory(str(self.index.parent), self.index.name)
 
 
         else:
         else:
             app.logger.info('development build')
             app.logger.info('development build')
 
 
-            # create service objects
-            self.__sio = socketio.Server(cors_allowed_origins='*', async_mode='eventlet')
-            self.app = app
-            self.wsgi_app = socketio.WSGIApp(self.__sio, app)
-
             # set access control header to allow requests from Vue.js development server
             # set access control header to allow requests from Vue.js development server
             @self.app.after_request
             @self.app.after_request
             def after_request(response):
             def after_request(response):
@@ -110,8 +90,9 @@ class WebServer:
                 response.headers['Access-Control-Allow-Origin'] = '*'
                 response.headers['Access-Control-Allow-Origin'] = '*'
                 return response
                 return response
 
 
-        # set json encoder so database objects are serialized correctly
+        # create service objects
-        self.app.json_encoder = JSONEncoder
+        self.sio = socketio.Server(**self.sio_kwargs(settings.allowedOrigins))
+        self.wsgi_app = socketio.WSGIApp(self.sio, app, static_files=self.static_files)
 
 
         self.host = settings.host
         self.host = settings.host
         self.port = settings.port
         self.port = settings.port
@@ -119,7 +100,7 @@ class WebServer:
         # create notification manager
         # create notification manager
         self.jobs = JobRunner()
         self.jobs = JobRunner()
         self.pipelines = PipelineCache(self.jobs, settings.get("pipeline_cache_time"))
         self.pipelines = PipelineCache(self.jobs, settings.get("pipeline_cache_time"))
-        self.notifications = NotificationManager(self.__sio)
+        self.notifications = NotificationManager(self.sio)
 
 
         self.jobs.on_create(self.notifications.create_job)
         self.jobs.on_create(self.notifications.create_job)
         self.jobs.on_start(self.notifications.edit_job)
         self.jobs.on_start(self.notifications.edit_job)
@@ -133,6 +114,45 @@ class WebServer:
             Model.discover("models/")
             Model.discover("models/")
             LabelProvider.discover("labels/")
             LabelProvider.discover("labels/")
 
 
+    def sio_kwargs(self, allowed_origins) -> T.Dict[str, T.Union[str, list]]:
+        """keyword arguments for the socketio.Server depending on the mode"""
+        kwargs: T.Dict[str, T.Union[str, list]] = dict(async_mode="eventlet")
+
+        if self.is_production:
+            if isinstance(allowed_origins, list) and len(allowed_origins) > 0:
+                kwargs["cors_allowed_origins"] = allowed_origins
+        else:
+            kwargs["cors_allowed_origins"] = "*"
+
+        return kwargs
+
+    @property
+    def is_production(self) -> bool:
+        """property checking, whether the UI is built (production mode)
+           or served by npm serve (development mode)"""
+        return self.index.exists()
+
+    @property
+    def static_files(self) -> T.Optional[T.Dict[str, T.Union[str, dict]]]:
+        """returns a dictionary of static files (production mode)
+           or None (development mode)"""
+
+        if not self.is_production:
+            return None
+
+        # find static files and folders
+        static_files: T.Dict[str, T.Union[str, dict]] = {}
+
+        for file_path in glob('webui/*'):
+            file_path = file_path.replace('\\', '/')
+            static_files[file_path[5:]] = file_path
+
+        # separately add svg files and set their correct mime type
+        for svg_path in glob('webui/img/*.svg'):
+            svg_path = svg_path.replace('\\', '/')
+            static_files[svg_path[5:]] = {'content_type': 'image/svg+xml', 'filename': svg_path}
+
+        return static_files
 
 
     def define_routes(self):
     def define_routes(self):
         """ defines app routes """
         """ defines app routes """

+ 3 - 1
pycs/frontend/endpoints/additional/FolderInformation.py

@@ -5,6 +5,8 @@ from flask import jsonify
 from flask import request
 from flask import request
 from flask.views import View
 from flask.views import View
 
 
+from pycs.util.FileOperations import find_images
+
 
 
 class FolderInformation(View):
 class FolderInformation(View):
     """
     """
@@ -28,7 +30,7 @@ class FolderInformation(View):
 
 
         # count files
         # count files
         if result['exists']:
         if result['exists']:
-            result['count'] = len(os.listdir(folder))
+            result['count'] = len(find_images(folder))
 
 
         # send result
         # send result
         return jsonify(result)
         return jsonify(result)

+ 4 - 4
pycs/frontend/endpoints/projects/CreateProject.py

@@ -46,7 +46,6 @@ class CreateProject(View):
         if description is None:
         if description is None:
             abort(400, "description argument is missing!")
             abort(400, "description argument is missing!")
 
 
-
         model_id = int(data['model'])
         model_id = int(data['model'])
         model = Model.get_or_404(model_id)
         model = Model.get_or_404(model_id)
 
 
@@ -58,7 +57,7 @@ class CreateProject(View):
 
 
         # create project folder
         # create project folder
         project_folder = Path(settings.projects_folder, str(uuid.uuid1()))
         project_folder = Path(settings.projects_folder, str(uuid.uuid1()))
-        project_folder.mkdir()
+        project_folder.mkdir(parents=True)
 
 
         temp_folder = project_folder / 'temp'
         temp_folder = project_folder / 'temp'
         temp_folder.mkdir()
         temp_folder.mkdir()
@@ -89,8 +88,9 @@ class CreateProject(View):
             model.flush()
             model.flush()
 
 
             if not is_new:
             if not is_new:
-                abort(400, # pragma: no cover
+                # pragma: no cover
-                    f"Could not copy model! Model in \"{model_folder}\" already exists!")
+                abort(400,
+                      f"Could not copy model! Model in \"{model_folder}\" already exists!")
             project = Project.new(name=name,
             project = Project.new(name=name,
                                   description=description,
                                   description=description,
                                   model_id=model.id,
                                   model_id=model.id,

+ 13 - 11
pycs/frontend/endpoints/projects/ExecuteExternalStorage.py

@@ -12,6 +12,7 @@ from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
 from pycs.jobs.JobRunner import JobRunner
 from pycs.jobs.JobRunner import JobRunner
 from pycs.util.FileOperations import file_info
 from pycs.util.FileOperations import file_info
+from pycs.util.FileOperations import find_images
 
 
 
 
 class ExecuteExternalStorage(View):
 class ExecuteExternalStorage(View):
@@ -62,31 +63,32 @@ class ExecuteExternalStorage(View):
         # pylint: disable=invalid-name
         # pylint: disable=invalid-name
         # find lists the given data folder and prepares item dictionaries
         # find lists the given data folder and prepares item dictionaries
         def find(data_folder):
         def find(data_folder):
-            files = os.listdir(data_folder)
+            image_paths = find_images(data_folder)
-            length = len(files)
+            length = len(image_paths)
 
 
             elements = []
             elements = []
             current = 0
             current = 0
 
 
-            for file_name in files:
+            for file_path in image_paths:
-                file_path = os.path.join(data_folder, file_name)
-                if not os.path.isfile(file_path):
-                    continue
 
 
-                file_name, file_extension = os.path.splitext(file_name)
+                file_folder = str(file_path.parent)
-                file_size = os.path.getsize(file_path)
+                file_name = file_path.stem
+                file_extension = file_path.suffix
+                file_size = os.path.getsize(str(file_path))
 
 
                 try:
                 try:
-                    ftype, frames, fps = file_info(data_folder, file_name, file_extension)
+                    ftype, frames, fps = file_info(file_folder, file_name, file_extension)
                 except ValueError:
                 except ValueError:
                     continue
                     continue
 
 
+                file_uuid = str(uuid.uuid1())
                 file_attrs = dict(
                 file_attrs = dict(
-                    uuid=str(uuid.uuid1()),
+                    uuid=file_uuid,
                     file_type=ftype,
                     file_type=ftype,
                     name=file_name,
                     name=file_name,
                     extension=file_extension,
                     extension=file_extension,
                     size=file_size,
                     size=file_size,
+                    filename=file_path.relative_to(data_folder).with_name(file_name),
                     frames=frames,
                     frames=frames,
                     fps=fps)
                     fps=fps)
 
 
@@ -95,7 +97,7 @@ class ExecuteExternalStorage(View):
 
 
                 if len(elements) >= 200:
                 if len(elements) >= 200:
                     yield elements, current, length
                     yield elements, current, length
-                    elements = []
+                    elements.clear()
 
 
             if len(elements) > 0:
             if len(elements) > 0:
                 yield elements, current, length
                 yield elements, current, length

+ 19 - 0
pycs/util/FileOperations.py

@@ -2,6 +2,7 @@ import os
 import typing as T
 import typing as T
 
 
 from collections import namedtuple
 from collections import namedtuple
+from pathlib import Path
 
 
 import cv2
 import cv2
 
 
@@ -14,6 +15,7 @@ DEFAULT_JPEG_QUALITY = 80
 
 
 BoundingBox = namedtuple("BoundingBox", "x y w h")
 BoundingBox = namedtuple("BoundingBox", "x y w h")
 
 
+
 def file_info(data_folder: str, file_name: str, file_ext: str):
 def file_info(data_folder: str, file_name: str, file_ext: str):
     """
     """
     Receive file type, frame count and frames per second.
     Receive file type, frame count and frames per second.
@@ -235,3 +237,20 @@ def crop_image(file_path: str, target_path: str, box: BoundingBox) -> bool:
     # save to file
     # save to file
     cropped_image.save(target_path, quality=DEFAULT_JPEG_QUALITY)
     cropped_image.save(target_path, quality=DEFAULT_JPEG_QUALITY)
     return True
     return True
+
+
+def find_images(folder,
+                suffixes: T.Optional[T.List[str]] = None) -> T.List[Path]:
+    """ walk recursively the folder and find images """
+
+    suffixes = suffixes if suffixes is not None else [".jpg", ".jpeg", ".png"]
+    images: T.List[Path] = list()
+    for root, _, files in os.walk(folder):
+        for file in files:
+            fpath = Path(root, file)
+            if fpath.suffix.lower() not in suffixes:
+                continue
+
+            images.append(fpath)
+
+    return images

+ 1 - 1
settings.json

@@ -3,7 +3,7 @@
   "port": 5000,
   "port": 5000,
   "allowedOrigins": [],
   "allowedOrigins": [],
   "projects_folder": "projects",
   "projects_folder": "projects",
-  "database": "data2.sqlite3",
+  "database": "db/data.sqlite3",
   "pipeline_cache_time": 120,
   "pipeline_cache_time": 120,
 
 
   "logging": {
   "logging": {

+ 3 - 7
tests/client/__init__.py

@@ -20,7 +20,6 @@ class FolderInformationTest(BaseTestCase):
         self.assertTrue(response.is_json)
         self.assertTrue(response.is_json)
         self.assertDictEqual(content_should, response.json)
         self.assertDictEqual(content_should, response.json)
 
 
-
     def test_folder_information(self):
     def test_folder_information(self):
 
 
         url = url_for("folder_information")
         url = url_for("folder_information")
@@ -28,14 +27,12 @@ class FolderInformationTest(BaseTestCase):
 
 
         with tempfile.TemporaryDirectory() as folder:
         with tempfile.TemporaryDirectory() as folder:
 
 
-            self._check(url, "/not_existing/folder",
+            self._check(url, "/not_existing/folder", dict(exists=False))
-                dict(exists=False))
 
 
             for i in range(10):
             for i in range(10):
-                self._check(url, folder,
+                self._check(url, folder, dict(exists=True, count=i))
-                    dict(exists=True, count=i))
 
 
-                f = tempfile.NamedTemporaryFile(dir=folder, delete=False)
+                tempfile.NamedTemporaryFile(dir=folder, delete=False, suffix=".jpg")
 
 
 
 
 class ListModelsAndLabelProviders(BaseTestCase):
 class ListModelsAndLabelProviders(BaseTestCase):
@@ -75,7 +72,6 @@ class ListModelsAndLabelProviders(BaseTestCase):
             model = models[entry["id"]]
             model = models[entry["id"]]
             self.assertDictEqual(model.serialize(), entry)
             self.assertDictEqual(model.serialize(), entry)
 
 
-
     def test_list_label_providers(self):
     def test_list_label_providers(self):
         self.assertEqual(0, LabelProvider.query.count())
         self.assertEqual(0, LabelProvider.query.count())
         url = url_for("label_providers")
         url = url_for("label_providers")