Browse Source

fixed database related stuff

Dimitri Korsch 3 years ago
parent
commit
11397a1a7e
36 changed files with 180 additions and 161 deletions
  1. 8 2
      pycs/database/Database.py
  2. 28 20
      pycs/database/File.py
  3. 5 4
      pycs/database/Project.py
  4. 10 0
      pycs/database/Result.py
  5. 5 1
      pycs/database/base.py
  6. 1 1
      pycs/frontend/WebServer.py
  7. 1 1
      pycs/frontend/endpoints/data/GetResizedFile.py
  8. 2 3
      pycs/frontend/endpoints/data/RemoveFile.py
  9. 1 1
      pycs/frontend/endpoints/data/UploadFile.py
  10. 1 1
      pycs/frontend/endpoints/labels/CreateLabel.py
  11. 1 1
      pycs/frontend/endpoints/labels/EditLabelName.py
  12. 1 1
      pycs/frontend/endpoints/labels/EditLabelParent.py
  13. 2 2
      pycs/frontend/endpoints/labels/RemoveLabel.py
  14. 13 19
      pycs/frontend/endpoints/pipelines/FitModel.py
  15. 1 1
      pycs/frontend/endpoints/pipelines/PredictFile.py
  16. 19 6
      pycs/frontend/endpoints/pipelines/PredictModel.py
  17. 3 3
      pycs/frontend/endpoints/projects/CreateProject.py
  18. 1 1
      pycs/frontend/endpoints/projects/EditProjectDescription.py
  19. 1 1
      pycs/frontend/endpoints/projects/EditProjectName.py
  20. 1 1
      pycs/frontend/endpoints/projects/ExecuteExternalStorage.py
  21. 2 2
      pycs/frontend/endpoints/projects/ExecuteLabelProvider.py
  22. 2 2
      pycs/frontend/endpoints/projects/RemoveProject.py
  23. 1 1
      pycs/frontend/endpoints/results/ConfirmResult.py
  24. 3 3
      pycs/frontend/endpoints/results/CreateResult.py
  25. 1 1
      pycs/frontend/endpoints/results/EditResultData.py
  26. 1 1
      pycs/frontend/endpoints/results/EditResultLabel.py
  27. 3 7
      pycs/frontend/endpoints/results/GetProjectResults.py
  28. 1 4
      pycs/frontend/endpoints/results/GetResults.py
  29. 1 1
      pycs/frontend/endpoints/results/RemoveResult.py
  30. 2 2
      pycs/frontend/endpoints/results/ResetResults.py
  31. 39 46
      pycs/frontend/notifications/NotificationManager.py
  32. 6 8
      pycs/interfaces/MediaFile.py
  33. 3 3
      pycs/interfaces/MediaFileList.py
  34. 6 7
      pycs/interfaces/MediaStorage.py
  35. 3 2
      pycs/util/PipelineCache.py
  36. 1 1
      settings.json

+ 8 - 2
pycs/database/Database.py

@@ -33,11 +33,11 @@ class Database:
             LabelProvider.discover("labels/")
             LabelProvider.discover("labels/")
 
 
     def __enter__(self):
     def __enter__(self):
-        app.logger.warning("REMOVE ME!")
+        app.logger.warning("Database.__enter__(): REMOVE ME!")
         return self
         return self
 
 
     def __exit__(self, exc_type, exc_val, exc_tb):
     def __exit__(self, exc_type, exc_val, exc_tb):
-        app.logger.warning("REMOVE ME!")
+        app.logger.warning("Database.__exit__(): REMOVE ME!")
 
 
         if exc_type is None:
         if exc_type is None:
             db.session.commit()
             db.session.commit()
@@ -45,6 +45,12 @@ class Database:
             app.logger.info("Rolling back a transaction!")
             app.logger.info("Rolling back a transaction!")
             db.session.rollback()
             db.session.rollback()
 
 
+    def close(self):
+        app.logger.warning("Database.close(): REMOVE ME!")
+
+    def commit(self):
+        db.session.commit()
+
     def copy(self):
     def copy(self):
         return Database(self.path, initialization=False, discovery=False)
         return Database(self.path, initialization=False, discovery=False)
 
 

+ 28 - 20
pycs/database/File.py

@@ -8,6 +8,7 @@ from datetime import datetime
 
 
 from pycs import db
 from pycs import db
 from pycs.database.Result import Result
 from pycs.database.Result import Result
+from pycs.database.Label import Label
 from pycs.database.Collection import Collection
 from pycs.database.Collection import Collection
 from pycs.database.base import NamedBaseModel
 from pycs.database.base import NamedBaseModel
 
 
@@ -49,13 +50,14 @@ class File(NamedBaseModel):
 
 
     # relationships to other models
     # relationships to other models
     results = db.relationship("Result", backref="file", lazy="dynamic")
     results = db.relationship("Result", backref="file", lazy="dynamic")
-    serialize_rules = ('-results',)
 
 
-    def serialize(self):
-        result = super().serialize()
-        if result["data"] is not None:
-            result["data"] = json.loads(result["data"])
-        return result
+    serialize_only = (
+        "id", "name", "uuid",
+        "extension", "type", "size",
+        "created", "path", "frames",
+        "fps", "project_id",
+        "collection_id",
+    )
 
 
     def set_collection(self, id: T.Optional[int]):
     def set_collection(self, id: T.Optional[int]):
         """
         """
@@ -88,7 +90,9 @@ class File(NamedBaseModel):
 
 
         :return: another file or None
         :return: another file or None
         """
         """
-        return self.project.files.filter(*query)\
+        # return self.project.files.filter(*query)\
+        return File.query.filter_by(project_id=self.project_id)\
+            .filter(*query)\
             .order_by(File.id.desc())\
             .order_by(File.id.desc())\
             .first()
             .first()
 
 
@@ -98,8 +102,7 @@ class File(NamedBaseModel):
 
 
         :return: another file or None
         :return: another file or None
         """
         """
-        query = File.id > self.id
-        return self._get_another_file(*query)
+        return self._get_another_file(File.id > self.id)
 
 
 
 
     def previous(self) -> T.Optional[File]:
     def previous(self) -> T.Optional[File]:
@@ -108,8 +111,7 @@ class File(NamedBaseModel):
 
 
         :return: another file or None
         :return: another file or None
         """
         """
-        query = File.id < self.id
-        return self._get_another_file(*query)
+        return self._get_another_file(File.id < self.id)
 
 
 
 
     def next_in_collection(self) -> T.Optional[File]:
     def next_in_collection(self) -> T.Optional[File]:
@@ -118,8 +120,7 @@ class File(NamedBaseModel):
 
 
         :return: another file or None
         :return: another file or None
         """
         """
-        query = File.id > self.id, Collection.id == self.collection_id
-        return self._get_another_file(*query)
+        return self._get_another_file(File.id > self.id, Collection.id == self.collection_id)
 
 
 
 
     def previous_in_collection(self) -> T.Optional[File]:
     def previous_in_collection(self) -> T.Optional[File]:
@@ -128,8 +129,7 @@ class File(NamedBaseModel):
 
 
         :return: another file or None
         :return: another file or None
         """
         """
-        query = File.id < self.id, Collection.id == self.collection_id
-        return self._get_another_file(*query)
+        return self._get_another_file(File.id < self.id, Collection.id == self.collection_id)
 
 
 
 
     def result(self, id: int) -> T.Optional[Result]:
     def result(self, id: int) -> T.Optional[Result]:
@@ -139,19 +139,27 @@ class File(NamedBaseModel):
     def create_result(self, origin, result_type, label, data: T.Optional[dict] = None):
     def create_result(self, origin, result_type, label, data: T.Optional[dict] = None):
         data = data if data is None else json.dumps(data)
         data = data if data is None else json.dumps(data)
 
 
-        result = Result.new(commit=True,
-                            file=self,
+        result = Result.new(file=self,
                             origin=origin,
                             origin=origin,
                             type=result_type,
                             type=result_type,
-                            label=label,
                             data=data)
                             data=data)
-        return result
 
 
+        if label is not None:
+            if isinstance(label, int):
+                label = Label.query.get(label)
+
+            elif not isinstance(label, Label):
+                raise ValueError(f"wrong label type: {type(label)}")
+
+            result.label = label
+
+        self.commit()
+        return result
 
 
     def remove_results(self, origin='pipeline'):
     def remove_results(self, origin='pipeline'):
 
 
         results = Result.query.filter(Result.file == self, Result.origin == origin)
         results = Result.query.filter(Result.file == self, Result.origin == origin)
 
 
-        results.remove()
+        results.delete()
 
 
         return results
         return results

+ 5 - 4
pycs/database/Project.py

@@ -54,7 +54,7 @@ class Project(NamedBaseModel):
         :param identifier: unique identifier
         :param identifier: unique identifier
         :return: label
         :return: label
         """
         """
-        return self.labels.get(id)
+        return self.labels.filter_by(id=id).one_or_none()
 
 
     def file(self, id: int) -> T.Optional[Label]:
     def file(self, id: int) -> T.Optional[Label]:
         """
         """
@@ -63,7 +63,7 @@ class Project(NamedBaseModel):
         :param identifier: unique identifier
         :param identifier: unique identifier
         :return: file
         :return: file
         """
         """
-        return self.files.get(id)
+        return self.files.filter_by(id=id).one_or_none()
 
 
     def collection(self, id: int) -> T.Optional[Collection]:
     def collection(self, id: int) -> T.Optional[Collection]:
         """
         """
@@ -72,7 +72,7 @@ class Project(NamedBaseModel):
         :param identifier: unique identifier
         :param identifier: unique identifier
         :return: collection
         :return: collection
         """
         """
-        return self.collections.get(id)
+        return self.collections.filter_by(id=id).one_or_none()
 
 
     def collection_by_reference(self, reference: str) -> T.Optional[Collection]:
     def collection_by_reference(self, reference: str) -> T.Optional[Collection]:
         """
         """
@@ -81,7 +81,7 @@ class Project(NamedBaseModel):
         :param identifier: unique identifier
         :param identifier: unique identifier
         :return: collection
         :return: collection
         """
         """
-        return self.collections.filter_by(reference=reference).one()
+        return self.collections.filter_by(reference=reference).one_or_none()
 
 
     def create_label(self, name: str, reference: str = None,
     def create_label(self, name: str, reference: str = None,
                      parent_id: int = None) -> Tuple[Optional[Label], bool]:
                      parent_id: int = None) -> Tuple[Optional[Label], bool]:
@@ -159,6 +159,7 @@ class Project(NamedBaseModel):
         :return:
         :return:
         """
         """
         self.description = description
         self.description = description
+        self.commit()
 
 
     def count_files(self) -> int:
     def count_files(self) -> int:
         """
         """

+ 10 - 0
pycs/database/Result.py

@@ -23,6 +23,16 @@ class Result(BaseModel):
 
 
     data = db.Column(db.String)
     data = db.Column(db.String)
 
 
+    serialize_only = (
+        "id",
+        "file_id",
+        "origin",
+        "type",
+        "label_id",
+        "data",
+    )
+
+
     def serialize(self):
     def serialize(self):
         result = super().serialize()
         result = super().serialize()
         if result["data"] is not None:
         if result["data"] is not None:

+ 5 - 1
pycs/database/base.py

@@ -20,7 +20,7 @@ class BaseModel(db.Model, ModelSerializer):
 
 
     @property
     @property
     def identifier(self) -> int:
     def identifier(self) -> int:
-        app.logger.warning("REMOVE ME!")
+        app.logger.warning("BaseModel.identifier: REMOVE ME!")
         return self.id
         return self.id
 
 
 
 
@@ -30,6 +30,10 @@ class BaseModel(db.Model, ModelSerializer):
         result["identifier"] = result["id"]
         result["identifier"] = result["id"]
         return result
         return result
 
 
+    def __repr__(self):
+        attrs = self.serialize()
+        content = ", ".join([f"{attr}={value}" for attr, value in attrs.items()])
+        return f"<{self.__class__.__name__}: {content}>"
 
 
     def remove(self, commit=True) -> None:
     def remove(self, commit=True) -> None:
         """
         """

+ 1 - 1
pycs/frontend/WebServer.py

@@ -242,7 +242,7 @@ class WebServer:
         # results
         # results
         self.app.add_url_rule(
         self.app.add_url_rule(
             '/projects/<int:project_id>/results',
             '/projects/<int:project_id>/results',
-            view_func=GetProjectResults.as_view('get_project_results', self.db)
+            view_func=GetProjectResults.as_view('get_project_results')
         )
         )
         self.app.add_url_rule(
         self.app.add_url_rule(
             '/data/<int:file_id>/results',
             '/data/<int:file_id>/results',

+ 1 - 1
pycs/frontend/endpoints/data/GetResizedFile.py

@@ -27,7 +27,7 @@ class GetResizedFile(View):
         if file is None:
         if file is None:
             return abort(404)
             return abort(404)
 
 
-        project = file.project()
+        project = file.project
 
 
         # extract desired resolution
         # extract desired resolution
         resolution = re.split(r'[^0-9]', resolution)
         resolution = re.split(r'[^0-9]', resolution)

+ 2 - 3
pycs/frontend/endpoints/data/RemoveFile.py

@@ -34,8 +34,7 @@ class RemoveFile(View):
                 return abort(400)
                 return abort(400)
 
 
             # check if project uses an external data directory
             # check if project uses an external data directory
-            project = file.project()
-            if project.external_data:
+            if file.project.external_data:
                 return abort(400)
                 return abort(400)
 
 
             # remove file from database
             # remove file from database
@@ -47,5 +46,5 @@ class RemoveFile(View):
             # TODO remove temp files
             # TODO remove temp files
 
 
         # send notification
         # send notification
-        self.nm.remove_file(file)
+        self.nm.remove_file(file.id)
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/data/UploadFile.py

@@ -65,7 +65,7 @@ class UploadFile(View):
                                        self.file_size, self.file_id, frames, fps)
                                        self.file_size, self.file_id, frames, fps)
 
 
         # send update
         # send update
-        self.nm.create_file(file)
+        self.nm.create_file(file.id)
 
 
         # return default success response
         # return default success response
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/labels/CreateLabel.py

@@ -38,7 +38,7 @@ class CreateLabel(View):
             label, _ = project.create_label(name, parent_id=parent)
             label, _ = project.create_label(name, parent_id=parent)
 
 
         # send notification
         # send notification
-        self.nm.create_label(label)
+        self.nm.create_label(label.id)
 
 
         # return success response
         # return success response
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/labels/EditLabelName.py

@@ -40,7 +40,7 @@ class EditLabelName(View):
             label.set_name(data['name'])
             label.set_name(data['name'])
 
 
         # send notification
         # send notification
-        self.nm.edit_label(label)
+        self.nm.edit_label(label.id)
 
 
         # return success response
         # return success response
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/labels/EditLabelParent.py

@@ -40,7 +40,7 @@ class EditLabelParent(View):
             label.set_parent(data['parent'])
             label.set_parent(data['parent'])
 
 
         # send notification
         # send notification
-        self.nm.edit_label(label)
+        self.nm.edit_label(label.id)
 
 
         # return success response
         # return success response
         return make_response()
         return make_response()

+ 2 - 2
pycs/frontend/endpoints/labels/RemoveLabel.py

@@ -42,11 +42,11 @@ class RemoveLabel(View):
             # remove children's parent entry
             # remove children's parent entry
             for child in children:
             for child in children:
                 child.set_parent(None)
                 child.set_parent(None)
-                self.nm.edit_label(child)
+                self.nm.edit_label(child.id)
 
 
             # remove label
             # remove label
             label.remove()
             label.remove()
-            self.nm.remove_label(label)
+            self.nm.remove_label(label.id)
 
 
         # return success response
         # return success response
         return make_response()
         return make_response()

+ 13 - 19
pycs/frontend/endpoints/pipelines/FitModel.py

@@ -2,6 +2,7 @@ from flask import make_response, request, abort
 from flask.views import View
 from flask.views import View
 
 
 from pycs.database.Database import Database
 from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.interfaces.MediaStorage import MediaStorage
 from pycs.interfaces.MediaStorage import MediaStorage
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
 from pycs.jobs.JobRunner import JobRunner
 from pycs.jobs.JobRunner import JobRunner
@@ -29,7 +30,7 @@ class FitModel(View):
             return abort(400)
             return abort(400)
 
 
         # find project
         # find project
-        project = self.db.project(project_id)
+        project = Project.query.get(project_id)
         if project is None:
         if project is None:
             return abort(404)
             return abort(404)
 
 
@@ -47,25 +48,18 @@ class FitModel(View):
 
 
     @staticmethod
     @staticmethod
     def load_and_fit(database: Database, pipelines: PipelineCache, project_id: int):
     def load_and_fit(database: Database, pipelines: PipelineCache, project_id: int):
-        db = None
         pipeline = None
         pipeline = None
 
 
-        # create new database instance
-        try:
-            db = database.copy()
-            project = db.project(project_id)
-            model = project.model()
-            storage = MediaStorage(db, project_id)
+        project = Project.query.get(project_id)
+        model = project.model()
+        storage = MediaStorage(project_id)
 
 
-            # load pipeline
-            try:
-                pipeline = pipelines.load_from_root_folder(project, model.root_folder)
-                yield from pipeline.fit(storage)
-            except TypeError:
-                pass
-            finally:
-                if pipeline is not None:
-                    pipelines.free_instance(model.root_folder)
+        # load pipeline
+        try:
+            pipeline = pipelines.load_from_root_folder(project, model.root_folder)
+            yield from pipeline.fit(storage)
+        except TypeError:
+            pass
         finally:
         finally:
-            if db is not None:
-                db.close()
+            if pipeline is not None:
+                pipelines.free_instance(model.root_folder)

+ 1 - 1
pycs/frontend/endpoints/pipelines/PredictFile.py

@@ -38,7 +38,7 @@ class PredictFile(View):
             return abort(404)
             return abort(404)
 
 
         # get project and model
         # get project and model
-        project = file.project()
+        project = file.project
 
 
         # create job
         # create job
         try:
         try:

+ 19 - 6
pycs/frontend/endpoints/pipelines/PredictModel.py

@@ -3,7 +3,9 @@ from typing import Any
 from flask import make_response, request, abort
 from flask import make_response, request, abort
 from flask.views import View
 from flask.views import View
 
 
+from pycs import app
 from pycs.database.Database import Database
 from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.interfaces.MediaFile import MediaFile
 from pycs.interfaces.MediaFile import MediaFile
@@ -32,11 +34,11 @@ class PredictModel(View):
         # extract request data
         # extract request data
         data = request.get_json(force=True)
         data = request.get_json(force=True)
 
 
-        if 'predict' not in data or data['predict'] not in ['all', 'new']:
+        if data.get('predict') not in ['all', 'new']:
             return abort(400)
             return abort(400)
 
 
         # find project
         # find project
-        project = self.db.project(project_id)
+        project = Project.query.get(project_id)
         if project is None:
         if project is None:
             return abort(404)
             return abort(404)
 
 
@@ -66,9 +68,9 @@ class PredictModel(View):
         # create new database instance
         # create new database instance
         try:
         try:
             db = database.copy()
             db = database.copy()
-            project = db.project(project_id)
-            model = project.model()
-            storage = MediaStorage(db, project_id, notifications)
+            project = Project.query.get(project_id)
+            model = project.model
+            storage = MediaStorage(project_id, notifications)
 
 
             # create a list of MediaFile
             # create a list of MediaFile
             if isinstance(file_filter, str):
             if isinstance(file_filter, str):
@@ -79,7 +81,7 @@ class PredictModel(View):
                 else:
                 else:
                     length = project.count_files()
                     length = project.count_files()
                     files = map(lambda f: MediaFile(f, notifications),
                     files = map(lambda f: MediaFile(f, notifications),
-                                project.files())
+                                project.files.all())
             else:
             else:
                 files = map(lambda f: MediaFile(project.file(f.identifier), notifications),
                 files = map(lambda f: MediaFile(project.file(f.identifier), notifications),
                             file_filter)
                             file_filter)
@@ -103,9 +105,20 @@ class PredictModel(View):
                     yield index / length, notifications
                     yield index / length, notifications
 
 
                     index += 1
                     index += 1
+            except Exception as e:
+                import traceback
+                traceback.print_exc()
+                app.logger.warning(f"Pipeline Error #2: {e}")
+
             finally:
             finally:
                 if pipeline is not None:
                 if pipeline is not None:
                     pipelines.free_instance(model.root_folder)
                     pipelines.free_instance(model.root_folder)
+
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            app.logger.warning(f"Pipeline Error #1: {e}")
+
         finally:
         finally:
             if db is not None:
             if db is not None:
                 db.close()
                 db.close()

+ 3 - 3
pycs/frontend/endpoints/projects/CreateProject.py

@@ -96,7 +96,7 @@ class CreateProject(View):
         # load model and add collections to the project
         # load model and add collections to the project
         def load_model_and_get_collections():
         def load_model_and_get_collections():
             with closing(load_pipeline(model.root_folder)) as pipeline:
             with closing(load_pipeline(model.root_folder)) as pipeline:
-                return pipeline.collections()
+                return pipeline.collections.all()
 
 
         def add_collections_to_project(provided_collections):
         def add_collections_to_project(provided_collections):
             with self.db:
             with self.db:
@@ -119,8 +119,8 @@ class CreateProject(View):
             ExecuteExternalStorage.find_media_files(self.db, self.nm, self.jobs, created)
             ExecuteExternalStorage.find_media_files(self.db, self.nm, self.jobs, created)
 
 
         # fire event
         # fire event
-        self.nm.create_model(model)
-        self.nm.create_project(created)
+        self.nm.create_model(model.id)
+        self.nm.create_project(created.id)
 
 
         # return success response
         # return success response
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/projects/EditProjectDescription.py

@@ -33,6 +33,6 @@ class EditProjectDescription(View):
 
 
             # set description
             # set description
             project.set_description(data['description'])
             project.set_description(data['description'])
-            self.nm.edit_project(project)
+            self.nm.edit_project(project.id)
 
 
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/projects/EditProjectName.py

@@ -33,6 +33,6 @@ class EditProjectName(View):
 
 
             # set name
             # set name
             project.set_name(data['name'])
             project.set_name(data['name'])
-            self.nm.edit_project(project)
+            self.nm.edit_project(project.id)
 
 
             return make_response()
             return make_response()

+ 1 - 1
pycs/frontend/endpoints/projects/ExecuteExternalStorage.py

@@ -104,7 +104,7 @@ class ExecuteExternalStorage(View):
                                                     file_size, file_name, frames, fps)
                                                     file_size, file_name, frames, fps)
 
 
                     if insert:
                     if insert:
-                        nm.create_file(file)
+                        nm.create_file(file.id)
 
 
             return current / length
             return current / length
 
 

+ 2 - 2
pycs/frontend/endpoints/projects/ExecuteLabelProvider.py

@@ -80,9 +80,9 @@ class ExecuteLabelProvider(View):
                                                                  label['parent'])
                                                                  label['parent'])
 
 
                     if insert:
                     if insert:
-                        nm.create_label(created_label)
+                        nm.create_label(created_label.id)
                     else:
                     else:
-                        nm.edit_label(created_label)
+                        nm.edit_label(created_label.id)
 
 
         # run job with given functions
         # run job with given functions
         jobs.run(project,
         jobs.run(project,

+ 2 - 2
pycs/frontend/endpoints/projects/RemoveProject.py

@@ -45,7 +45,7 @@ class RemoveProject(View):
             shutil.rmtree(project.root_folder)
             shutil.rmtree(project.root_folder)
 
 
             # send update
             # send update
-            self.nm.remove_model(model)
-            self.nm.remove_project(project)
+            self.nm.remove_model(model.id)
+            self.nm.remove_project(project.id)
 
 
             return make_response()
             return make_response()

+ 1 - 1
pycs/frontend/endpoints/results/ConfirmResult.py

@@ -33,5 +33,5 @@ class ConfirmResult(View):
         with self.db:
         with self.db:
             result.set_origin('user')
             result.set_origin('user')
 
 
-        self.nm.edit_result(result)
+        self.nm.edit_result(result.id)
         return make_response()
         return make_response()

+ 3 - 3
pycs/frontend/endpoints/results/CreateResult.py

@@ -50,13 +50,13 @@ class CreateResult(View):
         # start transaction
         # start transaction
         with self.db:
         with self.db:
             # find full-image labels and remove them
             # find full-image labels and remove them
-            for result in file.results():
+            for result in file.results.all():
                 if result.type == 'labeled-image':
                 if result.type == 'labeled-image':
                     result.remove()
                     result.remove()
-                    self.nm.remove_result(result)
+                    self.nm.remove_result(result.id)
 
 
             # insert into database
             # insert into database
             result = file.create_result('user', rtype, label, data)
             result = file.create_result('user', rtype, label, data)
-            self.nm.create_result(result)
+            self.nm.create_result(result.id)
 
 
         return jsonify(result)
         return jsonify(result)

+ 1 - 1
pycs/frontend/endpoints/results/EditResultData.py

@@ -34,5 +34,5 @@ class EditResultData(View):
             result.set_data(data['data'])
             result.set_data(data['data'])
             result.set_origin('user')
             result.set_origin('user')
 
 
-        self.nm.edit_result(result)
+        self.nm.edit_result(result.id)
         return make_response()
         return make_response()

+ 1 - 1
pycs/frontend/endpoints/results/EditResultLabel.py

@@ -38,5 +38,5 @@ class EditResultLabel(View):
             result.set_label(data['label'])
             result.set_label(data['label'])
             result.set_origin('user')
             result.set_origin('user')
 
 
-        self.nm.edit_result(result)
+        self.nm.edit_result(result.id)
         return make_response()
         return make_response()

+ 3 - 7
pycs/frontend/endpoints/results/GetProjectResults.py

@@ -1,7 +1,7 @@
 from flask import abort, jsonify
 from flask import abort, jsonify
 from flask.views import View
 from flask.views import View
 
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.interfaces.MediaStorage import MediaStorage
 from pycs.interfaces.MediaStorage import MediaStorage
 
 
 
 
@@ -12,18 +12,14 @@ class GetProjectResults(View):
     # pylint: disable=arguments-differ
     # pylint: disable=arguments-differ
     methods = ['GET']
     methods = ['GET']
 
 
-    def __init__(self, db: Database):
-        # pylint: disable=invalid-name
-        self.db = db
-
     def dispatch_request(self, project_id: int):
     def dispatch_request(self, project_id: int):
         # get project from database
         # get project from database
-        project = self.db.project(project_id)
+        project = Project.object.get(project_id)
         if project is None:
         if project is None:
             return abort(404)
             return abort(404)
 
 
         # map media files to a dict
         # map media files to a dict
-        ms = MediaStorage(self.db, project.identifier, None)
+        ms = MediaStorage(project.id)
         files = list(map(lambda f: f.serialize(), ms.files().iter()))
         files = list(map(lambda f: f.serialize(), ms.files().iter()))
 
 
         # return result
         # return result

+ 1 - 4
pycs/frontend/endpoints/results/GetResults.py

@@ -21,8 +21,5 @@ class GetResults(View):
         if file is None:
         if file is None:
             return abort(404)
             return abort(404)
 
 
-        # get results
-        results = file.results()
-
         # return result
         # return result
-        return jsonify(results)
+        return jsonify(file.results.all())

+ 1 - 1
pycs/frontend/endpoints/results/RemoveResult.py

@@ -33,5 +33,5 @@ class RemoveResult(View):
         with self.db:
         with self.db:
             result.remove()
             result.remove()
 
 
-        self.nm.remove_result(result)
+        self.nm.remove_result(result.id)
         return make_response()
         return make_response()

+ 2 - 2
pycs/frontend/endpoints/results/ResetResults.py

@@ -30,7 +30,7 @@ class ResetResults(View):
             return abort(404)
             return abort(404)
 
 
         # get results
         # get results
-        results = file.results()
+        results = file.results.all()
 
 
         # start transaction
         # start transaction
         with self.db:
         with self.db:
@@ -38,6 +38,6 @@ class ResetResults(View):
                 result.remove()
                 result.remove()
 
 
         for result in results:
         for result in results:
-            self.nm.remove_result(result)
+            self.nm.remove_result(result.id)
 
 
         return make_response()
         return make_response()

+ 39 - 46
pycs/frontend/notifications/NotificationManager.py

@@ -1,5 +1,6 @@
 from socketio import Server
 from socketio import Server
 
 
+from pycs import app
 from pycs.database.File import File
 from pycs.database.File import File
 from pycs.database.Label import Label
 from pycs.database.Label import Label
 from pycs.database.Model import Model
 from pycs.database.Model import Model
@@ -18,7 +19,16 @@ class NotificationManager:
         self.sio = sio
         self.sio = sio
         self.json = JSONEncoder()
         self.json = JSONEncoder()
 
 
-    def __emit(self, name, obj):
+    def __emit(self, name, obj_id, cls=None):
+        if cls is not None:
+            assert isinstance(obj_id, int), "Object ID must be an integer!"
+            obj = cls.query.get(obj_id)
+
+        else:
+            obj = obj_id
+
+        app.logger.debug(name, obj)
+
         enc = self.json.default(obj)
         enc = self.json.default(obj)
         self.sio.emit(name, enc)
         self.sio.emit(name, enc)
 
 
@@ -29,7 +39,6 @@ class NotificationManager:
         :param created_job:
         :param created_job:
         :return:
         :return:
         """
         """
-        print('create_job', created_job)
         self.__emit('create-job', created_job)
         self.__emit('create-job', created_job)
 
 
     def edit_job(self, edited_job: Job):
     def edit_job(self, edited_job: Job):
@@ -39,7 +48,6 @@ class NotificationManager:
         :param edited_job:
         :param edited_job:
         :return:
         :return:
         """
         """
-        print('edit_job', edited_job)
         self.__emit('edit-job', edited_job)
         self.__emit('edit-job', edited_job)
 
 
     def remove_job(self, removed_job: Job):
     def remove_job(self, removed_job: Job):
@@ -49,145 +57,130 @@ class NotificationManager:
         :param removed_job:
         :param removed_job:
         :return:
         :return:
         """
         """
-        print('remove_job', removed_job)
         self.__emit('remove-job', removed_job)
         self.__emit('remove-job', removed_job)
 
 
-    def create_model(self, created_model: Model):
+    def create_model(self, created_model_id: int):
         """
         """
         fire create-model event
         fire create-model event
 
 
         :param created_model:
         :param created_model:
         :return:
         :return:
         """
         """
-        print('create_model', created_model)
-        self.__emit('create-model', created_model)
+        self.__emit('create-model', created_model_id, Model)
 
 
-    def remove_model(self, removed_model: Model):
+    def remove_model(self, removed_model_id: int):
         """
         """
         fire remove-model event
         fire remove-model event
 
 
         :param removed_model:
         :param removed_model:
         :return:
         :return:
         """
         """
-        print('remove_model', removed_model)
-        self.__emit('remove-model', removed_model)
+        self.__emit('remove-model', removed_model_id, Model)
 
 
-    def create_project(self, created_project: Project):
+    def create_project(self, created_project_id: int):
         """
         """
         fire create-project event
         fire create-project event
 
 
         :param created_project:
         :param created_project:
         :return:
         :return:
         """
         """
-        print('create_project', created_project)
-        self.__emit('create-project', created_project)
+        self.__emit('create-project', created_project_id, Project)
 
 
-    def remove_project(self, removed_project: Project):
+    def remove_project(self, removed_project_id: int):
         """
         """
         fire remove-project event
         fire remove-project event
 
 
         :param removed_project:
         :param removed_project:
         :return:
         :return:
         """
         """
-        print('remove_project', removed_project)
-        self.__emit('remove-project', removed_project)
+        self.__emit('remove-project', removed_project_id, Project)
 
 
-    def edit_project(self, edited_project: Project):
+    def edit_project(self, edited_project_id: int):
         """
         """
         fire edit-project event
         fire edit-project event
 
 
         :param edited_project:
         :param edited_project:
         :return:
         :return:
         """
         """
-        print('edit_project', edited_project)
-        self.__emit('edit-project', edited_project)
+        self.__emit('edit-project', edited_project_id, Project)
 
 
-    def create_label(self, created_label: Label):
+    def create_label(self, created_label_id: int):
         """
         """
         fire create-label event
         fire create-label event
 
 
         :param created_label:
         :param created_label:
         :return:
         :return:
         """
         """
-        print('create_label', created_label)
-        self.__emit('create-label', created_label)
+        self.__emit('create-label', created_label_id, Label)
 
 
-    def edit_label(self, edited_label: Label):
+    def edit_label(self, edited_label_id: int):
         """
         """
         fire edit-label event
         fire edit-label event
 
 
         :param edited_label:
         :param edited_label:
         :return:
         :return:
         """
         """
-        print('edit_label', edited_label)
-        self.__emit('edit-label', edited_label)
+        self.__emit('edit-label', edited_label_id, Label)
 
 
-    def remove_label(self, removed_label: Label):
+    def remove_label(self, removed_label_id: int):
         """
         """
         fire remove-label event
         fire remove-label event
 
 
         :param removed_label:
         :param removed_label:
         :return:
         :return:
         """
         """
-        print('remove_label', removed_label)
-        self.__emit('remove-label', removed_label)
+        self.__emit('remove-label', removed_label_id, Label)
 
 
-    def create_file(self, created_file: File):
+    def create_file(self, created_file_id: int):
         """
         """
         fire create-file event
         fire create-file event
 
 
         :param created_file:
         :param created_file:
         :return:
         :return:
         """
         """
-        print('create_file', created_file)
-        self.__emit('create-file', created_file)
+        self.__emit('create-file', created_file_id, File)
 
 
-    def edit_file(self, edited_file: File):
+    def edit_file(self, edited_file_id: int):
         """
         """
         fire edit-file event
         fire edit-file event
 
 
         :param edited_file:
         :param edited_file:
         :return:
         :return:
         """
         """
-        print('edit_file', edited_file)
-        self.__emit('edit-file', edited_file)
+        self.__emit('edit-file', edited_file_id, File)
 
 
-    def remove_file(self, removed_file: File):
+    def remove_file(self, removed_file_id: int):
         """
         """
         fire remove-file event
         fire remove-file event
 
 
         :param removed_file:
         :param removed_file:
         :return:
         :return:
         """
         """
-        print('remove_file', removed_file)
-        self.__emit('remove-file', removed_file)
+        self.__emit('remove-file', removed_file_id, File)
 
 
-    def create_result(self, created_result: Result):
+    def create_result(self, created_result_id: int):
         """
         """
         fire create-result event
         fire create-result event
 
 
         :param created_result:
         :param created_result:
         :return:
         :return:
         """
         """
-        print('create_result', created_result)
-        self.__emit('create-result', created_result)
+        self.__emit('create-result', created_result_id, Result)
 
 
-    def edit_result(self, edited_result: Result):
+    def edit_result(self, edited_result_id: int):
         """
         """
         fire edit-result event
         fire edit-result event
 
 
         :param edited_result:
         :param edited_result:
         :return:
         :return:
         """
         """
-        print('edit_result', edited_result)
-        self.__emit('edit-result', edited_result)
+        self.__emit('edit-result', edited_result_id, Result)
 
 
-    def remove_result(self, removed_result: Result):
+    def remove_result(self, removed_result_id: int):
         """
         """
         fire remove-result event
         fire remove-result event
 
 
         :param removed_result:
         :param removed_result:
         :return:
         :return:
         """
         """
-        print('remove_result', removed_result)
-        self.__emit('remove-result', removed_result)
+        self.__emit('remove-result', removed_result_id, Result)

+ 6 - 8
pycs/interfaces/MediaFile.py

@@ -35,7 +35,7 @@ class MediaFile:
         :param reference: use None to remove this file's collection
         :param reference: use None to remove this file's collection
         """
         """
         self.__file.set_collection_by_reference(reference)
         self.__file.set_collection_by_reference(reference)
-        self.__notifications.add(self.__notifications.nm.edit_file, self.__file)
+        self.__notifications.add(self.__notifications.nm.edit_file, self.__file.id)
 
 
     def set_image_label(self, label: Union[int, MediaLabel], frame: int = None):
     def set_image_label(self, label: Union[int, MediaLabel], frame: int = None):
         """
         """
@@ -53,7 +53,7 @@ class MediaFile:
             data = None
             data = None
 
 
         created = self.__file.create_result('pipeline', 'labeled-image', label, data)
         created = self.__file.create_result('pipeline', 'labeled-image', label, data)
-        self.__notifications.add(self.__notifications.nm.create_result, created)
+        self.__notifications.add(self.__notifications.nm.create_result, created.id)
 
 
     def add_bounding_box(self, x: float, y: float, w: float, h: float,
     def add_bounding_box(self, x: float, y: float, w: float, h: float,
                          label: Union[int, MediaLabel] = None, frame: int = None):
                          label: Union[int, MediaLabel] = None, frame: int = None):
@@ -80,7 +80,7 @@ class MediaFile:
             label = label.identifier
             label = label.identifier
 
 
         created = self.__file.create_result('pipeline', 'bounding-box', label, result)
         created = self.__file.create_result('pipeline', 'bounding-box', label, result)
-        self.__notifications.add(self.__notifications.nm.create_result, created)
+        self.__notifications.add(self.__notifications.nm.create_result, created.id)
 
 
     def remove_predictions(self):
     def remove_predictions(self):
         """
         """
@@ -88,7 +88,7 @@ class MediaFile:
         """
         """
         removed = self.__file.remove_results(origin='pipeline')
         removed = self.__file.remove_results(origin='pipeline')
         for r in removed:
         for r in removed:
-            self.__notifications.add(self.__notifications.nm.remove_result, r)
+            self.__notifications.add(self.__notifications.nm.remove_result, r.id)
 
 
     def __get_results(self, origin: str) -> List[Union[MediaImageLabel, MediaBoundingBox]]:
     def __get_results(self, origin: str) -> List[Union[MediaImageLabel, MediaBoundingBox]]:
         def map_r(result: Result) -> Union[MediaImageLabel, MediaBoundingBox]:
         def map_r(result: Result) -> Union[MediaImageLabel, MediaBoundingBox]:
@@ -97,9 +97,7 @@ class MediaFile:
             else:
             else:
                 return MediaBoundingBox(result)
                 return MediaBoundingBox(result)
 
 
-        return list(map(map_r,
-                        filter(lambda r: r.origin == origin,
-                               self.__file.results())))
+        return list(map(map_r, self.__file.results.filter_by(origin=origin)))
 
 
     def results(self) -> List[Union[MediaImageLabel, MediaBoundingBox]]:
     def results(self) -> List[Union[MediaImageLabel, MediaBoundingBox]]:
         """
         """
@@ -125,6 +123,6 @@ class MediaFile:
             'fps': self.fps,
             'fps': self.fps,
             'path': self.path,
             'path': self.path,
             'filename': self.__file.name + self.__file.extension,
             'filename': self.__file.name + self.__file.extension,
-            'results': list(map(lambda r: r.serialize(), self.results())),
+            'results': list(map(lambda r: r.serialize(), self.results)),
             'predictions': list(map(lambda r: r.serialize(), self.predictions())),
             'predictions': list(map(lambda r: r.serialize(), self.predictions())),
         }
         }

+ 3 - 3
pycs/interfaces/MediaFileList.py

@@ -39,11 +39,11 @@ class MediaFileList:
             source = self.__project
             source = self.__project
 
 
         if self.__label is None:
         if self.__label is None:
-            for file in source.files():
+            for file in source.files.all():
                 yield MediaFile(file, self.__notifications)
                 yield MediaFile(file, self.__notifications)
         else:
         else:
-            for file in source.files():
-                for result in file.results():
+            for file in source.files.all():
+                for result in file.results.all():
                     if result.label == self.__label:
                     if result.label == self.__label:
                         yield MediaFile(file, self.__notifications)
                         yield MediaFile(file, self.__notifications)
                         break
                         break

+ 6 - 7
pycs/interfaces/MediaStorage.py

@@ -1,6 +1,6 @@
 from typing import List
 from typing import List
 
 
-from pycs.database.Database import Database
+from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.frontend.notifications.NotificationList import NotificationList
 from pycs.interfaces.MediaFileList import MediaFileList
 from pycs.interfaces.MediaFileList import MediaFileList
 from pycs.interfaces.MediaLabel import MediaLabel
 from pycs.interfaces.MediaLabel import MediaLabel
@@ -11,13 +11,12 @@ class MediaStorage:
     helper class for pipelines to interact with database entities
     helper class for pipelines to interact with database entities
     """
     """
 
 
-    def __init__(self, db: Database, project_id: int, notifications: NotificationList = None):
-        self.__db = db
+    def __init__(self, project_id: int, notifications: NotificationList = None):
         self.__project_id = project_id
         self.__project_id = project_id
         self.__notifications = notifications
         self.__notifications = notifications
 
 
-        self.__project = self.__db.project(self.__project_id)
-        self.__collections = self.__project.collections()
+        self.__project = Project.query.get(self.__project_id)
+        self.__collections = self.__project.collections.all()
 
 
     def labels(self) -> List[MediaLabel]:
     def labels(self) -> List[MediaLabel]:
         """
         """
@@ -25,7 +24,7 @@ class MediaStorage:
 
 
         :return: list of labels
         :return: list of labels
         """
         """
-        label_list = self.__project.labels()
+        label_list = self.__project.labels.all()
         label_dict = {la.identifier: MediaLabel(la) for la in label_list}
         label_dict = {la.identifier: MediaLabel(la) for la in label_list}
         result = []
         result = []
 
 
@@ -48,7 +47,7 @@ class MediaStorage:
         """
         """
         return list(filter(
         return list(filter(
             lambda ml: ml.parent is None,
             lambda ml: ml.parent is None,
-            self.labels()
+            self.labels.all()
         ))
         ))
 
 
     def files(self) -> MediaFileList:
     def files(self) -> MediaFileList:

+ 3 - 2
pycs/util/PipelineCache.py

@@ -46,7 +46,7 @@ class PipelineCache:
 
 
         # save instance to cache
         # save instance to cache
         with self.__lock:
         with self.__lock:
-            self.__pipelines[root_folder] = [1, pipeline, project]
+            self.__pipelines[root_folder] = [1, pipeline, project.id]
 
 
         # return
         # return
         return pipeline
         return pipeline
@@ -97,7 +97,8 @@ class PipelineCache:
     def __run(self):
     def __run(self):
         while True:
         while True:
             # get pipeline
             # get pipeline
-            pipeline, project = tpool.execute(self.__get)
+            pipeline, project_id = tpool.execute(self.__get)
+            project = Project.query.get(project_id)
 
 
             # create job to close pipeline
             # create job to close pipeline
             self.__jobs.run(project,
             self.__jobs.run(project,

+ 1 - 1
settings.json

@@ -18,7 +18,7 @@
       }
       }
     },
     },
     "root": {
     "root": {
-        "level": "DEBUG",
+        "level": "INFO",
         "handlers": ["wsgi"]
         "handlers": ["wsgi"]
     }
     }
   },
   },