fixing bugs resulting from the change

pycs/database/Collection.py

@@ -29,7 +29,8 @@ class Collection(NamedBaseModel):
     )
 
     # relationships to other models
-    files = db.relationship("File", backref="collection", lazy=True)
+    files = db.relationship("File", backref="collection", lazy="dynamic")
+    serialize_rules = ('-files',)
 
 
     def count_files(self) -> int:
@@ -40,3 +41,14 @@ class Collection(NamedBaseModel):
     #     files = File.query.filter_by(project_id=self.project_id, collection_id=self.id)
     #     raise NotImplementedError
 
+    def get_files(self, offset: int = 0, limit: int = -1):
+        """
+        get an iterator of files associated with this project
+
+        :param offset: file offset
+        :param limit: file limit
+        :return: iterator of files
+        """
+        from pycs.database.File import File
+        return self.files.order_by(File.id).offset(offset).limit(limit)
+

pycs/database/Database.py

@@ -3,14 +3,14 @@ from contextlib import closing
 from time import time
 from typing import Optional, Iterator
 
+from pycs import app
+from pycs import db
 from pycs.database.Collection import Collection
 from pycs.database.File import File
 from pycs.database.LabelProvider import LabelProvider
 from pycs.database.Model import Model
 from pycs.database.Project import Project
 from pycs.database.Result import Result
-from pycs.database.discovery.LabelProviderDiscovery import discover as discover_label_providers
-from pycs.database.discovery.ModelDiscovery import discover as discover_models
 
 
 class Database:
@@ -27,166 +27,26 @@ class Database:
         # save properties
         self.path = path
 
-        # initialize database connection
-        self.con = sqlite3.connect(path)
-        self.con.execute("PRAGMA foreign_keys = ON")
-
-        if initialization:
-            # create tables
-            with self, closing(self.con.cursor()) as cursor:
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS models (
-                        id          INTEGER PRIMARY KEY,
-                        name        TEXT                NOT NULL,
-                        description TEXT,
-                        root_folder TEXT                NOT NULL UNIQUE,
-                        supports    TEXT                NOT NULL
-                    )
-                ''')
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS label_providers (
-                        id          INTEGER PRIMARY KEY,
-                        name        TEXT                NOT NULL,
-                        description TEXT,
-                        root_folder TEXT                NOT NULL UNIQUE
-                    )
-                ''')
-
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS projects (
-                        id             INTEGER PRIMARY KEY,
-                        name           TEXT                NOT NULL,
-                        description    TEXT,
-                        created        INTEGER             NOT NULL,
-                        model          INTEGER,
-                        label_provider INTEGER,
-                        root_folder    TEXT                NOT NULL UNIQUE,
-                        external_data  BOOL                NOT NULL,
-                        data_folder    TEXT                NOT NULL,
-                        FOREIGN KEY (model) REFERENCES models(id)
-                            ON UPDATE CASCADE ON DELETE SET NULL,
-                        FOREIGN KEY (label_provider) REFERENCES label_providers(id)
-                            ON UPDATE CASCADE ON DELETE SET NULL
-                    )
-                ''')
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS labels (
-                        id        INTEGER PRIMARY KEY,
-                        project   INTEGER             NOT NULL,
-                        parent    INTEGER,
-                        created   INTEGER             NOT NULL,
-                        reference TEXT,
-                        name      TEXT                NOT NULL,
-                        FOREIGN KEY (project) REFERENCES projects(id)
-                            ON UPDATE CASCADE ON DELETE CASCADE,
-                        FOREIGN KEY (parent) REFERENCES labels(id)
-                            ON UPDATE CASCADE ON DELETE SET NULL,
-                        UNIQUE(project, reference)
-                    )
-                ''')
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS collections (
-                        id          INTEGER          PRIMARY KEY,
-                        project     INTEGER NOT NULL,
-                        reference   TEXT    NOT NULL,
-                        name        TEXT    NOT NULL,
-                        description TEXT,
-                        position    INTEGER NOT NULL,
-                        autoselect  INTEGER NOT NULL,
-                        FOREIGN KEY (project) REFERENCES projects(id)
-                            ON UPDATE CASCADE ON DELETE CASCADE,
-                        UNIQUE(project, reference)
-                    )
-                ''')
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS files (
-                        id         INTEGER PRIMARY KEY,
-                        uuid       TEXT                NOT NULL,
-                        project    INTEGER             NOT NULL,
-                        collection INTEGER,
-                        type       TEXT                NOT NULL,
-                        name       TEXT                NOT NULL,
-                        extension  TEXT                NOT NULL,
-                        size       INTEGER             NOT NULL,
-                        created    INTEGER             NOT NULL,
-                        path       TEXT                NOT NULL,
-                        frames     INTEGER,
-                        fps        FLOAT,
-                        FOREIGN KEY (project) REFERENCES projects(id)
-                            ON UPDATE CASCADE ON DELETE CASCADE,
-                        FOREIGN KEY (collection) REFERENCES collections(id)
-                            ON UPDATE CASCADE ON DELETE SET NULL,
-                        UNIQUE(project, path)
-                    )
-                ''')
-                cursor.execute('''
-                    CREATE TABLE IF NOT EXISTS results (
-                        id     INTEGER PRIMARY KEY,
-                        file   INTEGER             NOT NULL,
-                        origin TEXT                NOT NULL,
-                        type   TEXT                NOT NULL,
-                        label  INTEGER,
-                        data   TEXT,
-                        FOREIGN KEY (file) REFERENCES files(id)
-                            ON UPDATE CASCADE ON DELETE CASCADE
-                    )
-                ''')
-
         if discovery:
             # run discovery modules
-            with self:
-                discover_models(self.con)
-                discover_label_providers(self.con)
-
-    def close(self):
-        self.con.close()
-
-    def copy(self):
-        return Database(self.path, initialization=False, discovery=False)
-
-    def commit(self):
-        self.con.commit()
+            Model.discover("models/")
+            LabelProvider.discover("labels/")
 
     def __enter__(self):
-        self.con.__enter__()
+        app.logger.warning("REMOVE ME!")
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.con.__exit__(exc_type, exc_val, exc_tb)
-
-    def get_object_by_id(self, table_name: str, identifier: int, cls):
-        """
-        create an object from cls and a row fetched from table_name and
-        identified by the identifier
-
-        :param table_name: table name
-        :param identifier: unique identifier
-        :param cls: class that is used to create the object
-        :return: object of type cls
-        """
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute(f'SELECT * FROM {table_name} WHERE id = ?', [identifier])
-            row = cursor.fetchone()
-
-            if row is not None:
-                return cls(self, row)
-
-            return None
-
-    def get_objects(self, table_name: str, cls):
-        """
-        get a list of all available objects in the table
+        app.logger.warning("REMOVE ME!")
 
-        :param table_name: table name
-        :param cls: class that is used to create the objects
-        :return: list of object of type cls
-        """
-
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute(f'SELECT * FROM {table_name}')
-            for row in cursor:
-                yield cls(self, row)
+        if exc_type is None:
+            db.session.commit()
+        else:
+            app.logger.info("Rolling back a transaction!")
+            db.session.rollback()
 
+    def copy(self):
+        return Database(self.path, initialization=False, discovery=False)
 
     def models(self) -> Iterator[Model]:
         """
@@ -194,7 +54,7 @@ class Database:
 
         :return: iterator of models
         """
-        return self.get_objects("models", Model)
+        return Model.query.all()
 
     def model(self, identifier: int) -> Optional[Model]:
         """
@@ -203,7 +63,7 @@ class Database:
         :param identifier: unique identifier
         :return: model
         """
-        return self.get_object_by_id("models", identifier, Model)
+        return Model.query.get(identifier)
 
     def label_providers(self) -> Iterator[LabelProvider]:
         """
@@ -211,7 +71,7 @@ class Database:
 
         :return: iterator over label providers
         """
-        return self.get_objects("label_providers", LabelProvider)
+        return LabelProvider.query.all()
 
     def label_provider(self, identifier: int) -> Optional[LabelProvider]:
         """
@@ -220,7 +80,7 @@ class Database:
         :param identifier: unique identifier
         :return: label provider
         """
-        return self.get_object_by_id("label_providers", identifier, LabelProvider)
+        return LabelProvider.query.get(identifier)
 
     def projects(self) -> Iterator[Project]:
         """
@@ -228,7 +88,7 @@ class Database:
 
         :return: iterator over projects
         """
-        return self.get_objects("projects", Project)
+        return Project.query.all()
 
     def project(self, identifier: int) -> Optional[Project]:
         """
@@ -237,44 +97,7 @@ class Database:
         :param identifier: unique identifier
         :return: project
         """
-        return self.get_object_by_id("projects", identifier, Project)
-
-    def create_project(self,
-                       name: str,
-                       description: str,
-                       model: Model,
-                       label_provider: Optional[LabelProvider],
-                       root_folder: str,
-                       external_data: bool,
-                       data_folder: str):
-        """
-        insert a project into the database
-
-        :param name: project name
-        :param description: project description
-        :param model: used model
-        :param label_provider: used label provider (optional)
-        :param root_folder: path to project folder
-        :param external_data: whether an external data directory is used
-        :param data_folder: path to data folder
-        :return: created project
-        """
-        # prepare some values
-        created = int(time())
-        label_provider_id = label_provider.identifier if label_provider is not None else None
-
-        # insert statement
-        with closing(self.con.cursor()) as cursor:
-            cursor.execute('''
-                INSERT INTO projects (
-                    name, description, created, model, label_provider, root_folder, external_data, 
-                    data_folder
-                )
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-            ''', (name, description, created, model.identifier, label_provider_id, root_folder,
-                  external_data, data_folder))
-
-            return self.project(cursor.lastrowid)
+        return Project.query.get(identifier)
 
     def collection(self, identifier: int) -> Optional[Collection]:
         """
@@ -283,7 +106,7 @@ class Database:
         :param identifier: unique identifier
         :return: collection
         """
-        return self.get_object_by_id("collections", identifier, Collection)
+        return Collection.query.get(identifier)
 
     def file(self, identifier) -> Optional[File]:
         """
@@ -292,7 +115,7 @@ class Database:
         :param identifier: unique identifier
         :return: file
         """
-        return self.get_object_by_id("files", identifier, File)
+        return File.query.get(identifier)
 
     def result(self, identifier) -> Optional[Result]:
         """
@@ -301,4 +124,36 @@ class Database:
         :param identifier: unique identifier
         :return: result
         """
-        return self.get_object_by_id("results", identifier, Result)
+        return Result.query.get(identifier)
+
+    def create_project(self,
+                       name: str,
+                       description: str,
+                       model: Model,
+                       label_provider: Optional[LabelProvider],
+                       root_folder: str,
+                       external_data: bool,
+                       data_folder: str):
+        """
+        insert a project into the database
+
+        :param name: project name
+        :param description: project description
+        :param model: used model
+        :param label_provider: used label provider (optional)
+        :param root_folder: path to project folder
+        :param external_data: whether an external data directory is used
+        :param data_folder: path to data folder
+        :return: created project
+        """
+        # prepare some values
+
+        return Project.new(commit=True,
+            name=name,
+            description=description,
+            model=model,
+            label_provider=label_provider,
+            root_folder=root_folder,
+            external_data=external_data,
+            data_folder=data_folder
+        )

pycs/database/File.py

@@ -48,8 +48,14 @@ class File(NamedBaseModel):
 
 
     # relationships to other models
-    results = db.relationship("Result", backref="file", lazy=True)
+    results = db.relationship("Result", backref="file", lazy="dynamic")
+    serialize_rules = ('-results',)
 
+    def serialize(self):
+        result = super().serialize()
+        if result["data"] is not None:
+            result["data"] = json.loads(result["data"])
+        return result
 
     def set_collection(self, id: T.Optional[int]):
         """

pycs/database/Label.py

@@ -44,6 +44,13 @@ class Label(NamedBaseModel):
 
     # relationships to other models
     parent = db.relationship("Label", backref="children", remote_side=[id])
+    serialize_only = (
+        "id",
+        "name",
+        "project_id",
+        "parent_id",
+        "reference",
+    )
 
     def set_parent(self, parent_id: int):
         """

pycs/database/LabelProvider.py

@@ -1,3 +1,7 @@
+import json
+
+from pathlib import Path
+
 from pycs import db
 from pycs.database.base import NamedBaseModel
 
@@ -10,4 +14,22 @@ class LabelProvider(NamedBaseModel):
     root_folder = db.Column(db.String, nullable=False, unique=True)
 
     # relationships to other models
-    projects = db.relationship("Project", backref="label_provider", lazy=True)
+    projects = db.relationship("Project", backref="label_provider", lazy="dynamic")
+    serialize_rules = ('-projects',)
+
+    @classmethod
+    def discover(cls, root: Path, config_name: str = "configuration.json"):
+
+        for folder in Path(root).glob("*"):
+            with open(folder / config_name) as f:
+                config = json.load(f)
+
+            # extract data
+            name = config['name']
+            description = config.get('description', None)
+
+            provider, _ = cls.get_or_create(root_folder=str(folder))
+            provider.name = name
+            provider.description = description
+
+            db.session.commit()

pycs/database/Model.py

@@ -1,5 +1,7 @@
 import json
 
+from pathlib import Path
+
 from pycs import db
 from pycs.database.base import NamedBaseModel
 
@@ -13,26 +15,56 @@ class Model(NamedBaseModel):
     supports_encoded = db.Column(db.String, nullable=False)
 
     # relationships to other models
-    projects = db.relationship("Project", backref="model", lazy=True)
+    projects = db.relationship("Project", backref="model", lazy="dynamic")
+    serialize_rules = ('-projects',)
+
+    def serialize(self):
+        result = super().serialize()
+        result["supports"] = self.supports
+        return result
 
     @property
     def supports(self):
         return json.loads(self.supports_encoded)
 
+    @supports.setter
+    def supports(self, value):
+        if isinstance(value, str):
+            self.supports_encoded = value
 
-    def copy_to(self, new_name: str, new_root_folder: str):
+        elif isinstance(value, (dict, list)):
+            self.supports_encoded = json.dumps(value)
 
-        model = Model.query.get(root_folder=new_root_folder)
-        is_new = False
+        else:
+            raise ValueError(f"Not supported type: {type(value)}")
+
+    def copy_to(self, new_name: str, new_root_folder: str):
 
-        if model is None:
-            model = Model.new(root_folder=new_root_folder)
-            is_new = True
+        model, is_new = Model.get_or_create(root_folder=new_root_folder)
 
-        model.name = name
+        model.name = new_name
         model.description = self.description
         model.supports_encoded = self.supports_encoded
 
         self.commit()
 
         return model, is_new
+
+    @classmethod
+    def discover(cls, root: Path, config_name: str = "configuration.json"):
+        for folder in Path(root).glob("*"):
+            with open(folder / config_name) as f:
+                config = json.load(f)
+
+            # extract data
+            name = config['name']
+            description = config.get('description', None)
+            supports = config['supports']
+
+            model, _ = cls.get_or_create(root_folder=str(folder))
+
+            model.name = name
+            model.description = description
+            model.supports = supports
+
+            db.session.commit()

pycs/database/Project.py

@@ -37,10 +37,15 @@ class Project(NamedBaseModel):
     __table_args__ = ()
 
     # relationships to other models
-    files = db.relationship("File", backref="project", lazy=True)
-    labels = db.relationship("Label", backref="project", lazy=True)
-    collections = db.relationship("Collection", backref="project", lazy=True)
+    files = db.relationship("File", backref="project", lazy="dynamic")
+    labels = db.relationship("Label", backref="project", lazy="dynamic")
+    collections = db.relationship("Collection", backref="project", lazy="dynamic")
 
+    serialize_rules = (
+        '-files',
+        '-labels',
+        '-collections',
+    )
 
     def label(self, id: int) -> T.Optional[Label]:
         """
@@ -90,12 +95,7 @@ class Project(NamedBaseModel):
         :return: created or edited label, insert
         """
 
-        label = Label.query.get(project=self, reference=reference)
-        is_new = False
-
-        if label is None:
-            label = Label.new(project=self, reference=reference)
-            is_new = True
+        label, is_new = Label.get_or_create(project=self, reference=reference)
 
         label.set_name(name)
         label.set_parent(parent_id)
@@ -111,15 +111,7 @@ class Project(NamedBaseModel):
                           position: int,
                           autoselect: bool):
 
-
-        collection = Collection.query.get(project=self, reference=reference)
-        is_new = False
-
-        if collection is None:
-            collection = Collection.new(project=self,
-                                        reference=reference)
-            is_new = True
-
+        collection, is_new = Collection.get_or_create(project=self, reference=reference)
         collection.name = name
         collection.description = description
         collection.position = position
@@ -145,13 +137,9 @@ class Project(NamedBaseModel):
         """
         path = join(self.data_folder, filename + extension)
 
-        file = File.objects.get(project=self, path=path)
-        is_new = False
-
-        if file is None:
-            file = File.new(uuid=uuid, project=self, path=path)
-            is_new = True
+        file, is_new = File.get_or_create(project=self, path=path)
 
+        file.uuid = uuid
         file.type = file_type
         file.name = name
         file.extension = extension
@@ -171,7 +159,7 @@ class Project(NamedBaseModel):
         :return:
         """
         self.description = description
-        self
+
     def count_files(self) -> int:
         """
         count files associated with this project
@@ -188,7 +176,7 @@ class Project(NamedBaseModel):
         :param limit: file limit
         :return: iterator of files
         """
-        return self.files.order_by(File.id.acs()).offset(offset).limit(limit)
+        return self.files.order_by(File.id).offset(offset).limit(limit)
 
     def count_files_without_results(self) -> int:
         """

pycs/database/Result.py

@@ -1,7 +1,7 @@
 import typing as T
+import json
 
 from contextlib import closing
-from json import dumps, loads
 
 from pycs import db
 from pycs.database.base import BaseModel
@@ -23,6 +23,11 @@ class Result(BaseModel):
 
     data = db.Column(db.String)
 
+    def serialize(self):
+        result = super().serialize()
+        if result["data"] is not None:
+            result["data"] = json.loads(result["data"])
+        return result
 
     def set_origin(self, origin: str):
         """

pycs/database/__init__.py

@@ -0,0 +1,7 @@
+from pycs.database.Collection import Collection
+from pycs.database.File import File
+from pycs.database.Label import Label
+from pycs.database.LabelProvider import LabelProvider
+from pycs.database.Model import Model
+from pycs.database.Result import Result
+from pycs.database.Project import Project

pycs/database/base.py

@@ -1,20 +1,45 @@
+import typing as T
+import datetime
 
+from sqlalchemy.inspection import inspect
+from sqlalchemy.orm.collections import InstrumentedList
+from sqlalchemy_serializer import SerializerMixin
+
+from pycs import app
 from pycs import db
 
-class BaseModel(db.Model):
+class ModelSerializer(SerializerMixin):
+    date_format = '%s'  # Unixtimestamp (seconds)
+    datetime_format = '%d. %b. %Y %H:%M:%S'
+    time_format = '%H:%M'
+
+class BaseModel(db.Model, ModelSerializer):
     __abstract__ = True
 
     id = db.Column(db.Integer, primary_key=True)
 
+    @property
+    def identifier(self) -> int:
+        app.logger.warning("REMOVE ME!")
+        return self.id
+
+
+    def serialize(self) -> dict:
 
-    def remove(self) -> None:
+        result = self.to_dict()
+        result["identifier"] = result["id"]
+        return result
+
+
+    def remove(self, commit=True) -> None:
         """
         remove this instance from the database
 
         :return:
         """
         db.session.delete(self)
-        self.commit()
+        if commit:
+            self.commit()
 
     @classmethod
     def new(cls, commit=False, **kwargs):
@@ -22,7 +47,20 @@ class BaseModel(db.Model):
         db.session.add(obj)
 
         if commit:
-            self.commit()
+            db.session.commit()
+        return obj
+
+    @classmethod
+    def get_or_create(cls, **kwargs) -> T.Tuple[T.Any, bool]:
+
+        is_new = False
+        obj = cls.query.filter_by(**kwargs).one_or_none()
+
+        if obj is None:
+            obj = cls.new(commit=False, **kwargs)
+            is_new = True
+
+        return obj, is_new
 
     def commit(self):
         db.session.commit()

pycs/database/discovery/LabelProviderDiscovery.py

@@ -1,31 +0,0 @@
-from contextlib import closing
-from glob import glob
-from json import load
-from os import path
-
-
-def discover(database):
-    """
-    find label providers in the corresponding folder and add them to the database
-
-    :param database:
-    :return:
-    """
-    with closing(database.cursor()) as cursor:
-        # list folders in labels/
-        for folder in glob('labels/*'):
-            # load distribution.json
-            with open(path.join(folder, 'configuration.json'), 'r') as file:
-                label = load(file)
-
-            # extract data
-            name = label['name']
-            description = label['description'] if 'description' in label else None
-
-            # save to database
-            cursor.execute('''
-                INSERT INTO label_providers (name, description, root_folder)
-                VALUES (?, ?, ?)
-                ON CONFLICT (root_folder)
-                DO UPDATE SET name = ?, description = ?
-            ''', (name, description, folder, name, description))

pycs/database/discovery/ModelDiscovery.py

@@ -1,32 +0,0 @@
-from contextlib import closing
-from glob import glob
-from json import load, dumps
-from os import path
-
-
-def discover(database):
-    """
-    find models in the corresponding folder and add them to the database
-
-    :param database:
-    :return:
-    """
-    with closing(database.cursor()) as cursor:
-        # list folders in models/
-        for folder in glob('models/*'):
-            # load distribution.json
-            with open(path.join(folder, 'configuration.json'), 'r') as file:
-                model = load(file)
-
-            # extract data
-            name = model['name']
-            description = model['description'] if 'description' in model else None
-            supports = dumps(model['supports'])
-
-            # save to database
-            cursor.execute('''
-                INSERT INTO models (name, description, root_folder, supports)
-                VALUES (?, ?, ?, ?)
-                ON CONFLICT (root_folder)
-                DO UPDATE SET name = ?, description = ?, supports = ?
-            ''', (name, description, folder, supports, name, description, supports))

pycs/database/util/JSONEncoder.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from flask.json import JSONEncoder as Base
 
+from pycs.database.base import BaseModel
 
 class JSONEncoder(Base):
     """
@@ -9,6 +10,12 @@ class JSONEncoder(Base):
     """
 
     def default(self, o: Any) -> Any:
-        copy = o.__dict__.copy()
-        del copy['database']
-        return copy
+        if isinstance(o, BaseModel):
+            return o.serialize()
+
+        else:
+            return o.__dict__.copy()
+        # copy = o.__dict__.copy()
+        # if "database" in copy:
+        #     del copy['database']
+        # return copy

pycs/frontend/endpoints/labels/ListLabels.py

@@ -21,8 +21,4 @@ class ListLabels(View):
         if project is None:
             abort(404)
 
-        # get labels
-        labels = project.labels()
-
-        # return labels
-        return jsonify(labels)
+        return jsonify(project.labels.all())

pycs/frontend/endpoints/projects/CreateProject.py

@@ -33,7 +33,7 @@ class CreateProject(View):
         data = request.get_json(force=True)
 
         if 'name' not in data or 'description' not in data:
-            return abort(400)
+            return abort(400, "name and description information missing!")
 
         name = data['name']
         description = data['description']
@@ -45,7 +45,7 @@ class CreateProject(View):
             model = self.db.model(model_id)
 
             if model is None:
-                return abort(404)
+                return abort(404, "Model not found")
 
             # find label provider
             if data['label'] is None:
@@ -55,7 +55,7 @@ class CreateProject(View):
                 label_provider = self.db.label_provider(label_provider_id)
 
                 if label_provider is None:
-                    return abort(404)
+                    return abort(404, "Label provider not found")
 
             # create project folder
             project_folder = path.join('projects', str(uuid1()))

pycs/frontend/endpoints/projects/ExecuteLabelProvider.py

@@ -38,7 +38,7 @@ class ExecuteLabelProvider(View):
             return abort(404)
 
         # get label provider
-        label_provider = project.label_provider()
+        label_provider = project.label_provider
         if label_provider is None:
             return abort(400)
 

pycs/frontend/endpoints/projects/GetProjectModel.py

@@ -21,8 +21,5 @@ class GetProjectModel(View):
         if project is None:
             abort(404)
 
-        # get model
-        model = project.model()
-
         # return model
-        return jsonify(model)
+        return jsonify(project.model)

pycs/frontend/endpoints/projects/ListCollections.py

@@ -23,7 +23,7 @@ class ListCollections(View):
             return abort(404)
 
         # get collection list
-        collections = project.collections()
+        collections = project.collections.all()
 
         # disable autoselect if there are no elements in the collection
         found = False

pycs/frontend/endpoints/projects/ListFiles.py

@@ -35,7 +35,7 @@ class ListFiles(View):
                 files = list(collection.files(start, length))
         else:
             count = project.count_files()
-            files = list(project.files(start, length))
+            files = list(project.get_files(start, length))
 
         # return files
         return jsonify({

pycs/frontend/endpoints/projects/RemoveProject.py

@@ -1,8 +1,9 @@
-from shutil import rmtree
+import shutil
 
 from flask import make_response, request, abort
 from flask.views import View
 
+from pycs import db
 from pycs.database.Database import Database
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 
@@ -31,17 +32,17 @@ class RemoveProject(View):
             # find project
             project = self.db.project(identifier)
             if project is None:
-                abort(404)
-
-            # remove from database
-            project.remove()
+                abort(404, "Project not found")
 
             # remove model from database
-            model = project.model()
-            model.remove()
+            model = project.model
+            model.remove(commit=False)
+
+            # remove from database
+            project.remove(commit=False)
 
             # remove from file system
-            rmtree(project.root_folder)
+            shutil.rmtree(project.root_folder)
 
             # send update
             self.nm.remove_model(model)

pycs/frontend/util/JSONEncoder.py

@@ -1,3 +1,5 @@
+import datetime
+
 from typing import Any
 
 from flask.json import JSONEncoder as Base
@@ -16,7 +18,16 @@ class JSONEncoder(Base):
 
         if module.startswith('pycs.database'):
             return Database().default(o)
+
         if module.startswith('pycs.jobs'):
             return Jobs().default(o)
 
-        return o.__dict__
+        if isinstance(o, datetime.datetime):
+            return str(o)
+
+        try:
+            return o.__dict__
+        except:
+            import pdb; pdb.set_trace()
+            raise
+

pycs/jobs/JobRunner.py

@@ -219,6 +219,8 @@ class JobRunner:
 
             # save exceptions to show in ui
             except Exception as e:
+                import traceback
+                traceback.print_exc()
                 job.exception = f'{type(e).__name__} ({str(e)})'
 
             # remove from group dict

requirements.txt

@@ -7,6 +7,7 @@ flask
 flask-socketio
 flask-sqlalchemy
 flask-migrate
+SQLAlchemy-serializer
 # python-socketio
 munch
 scikit-image

settings.json

@@ -18,10 +18,10 @@
       }
     },
     "root": {
-        "level": "INFO",
+        "level": "DEBUG",
         "handlers": ["wsgi"]
     }
   },
 
-  "database": "data.sqlite3"
+  "database": "data2.sqlite3"
 }

webui/package-lock.json

@@ -1715,6 +1715,16 @@
           "integrity": "sha1-/q7SVZc9LndVW4PbwIhRpsY1IPo=",
           "dev": true
         },
+        "ansi-styles": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
         "cacache": {
           "version": "13.0.1",
           "resolved": "https://registry.npm.taobao.org/cacache/download/cacache-13.0.1.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcacache%2Fdownload%2Fcacache-13.0.1.tgz",
@@ -1741,6 +1751,53 @@
             "unique-filename": "^1.1.1"
           }
         },
+        "chalk": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+          "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true,
+          "optional": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true,
+          "optional": true
+        },
+        "loader-utils": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz",
+          "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "big.js": "^5.2.2",
+            "emojis-list": "^3.0.0",
+            "json5": "^2.1.2"
+          }
+        },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npm.taobao.org/source-map/download/source-map-0.6.1.tgz",
@@ -1757,6 +1814,16 @@
             "minipass": "^3.1.1"
           }
         },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        },
         "terser-webpack-plugin": {
           "version": "2.3.8",
           "resolved": "https://registry.npm.taobao.org/terser-webpack-plugin/download/terser-webpack-plugin-2.3.8.tgz?cache=0&sync_timestamp=1610194258495&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fterser-webpack-plugin%2Fdownload%2Fterser-webpack-plugin-2.3.8.tgz",
@@ -1773,6 +1840,18 @@
             "terser": "^4.6.12",
             "webpack-sources": "^1.4.3"
           }
+        },
+        "vue-loader-v16": {
+          "version": "npm:vue-loader@16.3.0",
+          "resolved": "https://registry.npmjs.org/vue-loader/-/vue-loader-16.3.0.tgz",
+          "integrity": "sha512-UDgni/tUVSdwHuQo+vuBmEgamWx88SuSlEb5fgdvHrlJSPB9qMBRF6W7bfPWSqDns425Gt1wxAUif+f+h/rWjg==",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "chalk": "^4.1.0",
+            "hash-sum": "^2.0.0",
+            "loader-utils": "^2.0.0"
+          }
         }
       }
     },
@@ -11027,87 +11106,6 @@
         }
       }
     },
-    "vue-loader-v16": {
-      "version": "npm:vue-loader@16.1.2",
-      "resolved": "https://registry.npm.taobao.org/vue-loader/download/vue-loader-16.1.2.tgz?cache=0&sync_timestamp=1608188078235&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fvue-loader%2Fdownload%2Fvue-loader-16.1.2.tgz",
-      "integrity": "sha1-XAO2xQ0qX5g8fOuhXFDXjKKymPQ=",
-      "dev": true,
-      "optional": true,
-      "requires": {
-        "chalk": "^4.1.0",
-        "hash-sum": "^2.0.0",
-        "loader-utils": "^2.0.0"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "4.3.0",
-          "resolved": "https://registry.npm.taobao.org/ansi-styles/download/ansi-styles-4.3.0.tgz?cache=0&sync_timestamp=1606792436886&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fansi-styles%2Fdownload%2Fansi-styles-4.3.0.tgz",
-          "integrity": "sha1-7dgDYornHATIWuegkG7a00tkiTc=",
-          "dev": true,
-          "optional": true,
-          "requires": {
-            "color-convert": "^2.0.1"
-          }
-        },
-        "chalk": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npm.taobao.org/chalk/download/chalk-4.1.0.tgz?cache=0&sync_timestamp=1591687018980&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fchalk%2Fdownload%2Fchalk-4.1.0.tgz",
-          "integrity": "sha1-ThSHCmGNni7dl92DRf2dncMVZGo=",
-          "dev": true,
-          "optional": true,
-          "requires": {
-            "ansi-styles": "^4.1.0",
-            "supports-color": "^7.1.0"
-          }
-        },
-        "color-convert": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npm.taobao.org/color-convert/download/color-convert-2.0.1.tgz",
-          "integrity": "sha1-ctOmjVmMm9s68q0ehPIdiWq9TeM=",
-          "dev": true,
-          "optional": true,
-          "requires": {
-            "color-name": "~1.1.4"
-          }
-        },
-        "color-name": {
-          "version": "1.1.4",
-          "resolved": "https://registry.npm.taobao.org/color-name/download/color-name-1.1.4.tgz",
-          "integrity": "sha1-wqCah6y95pVD3m9j+jmVyCbFNqI=",
-          "dev": true,
-          "optional": true
-        },
-        "has-flag": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npm.taobao.org/has-flag/download/has-flag-4.0.0.tgz",
-          "integrity": "sha1-lEdx/ZyByBJlxNaUGGDaBrtZR5s=",
-          "dev": true,
-          "optional": true
-        },
-        "loader-utils": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npm.taobao.org/loader-utils/download/loader-utils-2.0.0.tgz?cache=0&sync_timestamp=1598867216219&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Floader-utils%2Fdownload%2Floader-utils-2.0.0.tgz",
-          "integrity": "sha1-5MrOW4FtQloWa18JfhDNErNgZLA=",
-          "dev": true,
-          "optional": true,
-          "requires": {
-            "big.js": "^5.2.2",
-            "emojis-list": "^3.0.0",
-            "json5": "^2.1.2"
-          }
-        },
-        "supports-color": {
-          "version": "7.2.0",
-          "resolved": "https://registry.npm.taobao.org/supports-color/download/supports-color-7.2.0.tgz?cache=0&sync_timestamp=1608035619713&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fsupports-color%2Fdownload%2Fsupports-color-7.2.0.tgz",
-          "integrity": "sha1-G33NyzK4E4gBs+R4umpRyqiWSNo=",
-          "dev": true,
-          "optional": true,
-          "requires": {
-            "has-flag": "^4.0.0"
-          }
-        }
-      }
-    },
     "vue-style-loader": {
       "version": "4.1.2",
       "resolved": "https://registry.npm.taobao.org/vue-style-loader/download/vue-style-loader-4.1.2.tgz",

webui/src/components/projects/project-open-window.vue

@@ -20,7 +20,7 @@
           <div class="description">{{ project.description }}</div>
 
           <div>
-            {{ datetime(project.created) }}
+            {{ project.created }}
           </div>
         </div>