implemented label provider execution as bulk update

labels/lepiforum_version_7/Provider.py

@@ -42,6 +42,7 @@ class Provider(LabelProvider):
             hierarchy_levels = (('family', 'Familie'),
                                 ('genus', 'Gattung'))
 
+        parents = set()
         for entry in entries:
             entry = entry.__dict__
             parent_reference = None
@@ -50,12 +51,16 @@ class Provider(LabelProvider):
             for level, level_name in hierarchy_levels:
                 if entry[level] is not None:
                     reference, name = f'{level}_{entry[level].lower()}', entry[level]
-                    result.append(self.create_label(reference, name, parent_reference, level_name))
+
+                    # parents should be added once
+                    if reference not in parents:
+                        result.append(self.create_label(reference, name, parent_reference, level_name))
+                        parents.add(reference)
 
                     parent_reference = reference
 
             # add element
-            if entry['kr_number'].isnumeric():
+            if entry['kr_number'].isalnum():
                 name = f'{entry["genus"]} {entry["species"]} ({entry["kr_number"]})'
                 reference = entry['kr_number']
             else:

pycs/database/Project.py

@@ -5,6 +5,7 @@ import warnings
 
 from datetime import datetime
 
+from pycs import app
 from pycs import db
 from pycs.database.base import NamedBaseModel
 
@@ -220,6 +221,56 @@ class Project(NamedBaseModel):
 
         return label, is_new
 
+    @commit_on_return
+    def bulk_create_labels(self, labels: T.List[T.Dict], clean_old_labels: bool = True):
+        """
+            Inserts a all labels at once.
+
+            :raises:
+                - AssertionError if project_id and reference are not unique
+                - ValueError if a cycle in the hierarchy is found
+        """
+        if clean_old_labels:
+            self.labels.delete()
+
+        for label in labels:
+            label["project_id"] = self.id
+
+        self.__check_labels(labels)
+        app.logger.info(f"Inserting {len(labels):,d} labels")
+        db.engine.execute(Label.__table__.insert(), labels)
+        self.__set_parents(labels)
+
+        return labels
+
+    def __set_parents(self, labels):
+        """ after the bul insert, we need to set correct parent_ids """
+        app.logger.info("Setting parents of the labels")
+
+        self.flush()
+        for label in labels:
+            if label["parent"] is None:
+                continue
+
+            label_obj = self.label_by_reference(label["reference"])
+            parent_label_obj = self.label_by_reference(label["parent"])
+
+            label_obj.parent_id = parent_label_obj.id
+
+    def __check_labels(self, labels):
+        """ check labels for unique keys and cycles """
+
+        unique_keys = dict()
+
+        for label in labels:
+            key = (label["project_id"], label["reference"])
+
+            assert key not in unique_keys, \
+                f"{key} was not unique: ({label=} vs {unique_keys[key]=})!"
+
+            unique_keys[key] = label
+
+
 
     # pylint: disable=too-many-arguments
     @commit_on_return

pycs/frontend/endpoints/projects/ExecuteLabelProvider.py

@@ -7,14 +7,17 @@ from flask.views import View
 
 from pycs import db
 from pycs.database.LabelProvider import LabelProvider
+from pycs.database.Label import Label
 from pycs.database.Project import Project
 from pycs.frontend.notifications.NotificationManager import NotificationManager
 from pycs.jobs.JobGroupBusyException import JobGroupBusyException
 from pycs.jobs.JobRunner import JobRunner
 
+from tqdm import tqdm
+
 
 class ExecuteLabelProvider(View):
-    """
+    """db
     execute the label provider associated with a passed project identifier
     """
     # pylint: disable=arguments-differ
@@ -68,17 +71,12 @@ class ExecuteLabelProvider(View):
             with closing(label_provider.load()) as label_provider_impl:
                 return label_provider_impl.get_labels()
 
+        project_id = project.id
         # result adds the received labels to the database and fires events
         def result(provided_labels):
-            with db.session.begin_nested():
+            with db.session.begin():
-                for label in provided_labels:
+                project = Project.query.get(project_id)
-                    created_label, is_new = project.create_label(commit=False, **label)
+                labels = project.bulk_create_labels(provided_labels)
-                    project.flush()
-
-                    if is_new:
-                        nm.create_label(created_label)
-                    else:
-                        nm.edit_label(created_label)
 
         # run job with given functions
         jobs.run(project,

pycs/frontend/notifications/NotificationManager.py

@@ -30,7 +30,7 @@ class NotificationManager:
         :param created_job:
         :return:
         """
-        app.logger.info(created_job)
+        app.logger.debug(created_job)
         self.__emit('create-job', created_job)
 
     def edit_job(self, edited_job: Job):
@@ -40,7 +40,7 @@ class NotificationManager:
         :param edited_job:
         :return:
         """
-        app.logger.info(edited_job)
+        app.logger.debug(edited_job)
         self.__emit('edit-job', edited_job)
 
     def remove_job(self, removed_job: Job):
@@ -50,7 +50,7 @@ class NotificationManager:
         :param removed_job:
         :return:
         """
-        app.logger.info(removed_job)
+        app.logger.debug(removed_job)
         self.__emit('remove-job', removed_job)
 
     def create_model(self, created_model: Model):
@@ -60,7 +60,7 @@ class NotificationManager:
         :param created_model:
         :return:
         """
-        app.logger.info(created_model)
+        app.logger.debug(created_model)
         self.__emit('create-model', created_model)
 
     def remove_model(self, removed_model: Model):
@@ -70,7 +70,7 @@ class NotificationManager:
         :param removed_model:
         :return:
         """
-        app.logger.info(removed_model)
+        app.logger.debug(removed_model)
         self.__emit('remove-model', removed_model)
 
     def create_project(self, created_project: Project):
@@ -80,7 +80,7 @@ class NotificationManager:
         :param created_project:
         :return:
         """
-        app.logger.info(created_project)
+        app.logger.debug(created_project)
         self.__emit('create-project', created_project)
 
     def remove_project(self, removed_project: Project):
@@ -90,7 +90,7 @@ class NotificationManager:
         :param removed_project:
         :return:
         """
-        app.logger.info(removed_project)
+        app.logger.debug(removed_project)
         self.__emit('remove-project', removed_project)
 
     def edit_project(self, edited_project: Project):
@@ -100,7 +100,7 @@ class NotificationManager:
         :param edited_project:
         :return:
         """
-        app.logger.info(edited_project)
+        app.logger.debug(edited_project)
         self.__emit('edit-project', edited_project)
 
     def create_label(self, created_label: Label):
@@ -110,7 +110,7 @@ class NotificationManager:
         :param created_label:
         :return:
         """
-        app.logger.info(created_label)
+        app.logger.debug(created_label)
         self.__emit('create-label', created_label)
 
     def edit_label(self, edited_label: Label):
@@ -120,7 +120,7 @@ class NotificationManager:
         :param edited_label:
         :return:
         """
-        app.logger.info(edited_label)
+        app.logger.debug(edited_label)
         self.__emit('edit-label', edited_label)
 
     def remove_label(self, removed_label: Label):
@@ -130,7 +130,7 @@ class NotificationManager:
         :param removed_label:
         :return:
         """
-        app.logger.info(removed_label)
+        app.logger.debug(removed_label)
         self.__emit('remove-label', removed_label)
 
     def create_file(self, created_file: File):
@@ -140,7 +140,7 @@ class NotificationManager:
         :param created_file:
         :return:
         """
-        app.logger.info(created_file)
+        app.logger.debug(created_file)
         self.__emit('create-file', created_file)
 
     def edit_file(self, edited_file: File):
@@ -150,7 +150,7 @@ class NotificationManager:
         :param edited_file:
         :return:
         """
-        app.logger.info(edited_file)
+        app.logger.debug(edited_file)
         self.__emit('edit-file', edited_file)
 
     def remove_file(self, removed_file: File):
@@ -160,7 +160,7 @@ class NotificationManager:
         :param removed_file:
         :return:
         """
-        app.logger.info(removed_file)
+        app.logger.debug(removed_file)
         self.__emit('remove-file', removed_file)
 
     def create_result(self, created_result: Result):
@@ -170,7 +170,7 @@ class NotificationManager:
         :param created_result:
         :return:
         """
-        app.logger.info(created_result)
+        app.logger.debug(created_result)
         self.__emit('create-result', created_result)
 
     def edit_result(self, edited_result: Result):
@@ -180,7 +180,7 @@ class NotificationManager:
         :param edited_result:
         :return:
         """
-        app.logger.info(edited_result)
+        app.logger.debug(edited_result)
         self.__emit('edit-result', edited_result)
 
     def remove_result(self, removed_result: Result):
@@ -190,5 +190,5 @@ class NotificationManager:
         :param removed_result:
         :return:
         """
-        app.logger.info(removed_result)
+        app.logger.debug(removed_result)
         self.__emit('remove-result', removed_result)