Resolve "integrate jobs into project objects"

app.py

@@ -1,7 +1,6 @@
 from pycs.ApplicationStatus import ApplicationStatus
 from pycs.frontend.WebServer import WebServer
 from pycs.models.ModelManager import ModelManager
-from pycs.pipeline.PipelineManager import PipelineManager
 from pycs.projects.ProjectManager import ProjectManager
 
 if __name__ == '__main__':
@@ -13,14 +12,10 @@ if __name__ == '__main__':
     print('- load model manager')
     model_manager = ModelManager(app_status)
 
-    # load pipeline manager
-    print('- load pipeline manager')
-    pipeline_manager = PipelineManager(app_status)
-
     # load project manager
     print('- load project manager')
     project_manager = ProjectManager(app_status)
 
     # start web server
     print('- start web server')
-    web_server = WebServer(app_status, pipeline_manager)
+    web_server = WebServer(app_status)

pycs/ApplicationStatus.py

@@ -21,8 +21,7 @@ class ApplicationStatus(ObservableDict):
         super().__init__({
             'settings': settings,
             'models': {},
-            'projects': {},
-            'jobs': []
+            'projects': {}
         })
 
         # subscribe to settings change to write it to file

pycs/frontend/WebServer.py

@@ -2,7 +2,6 @@ from glob import glob
 from os import path, mkdir, getcwd
 from os.path import exists
 from time import time
-from uuid import uuid1
 
 import eventlet
 import socketio
@@ -10,7 +9,6 @@ from flask import Flask, make_response, send_from_directory, request
 from werkzeug import formparser
 
 from pycs.ApplicationStatus import ApplicationStatus
-from pycs.pipeline.PipelineManager import PipelineManager
 from pycs.util.GenericWrapper import GenericWrapper
 from pycs.util.ProgressFileWriter import ProgressFileWriter
 from pycs.util.RecursiveDictionary import set_recursive
@@ -106,14 +104,15 @@ class WebServer:
                 file_path = path.join(upload_path, f'{file_uuid}{file_extension.value}')
 
                 # add job to app status
-                job.value = app_status['jobs'].append({
+                project['jobs'][file_uuid] = {
                     'id': file_uuid,
                     'type': 'upload',
                     'progress': 0,
                     'filename': filename,
                     'created': int(time()),
                     'finished': None
-                })
+                }
+                job.value = project['jobs'][file_uuid]
 
                 # create upload path if not exists
                 if not path.exists(upload_path):

pycs/observable/Observable.py

@@ -25,9 +25,9 @@ class Observable:
         if immediate:
             handler(self)
 
-    def _notify(self):
+    def notify(self):
         for s in self.subscriptions:
             s(self)
 
         if self.parent is not None:
-            self.parent._notify()
+            self.parent.notify()

pycs/observable/ObservableDict.py

@@ -1,4 +1,4 @@
-from .Observable import Observable
+from . import Observable
 
 
 class ObservableDict(dict, Observable):
@@ -11,8 +11,21 @@ class ObservableDict(dict, Observable):
 
     def __setitem__(self, key, value):
         dict.__setitem__(self, key, Observable.create(value, self))
-        Observable._notify(self)
+        Observable.notify(self)
 
     def __delitem__(self, key):
         super().__delitem__(key)
-        Observable._notify(self)
+        Observable.notify(self)
+
+    def copy(self):
+        result = {}
+        for key in self.keys():
+            if type(self[key]) == ObservableDict or type(self[key]) == ObservableList:
+                result[key] = self[key].copy()
+            else:
+                result[key] = self[key]
+
+        return result
+
+
+from . import ObservableList

pycs/observable/ObservableList.py

@@ -1,4 +1,4 @@
-from .Observable import Observable
+from . import Observable
 
 
 class ObservableList(list, Observable):
@@ -14,16 +14,28 @@ class ObservableList(list, Observable):
 
     def __setitem__(self, key, value):
         super().__setitem__(key, Observable.create(value, self))
-        Observable._notify(self)
+        Observable.notify(self)
 
     def __delitem__(self, key):
         super().__delitem__(key)
-        Observable._notify(self)
+        Observable.notify(self)
 
     def append(self, value):
         obs = Observable.create(value, self)
 
         super().append(obs)
-        Observable._notify(self)
+        Observable.notify(self)
 
         return obs
+
+    def __copy__(self):
+        def c(e):
+            if isinstance(e, (ObservableDict, ObservableList)):
+                return e.copy()
+            else:
+                return e
+
+        return list(map(c, self))
+
+
+from . import ObservableDict

pycs/pipeline/PipelineManager.py

@@ -3,7 +3,6 @@ from os import path
 
 from eventlet import tpool
 
-from pycs.ApplicationStatus import ApplicationStatus
 from pycs.pipeline.Job import Job
 from pycs.pipeline.tf1.pipeline import Pipeline as TF1Pipeline
 from pycs.projects.Project import Project

pycs/projects/ProjectManager.py

@@ -26,11 +26,15 @@ class ProjectManager(ObservableDict):
             # load project.json
             with open(path.join(folder, 'project.json'), 'r') as file:
                 project = Project(load(file), self)
+                project['jobs'] = {}
                 self[project['id']] = project
 
     def __write_project(self, uuid):
         with open(path.join('projects', uuid, 'project.json'), 'w') as file:
-            dump(self[uuid], file, indent=4)
+            copy = self[uuid].copy()
+            del copy['jobs']
+
+            dump(copy, file, indent=4)
 
     def create_project(self, name, description, model):
         # create dict representation
@@ -43,7 +47,8 @@ class ProjectManager(ObservableDict):
             'pipeline': {
                 'model-distribution': model
             },
-            'data': {}
+            'data': {},
+            'jobs': {}
         }, self)
 
         # create project directory

test/test_application_status.py

@@ -9,8 +9,7 @@ class TestApplicationStatus(unittest.TestCase):
         self.assertEqual({
             'settings': {},
             'models': {},
-            'projects': {},
-            'jobs': []
+            'projects': {}
         }, aso)
 
     def test_load_from_object(self):
@@ -23,8 +22,7 @@ class TestApplicationStatus(unittest.TestCase):
         self.assertEqual({
             'settings': settings,
             'models': {},
-            'projects': {},
-            'jobs': []
+            'projects': {}
         }, aso)
 
 

webui/src/App.vue

@@ -31,7 +31,6 @@
 
         <project-data-add-window v-if="window.content === 'add_data'"
                                  :current-project="currentProject"
-                                 :status="status"
                                  :socket="socket"/>
 
         <project-data-view-window v-if="window.content === 'view_data'"

webui/src/components/projects/project-data-add-window.vue

@@ -23,11 +23,13 @@ import ProgressBar from "@/components/base/progress-bar";
 export default {
   name: "project-data-add-window",
   components: {ProgressBar, FileInput},
-  props: ['status', 'socket', 'currentProject'],
+  props: ['socket', 'currentProject'],
   computed: {
     uploads: function() {
-      // TODO filter current project
-      const filtered = this.status.jobs.filter(x => x.type === 'upload');
+      const filtered = Object.keys(this.currentProject.jobs)
+          .map(e => this.currentProject.jobs[e])
+          .filter(e => e.type === 'upload');
+
       filtered.sort(this.sortUploads);
       return filtered;
     }