WebServer.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. import logging.config
  2. import typing as T
  3. from glob import glob
  4. from pathlib import Path
  5. import eventlet
  6. import munch
  7. import socketio
  8. from flask import send_from_directory
  9. from pycs.database.Model import Model
  10. from pycs.database.LabelProvider import LabelProvider
  11. from pycs.frontend.endpoints.ListJobs import ListJobs
  12. from pycs.frontend.endpoints.ListLabelProviders import ListLabelProviders
  13. from pycs.frontend.endpoints.ListModels import ListModels
  14. from pycs.frontend.endpoints.ListProjects import ListProjects
  15. from pycs.frontend.endpoints.additional.FolderInformation import FolderInformation
  16. from pycs.frontend.endpoints.data.GetCroppedFile import GetCroppedFile
  17. from pycs.frontend.endpoints.data.GetFile import GetFile
  18. from pycs.frontend.endpoints.data.GetPreviousAndNextFile import GetPreviousAndNextFile
  19. from pycs.frontend.endpoints.data.GetResizedFile import GetResizedFile
  20. from pycs.frontend.endpoints.data.RemoveFile import RemoveFile
  21. from pycs.frontend.endpoints.data.UploadFile import UploadFile
  22. from pycs.frontend.endpoints.jobs.RemoveJob import RemoveJob
  23. from pycs.frontend.endpoints.labels.CreateLabel import CreateLabel
  24. from pycs.frontend.endpoints.labels.EditLabelName import EditLabelName
  25. from pycs.frontend.endpoints.labels.EditLabelParent import EditLabelParent
  26. from pycs.frontend.endpoints.labels.ListLabelTree import ListLabelTree
  27. from pycs.frontend.endpoints.labels.ListLabels import ListLabels
  28. from pycs.frontend.endpoints.labels.RemoveLabel import RemoveLabel
  29. from pycs.frontend.endpoints.pipelines.FitModel import FitModel
  30. from pycs.frontend.endpoints.pipelines.PredictFile import PredictFile
  31. from pycs.frontend.endpoints.pipelines.PredictModel import PredictModel
  32. from pycs.frontend.endpoints.projects.CreateProject import CreateProject
  33. from pycs.frontend.endpoints.projects.EditProjectDescription import EditProjectDescription
  34. from pycs.frontend.endpoints.projects.EditProjectName import EditProjectName
  35. from pycs.frontend.endpoints.projects.ExecuteExternalStorage import ExecuteExternalStorage
  36. from pycs.frontend.endpoints.projects.ExecuteLabelProvider import ExecuteLabelProvider
  37. from pycs.frontend.endpoints.projects.GetProjectModel import GetProjectModel
  38. from pycs.frontend.endpoints.projects.ListProjectCollections import ListProjectCollections
  39. from pycs.frontend.endpoints.projects.ListProjectFiles import ListProjectFiles
  40. from pycs.frontend.endpoints.projects.RemoveProject import RemoveProject
  41. from pycs.frontend.endpoints.results.ConfirmResult import ConfirmResult
  42. from pycs.frontend.endpoints.results.CreateResult import CreateResult
  43. from pycs.frontend.endpoints.results.EditResultData import EditResultData
  44. from pycs.frontend.endpoints.results.EditResultLabel import EditResultLabel
  45. from pycs.frontend.endpoints.results.GetProjectResults import GetProjectResults
  46. from pycs.frontend.endpoints.results.GetResults import GetResults
  47. from pycs.frontend.endpoints.results.RemoveResult import RemoveResult
  48. from pycs.frontend.endpoints.results.ResetResults import ResetResults
  49. from pycs.frontend.notifications.NotificationManager import NotificationManager
  50. from pycs.frontend.util.JSONEncoder import JSONEncoder
  51. from pycs.jobs.JobRunner import JobRunner
  52. from pycs.util.PipelineCache import PipelineCache
  53. class WebServer:
  54. """
  55. wrapper class for flask and socket.io which initializes most networking
  56. """
  57. index: Path = Path.cwd() / 'webui' / 'index.html'
  58. def __init__(self, app, settings: munch.Munch, discovery: bool = True):
  59. logging.config.dictConfig(settings.logging)
  60. self.app = app
  61. # set json encoder so database objects are serialized correctly
  62. self.app.json_encoder = JSONEncoder
  63. # initialize web server
  64. if self.is_production:
  65. app.logger.info('production build')
  66. # overwrite root path to serve index.html
  67. @self.app.route('/', methods=['GET'])
  68. def index():
  69. # pylint: disable=unused-variable
  70. return send_from_directory(str(self.index.parent), self.index.name)
  71. else:
  72. app.logger.info('development build')
  73. # set access control header to allow requests from Vue.js development server
  74. @self.app.after_request
  75. def after_request(response):
  76. # pylint: disable=unused-variable
  77. response.headers['Access-Control-Allow-Origin'] = '*'
  78. return response
  79. # create service objects
  80. self.sio = socketio.Server(**self.sio_kwargs(settings.allowedOrigins))
  81. self.wsgi_app = socketio.WSGIApp(self.sio, app, static_files=self.static_files)
  82. self.host = settings.host
  83. self.port = settings.port
  84. # create notification manager
  85. self.jobs = JobRunner()
  86. self.pipelines = PipelineCache(self.jobs, settings.get("pipeline_cache_time"))
  87. self.notifications = NotificationManager(self.sio)
  88. self.jobs.on_create(self.notifications.create_job)
  89. self.jobs.on_start(self.notifications.edit_job)
  90. self.jobs.on_progress(self.notifications.edit_job)
  91. self.jobs.on_finish(self.notifications.edit_job)
  92. self.jobs.on_remove(self.notifications.remove_job)
  93. self.define_routes()
  94. if discovery:
  95. Model.discover("models/")
  96. LabelProvider.discover("labels/")
  97. def sio_kwargs(self, allowed_origins) -> T.Dict[str, T.Union[str, list]]:
  98. """keyword arguments for the socketio.Server depending on the mode"""
  99. kwargs: T.Dict[str, T.Union[str, list]] = dict(async_mode="eventlet")
  100. if self.is_production:
  101. if isinstance(allowed_origins, list) and len(allowed_origins) > 0:
  102. kwargs["cors_allowed_origins"] = allowed_origins
  103. else:
  104. kwargs["cors_allowed_origins"] = "*"
  105. return kwargs
  106. @property
  107. def is_production(self) -> bool:
  108. """property checking, whether the UI is built (production mode)
  109. or served by npm serve (development mode)"""
  110. return self.index.exists()
  111. @property
  112. def static_files(self) -> T.Optional[T.Dict[str, T.Union[str, dict]]]:
  113. """returns a dictionary of static files (production mode)
  114. or None (development mode)"""
  115. if not self.is_production:
  116. return None
  117. # find static files and folders
  118. static_files: T.Dict[str, T.Union[str, dict]] = {}
  119. for file_path in glob('webui/*'):
  120. file_path = file_path.replace('\\', '/')
  121. static_files[file_path[5:]] = file_path
  122. # separately add svg files and set their correct mime type
  123. for svg_path in glob('webui/img/*.svg'):
  124. svg_path = svg_path.replace('\\', '/')
  125. static_files[svg_path[5:]] = {'content_type': 'image/svg+xml', 'filename': svg_path}
  126. return static_files
  127. def define_routes(self):
  128. """ defines app routes """
  129. # additional
  130. self.app.add_url_rule(
  131. '/folder',
  132. view_func=FolderInformation.as_view('folder_information')
  133. )
  134. # jobs
  135. self.app.add_url_rule(
  136. '/jobs',
  137. view_func=ListJobs.as_view('list_jobs', self.jobs)
  138. )
  139. self.app.add_url_rule(
  140. '/jobs/<job_id>/remove',
  141. view_func=RemoveJob.as_view('remove_job', self.jobs)
  142. )
  143. # models
  144. self.app.add_url_rule(
  145. '/models',
  146. view_func=ListModels.as_view('list_models')
  147. )
  148. self.app.add_url_rule(
  149. '/projects/<int:project_id>/model',
  150. view_func=GetProjectModel.as_view('get_project_model')
  151. )
  152. # labels
  153. self.app.add_url_rule(
  154. '/label_providers',
  155. view_func=ListLabelProviders.as_view('label_providers')
  156. )
  157. self.app.add_url_rule(
  158. '/projects/<int:project_id>/labels',
  159. view_func=ListLabels.as_view('list_labels')
  160. )
  161. self.app.add_url_rule(
  162. '/projects/<int:project_id>/labels/tree',
  163. view_func=ListLabelTree.as_view('list_label_tree')
  164. )
  165. self.app.add_url_rule(
  166. '/projects/<int:project_id>/labels',
  167. view_func=CreateLabel.as_view('create_label', self.notifications)
  168. )
  169. self.app.add_url_rule(
  170. '/projects/<int:project_id>/labels/<int:label_id>/remove',
  171. view_func=RemoveLabel.as_view('remove_label', self.notifications)
  172. )
  173. self.app.add_url_rule(
  174. '/projects/<int:project_id>/labels/<int:label_id>/name',
  175. view_func=EditLabelName.as_view('edit_label_name', self.notifications)
  176. )
  177. self.app.add_url_rule(
  178. '/projects/<int:project_id>/labels/<int:label_id>/parent',
  179. view_func=EditLabelParent.as_view('edit_label_parent', self.notifications)
  180. )
  181. # collections
  182. self.app.add_url_rule(
  183. '/projects/<int:project_id>/collections',
  184. view_func=ListProjectCollections.as_view('list_collections')
  185. )
  186. self.app.add_url_rule(
  187. '/projects/<int:project_id>/data/<int:collection_id>/<int:start>/<int:length>',
  188. view_func=ListProjectFiles.as_view('list_collection_files')
  189. )
  190. # data
  191. self.app.add_url_rule(
  192. '/projects/<int:project_id>/data',
  193. view_func=UploadFile.as_view('upload_file', self.notifications)
  194. )
  195. self.app.add_url_rule(
  196. '/projects/<int:project_id>/data',
  197. view_func=ListProjectFiles.as_view('list_all_files')
  198. )
  199. self.app.add_url_rule(
  200. '/projects/<int:project_id>/data/<int:start>/<int:length>',
  201. view_func=ListProjectFiles.as_view('list_files')
  202. )
  203. self.app.add_url_rule(
  204. '/data/<int:file_id>/remove',
  205. view_func=RemoveFile.as_view('remove_file', self.notifications)
  206. )
  207. self.app.add_url_rule(
  208. '/data/<int:file_id>',
  209. view_func=GetFile.as_view('get_file')
  210. )
  211. self.app.add_url_rule(
  212. '/data/<int:file_id>/<resolution>',
  213. view_func=GetResizedFile.as_view('get_resized_file')
  214. )
  215. self.app.add_url_rule(
  216. '/data/<int:file_id>/<resolution>/<crop_box>',
  217. view_func=GetCroppedFile.as_view('get_cropped_file')
  218. )
  219. self.app.add_url_rule(
  220. '/data/<int:file_id>/previous_next',
  221. view_func=GetPreviousAndNextFile.as_view('get_previous_and_next_file')
  222. )
  223. # results
  224. self.app.add_url_rule(
  225. '/projects/<int:project_id>/results',
  226. view_func=GetProjectResults.as_view('get_project_results')
  227. )
  228. self.app.add_url_rule(
  229. '/data/<int:file_id>/results',
  230. view_func=GetResults.as_view('get_results')
  231. )
  232. self.app.add_url_rule(
  233. '/data/<int:file_id>/results',
  234. view_func=CreateResult.as_view('create_result', self.notifications)
  235. )
  236. self.app.add_url_rule(
  237. '/data/<int:file_id>/reset',
  238. view_func=ResetResults.as_view('reset_results', self.notifications)
  239. )
  240. self.app.add_url_rule(
  241. '/results/<int:result_id>/remove',
  242. view_func=RemoveResult.as_view('remove_result', self.notifications)
  243. )
  244. self.app.add_url_rule(
  245. '/results/<int:result_id>/confirm',
  246. view_func=ConfirmResult.as_view('confirm_result', self.notifications)
  247. )
  248. self.app.add_url_rule(
  249. '/results/<int:result_id>/label',
  250. view_func=EditResultLabel.as_view('edit_result_label', self.notifications)
  251. )
  252. self.app.add_url_rule(
  253. '/results/<int:result_id>/data',
  254. view_func=EditResultData.as_view('edit_result_data', self.notifications)
  255. )
  256. # projects
  257. self.app.add_url_rule(
  258. '/projects',
  259. view_func=ListProjects.as_view('list_projects')
  260. )
  261. self.app.add_url_rule(
  262. '/projects',
  263. view_func=CreateProject.as_view('create_project', self.notifications, self.jobs)
  264. )
  265. self.app.add_url_rule(
  266. '/projects/<int:project_id>/label_provider',
  267. view_func=ExecuteLabelProvider.as_view('execute_label_provider',
  268. self.notifications, self.jobs)
  269. )
  270. self.app.add_url_rule(
  271. '/projects/<int:project_id>/external_storage',
  272. view_func=ExecuteExternalStorage.as_view('execute_external_storage',
  273. self.notifications, self.jobs)
  274. )
  275. self.app.add_url_rule(
  276. '/projects/<int:project_id>/remove',
  277. view_func=RemoveProject.as_view('remove_project', self.notifications)
  278. )
  279. self.app.add_url_rule(
  280. '/projects/<int:project_id>/name',
  281. view_func=EditProjectName.as_view('edit_project_name', self.notifications)
  282. )
  283. self.app.add_url_rule(
  284. '/projects/<int:project_id>/description',
  285. view_func=EditProjectDescription.as_view('edit_project_description', self.notifications)
  286. )
  287. # pipelines
  288. self.app.add_url_rule(
  289. '/projects/<int:project_id>/pipelines/fit',
  290. view_func=FitModel.as_view('fit_model', self.jobs, self.pipelines)
  291. )
  292. self.app.add_url_rule(
  293. '/projects/<int:project_id>/pipelines/predict',
  294. view_func=PredictModel.as_view('predict_model', self.notifications, self.jobs,
  295. self.pipelines)
  296. )
  297. self.app.add_url_rule(
  298. '/data/<int:file_id>/predict',
  299. view_func=PredictFile.as_view('predict_file', self.notifications,
  300. self.jobs, self.pipelines)
  301. )
  302. def run(self):
  303. """ start web server """
  304. self.pipelines.start()
  305. eventlet.wsgi.server(eventlet.listen((self.host, self.port)), self.wsgi_app)