import os import uuid from flask import abort from flask import make_response from flask import request from flask.views import View from pycs import db from pycs.database.Project import Project from pycs.frontend.notifications.NotificationManager import NotificationManager from pycs.jobs.JobGroupBusyException import JobGroupBusyException from pycs.jobs.JobRunner import JobRunner from pycs.util.FileOperations import file_info from pycs.util.FileOperations import find_images class ExecuteExternalStorage(View): """ find media files stored in a projects data_folder """ # pylint: disable=arguments-differ methods = ['POST'] def __init__(self, nm: NotificationManager, jobs: JobRunner): # pylint: disable=invalid-name self.nm = nm self.jobs = jobs def dispatch_request(self, user: str, project_id: int): # pylint: disable=unused-argument # extract request data data = request.get_json(force=True) if not data.get('execute', False): return abort(400) # find project project = Project.get_or_404(project_id) if not project.external_data: return abort(400, "External data is not set!") # execute label provider and add labels to project try: ExecuteExternalStorage.find_media_files(self.nm, self.jobs, project) except JobGroupBusyException: return abort(400, "Job is already running!") return make_response() @staticmethod def find_media_files(nm: NotificationManager, jobs: JobRunner, project: Project): """ start a job that finds media files in the projects data_folder and adds them to the database afterwards :param nm: notification manager object :param jobs: job runner object :param project: project :return: """ # pylint: disable=invalid-name # find lists the given data folder and prepares item dictionaries def find(data_folder): image_paths = find_images(data_folder) length = len(image_paths) elements = [] current = 0 for file_path in image_paths: file_folder = str(file_path.parent) file_name = file_path.stem file_extension = file_path.suffix file_size = os.path.getsize(str(file_path)) try: ftype, frames, fps = file_info(file_folder, file_name, file_extension) except ValueError: continue file_uuid = str(uuid.uuid1()) file_attrs = dict( uuid=file_uuid, file_type=ftype, name=file_name, extension=file_extension, size=file_size, filename=file_path.relative_to(data_folder).with_name(file_name), frames=frames, fps=fps) elements.append(file_attrs) current += 1 if len(elements) >= 200: yield elements, current, length elements.clear() if len(elements) > 0: yield elements, current, length # progress inserts elements into the database and fires events def progress(elements, current, length): with db.session.begin_nested(): for file_attrs in elements: file, is_new = project.add_file(commit=False, **file_attrs) if is_new: nm.create_file(file) return current / length # run job with given functions jobs.run(project, 'Find Media Files', project.name, f'{project.id}/find-files', find, project.data_folder, progress=progress)