PyCS/pycs/frontend/endpoints/projects/ExecuteExternalStorage.py

import os
import uuid

from flask import abort
from flask import make_response
from flask import request
from flask.views import View

from pycs import db
from pycs.database.Project import Project
from pycs.frontend.notifications.NotificationManager import NotificationManager
from pycs.jobs.JobGroupBusyException import JobGroupBusyException
from pycs.jobs.JobRunner import JobRunner
from pycs.util.FileOperations import file_info
from pycs.util.FileOperations import find_images


class ExecuteExternalStorage(View):
    """
    find media files stored in a projects data_folder
    """
    # pylint: disable=arguments-differ
    methods = ['POST']

    def __init__(self, nm: NotificationManager, jobs: JobRunner):
        # pylint: disable=invalid-name
        self.nm = nm
        self.jobs = jobs

    def dispatch_request(self, project_id: int):
        # extract request data
        data = request.get_json(force=True)

        if not data.get('execute', False):
            return abort(400)

        # find project
        project = Project.get_or_404(project_id)

        if not project.external_data:
            return abort(400, "External data is not set!")

        # execute label provider and add labels to project
        try:
            ExecuteExternalStorage.find_media_files(self.nm, self.jobs, project)
        except JobGroupBusyException:
            return abort(400, "Job is already running!")

        return make_response()

    @staticmethod
    def find_media_files(nm: NotificationManager, jobs: JobRunner, project: Project):
        """
        start a job that finds media files in the projects data_folder and adds them to the
        database afterwards

        :param nm: notification manager object
        :param jobs: job runner object
        :param project: project
        :return:
        """

        # pylint: disable=invalid-name
        # find lists the given data folder and prepares item dictionaries
        def find(data_folder):
            image_paths = find_images(data_folder)
            length = len(image_paths)

            elements = []
            current = 0

            for file_path in image_paths:

                file_folder = str(file_path.parent)
                file_name = file_path.stem
                file_extension = file_path.suffix
                file_size = os.path.getsize(str(file_path))

                try:
                    ftype, frames, fps = file_info(file_folder, file_name, file_extension)
                except ValueError:
                    continue

                file_uuid = str(uuid.uuid1())
                file_attrs = dict(
                    uuid=file_uuid,
                    file_type=ftype,
                    name=file_name,
                    extension=file_extension,
                    size=file_size,
                    filename=file_path.relative_to(data_folder).with_name(file_name),
                    frames=frames,
                    fps=fps)

                elements.append(file_attrs)
                current += 1

                if len(elements) >= 200:
                    yield elements, current, length
                    elements.clear()

            if len(elements) > 0:
                yield elements, current, length

        # progress inserts elements into the database and fires events
        def progress(elements, current, length):
            with db.session.begin_nested():
                for file_attrs in elements:
                    file, is_new = project.add_file(commit=False, **file_attrs)

                    if is_new:
                        nm.create_file(file)

            return current / length

        # run job with given functions
        jobs.run(project,
                 'Find Media Files',
                 project.name,
                 f'{project.id}/find-files',
                 find, project.data_folder,
                 progress=progress)