PyCS/pycs/frontend/endpoints/projects/ExecuteExternalStorage.py

import os

from uuid import uuid1

from flask import abort
from flask import make_response
from flask import request
from flask.views import View

from pycs import db
from pycs.database.Project import Project
from pycs.database.File import File
from pycs.frontend.notifications.NotificationManager import NotificationManager
from pycs.jobs.JobGroupBusyException import JobGroupBusyException
from pycs.jobs.JobRunner import JobRunner
from pycs.util.FileParser import file_info


class ExecuteExternalStorage(View):
    """
    find media files stored in a projects data_folder
    """
    # pylint: disable=arguments-differ
    methods = ['POST']


    def dispatch_request(self, identifier):
        # extract request data
        data = request.get_json(force=True)

        if not data.get('execute', False):
            return abort(400)

        # find project
        project = Project.query.get(identifier)
        if project is None:
            return abort(404)

        if not project.external_data:
            return abort(400)

        # execute label provider and add labels to project
        try:
            ExecuteExternalStorage.find_media_files(project)

        except JobGroupBusyException:
            return abort(400)

        return make_response()

    @staticmethod
    def find_media_files(project: Project):
        """
        start a job that finds media files in the projects data_folder and adds them to the
        database afterwards

        :param project: project
        :return:
        """

        data_folder = project.data_folder
        project_id = project.id

        # pylint: disable=invalid-name
        # find lists the given data folder and prepares item dictionaries
        def find():
            files = os.listdir(data_folder)
            length = len(files)

            elements = []
            current = 0

            for file_name in files:
                file_path = os.path.join(data_folder, file_name)
                if not os.path.isfile(file_path):
                    continue

                file_name, file_extension = os.path.splitext(file_name)
                file_size = os.path.getsize(file_path)

                try:
                    ftype, frames, fps = file_info(data_folder, file_name, file_extension)
                except ValueError:
                    continue

                elements.append((ftype, file_name, file_extension, file_size, frames, fps))
                current += 1

                if len(elements) >= 200:
                    yield elements, current, length
                    elements = []

            if len(elements) > 0:
                yield elements, current, length

        # progress inserts elements into the database and fires events
        def progress(elements, current, length):
            with db.session.begin_nested():
                project = Project.query.get(project_id)
                for ftype, file_name, file_extension, file_size, frames, fps in elements:
                    file, is_new = project.add_file(str(uuid1()), ftype, file_name,
                                                    file_extension, file_size, file_name,
                                                    frames, fps)

                    if is_new:
                        NotificationManager.created("file", file.id, File)

            return current / length

        # run job with given functions
        JobRunner.run(project,
                      'Find Media Files',
                      project.name,
                      f'{project.id}/find-files',
                      find,
                      progress=progress)