PyCS/pycs/frontend/endpoints/projects/ExecuteExternalStorage.py

from os import listdir
from os import path
from os.path import isfile
from uuid import uuid1

from flask import make_response, request, abort
from flask.views import View

from pycs.database.Database import Database
from pycs.database.Project import Project
from pycs.frontend.notifications.NotificationManager import NotificationManager
from pycs.jobs.JobGroupBusyException import JobGroupBusyException
from pycs.jobs.JobRunner import JobRunner
from pycs.util.FileParser import file_info


class ExecuteExternalStorage(View):
    """
    find media files stored in a projects data_folder
    """
    # pylint: disable=arguments-differ
    methods = ['POST']

    def __init__(self, db: Database, nm: NotificationManager, jobs: JobRunner):
        # pylint: disable=invalid-name
        self.db = db
        self.nm = nm
        self.jobs = jobs

    def dispatch_request(self, identifier):
        # extract request data
        data = request.get_json(force=True)

        if 'execute' not in data or data['execute'] is not True:
            return abort(400)

        # find project
        project = self.db.project(identifier)
        if project is None:
            return abort(404)

        if not project.external_data:
            return abort(400)

        # execute label provider and add labels to project
        try:
            self.find_media_files(self.db, self.nm, self.jobs, project)
        except JobGroupBusyException:
            return abort(400)

        return make_response()

    @staticmethod
    def find_media_files(db: Database, nm: NotificationManager, jobs: JobRunner, project: Project):
        """
        start a job that finds media files in the projects data_folder and adds them to the
        database afterwards

        :param db: database object
        :param nm: notification manager object
        :param jobs: job runner object
        :param project: project
        :return:
        """

        # pylint: disable=invalid-name
        # find lists the given data folder and prepares item dictionaries
        def find():
            files = listdir(project.data_folder)
            length = len(files)

            elements = []
            current = 0

            for file_name in files:
                file_path = path.join(project.data_folder, file_name)
                if not isfile(file_path):
                    continue

                file_name, file_extension = path.splitext(file_name)
                file_size = path.getsize(file_path)

                try:
                    ftype, frames, fps = file_info(project.data_folder, file_name, file_extension)
                except ValueError:
                    continue

                elements.append((ftype, file_name, file_extension, file_size, frames, fps))
                current += 1

                if len(elements) >= 200:
                    yield elements, current, length
                    elements = []

            if len(elements) > 0:
                yield elements, current, length

        # progress inserts elements into the database and fires events
        def progress(elements, current, length):
            with db:
                for ftype, file_name, file_extension, file_size, frames, fps in elements:
                    uuid = str(uuid1())
                    file, insert = project.add_file(uuid, ftype, file_name, file_extension,
                                                    file_size, file_name, frames, fps)

                    if insert:
                        nm.create_file(file)

            return current / length

        # run job with given functions
        jobs.run(project,
                 'Find Media Files',
                 project.name,
                 f'{project.identifier}/find-files',
                 find,
                 progress=progress)