Resolve "support for video files"

pycs/frontend/WebServer.py

@@ -148,13 +148,7 @@ class WebServer:
             job['finished'] = int(time())
 
             # add to project files
-            project.add_media_file({
-                'id': file_uuid,
-                'name': file_name.value,
-                'extension': file_extension.value,
-                'size': file_size.value,
-                'created': job['created']
-            })
+            project.add_media_file(file_uuid, file_name.value, file_extension.value, file_size.value, job['created'])
 
             # return default success response
             return response()

pycs/projects/ImageFile.py

@@ -0,0 +1,42 @@
+from os import path
+
+from PIL import Image
+
+from pycs.projects.MediaFile import MediaFile
+
+
+class ImageFile(MediaFile):
+    def __init__(self, obj, project_id):
+        obj['type'] = 'image'
+        super().__init__(obj, project_id)
+
+    def resize(self, maximum_width):
+        # check if resized file already exists
+        resized = MediaFile(self.copy(), self.project_id)
+        resized['id'] = self['id'] + '-' + maximum_width
+
+        target_directory, target_name = self._get_file(resized['id'])
+        target_path = path.join(target_directory, target_name)
+
+        if path.exists(target_path):
+            return resized
+
+        # load full size image
+        current_directory, current_name = self.get_file()
+        image = Image.open(path.join(current_directory, current_name))
+        image_width, image_height = image.size
+
+        # calculate target height
+        maximum_width = int(maximum_width)
+        maximum_height = int(maximum_width * image_height / image_width)
+
+        # return self if requested size is larger than the image
+        if image_width < maximum_width:
+            return self
+
+        # resize image
+        resized_image = image.resize((maximum_width, maximum_height))
+
+        # save to file
+        resized_image.save(target_path, quality=80)
+        return resized

pycs/projects/MediaFile.py

@@ -1,8 +1,6 @@
 from os import path, getcwd
 from uuid import uuid1
 
-from PIL import Image
-
 from pycs.observable import ObservableDict
 
 
@@ -14,14 +12,14 @@ class MediaFile(ObservableDict):
         self.project_id = project_id
         super().__init__(obj)
 
-    def __get_file(self, identifier):
+    def _get_file(self, identifier):
         file_directory = path.join(getcwd(), 'projects', self.project_id, 'data')
         file_name = identifier + self['extension']
 
         return file_directory, file_name
 
     def get_file(self):
-        return self.__get_file(self['id'])
+        return self._get_file(self['id'])
 
     def add_global_result(self, result, origin='user'):
         self.remove_global_result()
@@ -58,32 +56,4 @@ class MediaFile(ObservableDict):
         self['predictionResults'][identifier] = result
 
     def resize(self, maximum_width):
-        # check if resized file already exists
-        resized = MediaFile(self.copy(), self.project_id)
-        resized['id'] = self['id'] + '-' + maximum_width
-
-        target_directory, target_name = self.__get_file(resized['id'])
-        target_path = path.join(target_directory, target_name)
-
-        if path.exists(target_path):
-            return resized
-
-        # load full size image
-        current_directory, current_name = self.get_file()
-        image = Image.open(path.join(current_directory, current_name))
-        image_width, image_height = image.size
-
-        # calculate target height
-        maximum_width = int(maximum_width)
-        maximum_height = int(maximum_width * image_height / image_width)
-
-        # return self if requested size is larger than the image
-        if image_width < maximum_width:
-            return self
-
-        # resize image
-        resized_image = image.resize((maximum_width, maximum_height))
-
-        # save to file
-        resized_image.save(target_path, quality=80)
-        return resized
+        raise NotImplementedError

pycs/projects/Project.py

@@ -6,7 +6,8 @@ from eventlet import spawn_after
 
 from pycs.observable import ObservableDict
 from pycs.pipeline.PipelineManager import PipelineManager
-from pycs.projects.MediaFile import MediaFile
+from pycs.projects.ImageFile import ImageFile
+from pycs.projects.VideoFile import VideoFile
 from pycs.util.RecursiveDictionary import set_recursive
 
 
@@ -32,7 +33,7 @@ class Project(ObservableDict):
 
         # save data as MediaFile objects
         for key in obj['data'].keys():
-            obj['data'][key] = MediaFile(obj['data'][key], obj['id'])
+            obj['data'][key] = self.create_media_file(obj['data'][key], obj['id'])
 
         # initialize super
         super().__init__(obj, parent)
@@ -46,9 +47,29 @@ class Project(ObservableDict):
     def new_media_file_path(self):
         return path.join('projects', self['id'], 'data'), str(uuid1())
 
-    def add_media_file(self, file):
-        file = MediaFile(file, self['id'])
-        self['data'][file['id']] = file
+    def create_media_file(self, file, project_id=None):
+        if project_id is None:
+            project_id = self['id']
+
+        # TODO check file extension
+        # TODO determine type
+        # TODO filter supported types
+        if file['extension'] in ['.jpg', '.png']:
+            return ImageFile(file, project_id)
+        if file['extension'] in ['.mp4']:
+            return VideoFile(file, project_id)
+
+        raise NotImplementedError
+
+    def add_media_file(self, uuid, name, extension, size, created):
+        file = {
+            'id': uuid,
+            'name': name,
+            'extension': extension,
+            'size': size,
+            'created': created
+        }
+        self['data'][file['id']] = self.create_media_file(file)
 
     def remove_media_file(self, file_id):
         del self['data'][file_id]

pycs/projects/VideoFile.py

@@ -0,0 +1,50 @@
+from os.path import splitext, join, exists
+
+import cv2
+
+from pycs.projects.ImageFile import ImageFile
+from pycs.projects.MediaFile import MediaFile
+
+
+class VideoFile(MediaFile):
+    def __init__(self, obj, project_id):
+        # add type to object
+        obj['type'] = 'video'
+
+        # call parent constructor
+        super().__init__(obj, project_id)
+
+        # generate thumbnail
+        self.__thumbnail = self.__read_video()
+
+    def __read_video(self):
+        # determine some properties
+        path, name = self.get_file()
+
+        file_name, file_ext = splitext(name)
+        video_path = join(path, name)
+        image_path = join(path, f'{file_name}.jpg')
+
+        # open video file
+        video = cv2.VideoCapture(video_path)
+
+        # read fps value
+        self['fps'] = video.get(cv2.CAP_PROP_FPS)
+        self['frameCount'] = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+
+        # create thumbnail
+        if not exists(image_path):
+            _, image = video.read()
+            cv2.imwrite(image_path, image)
+
+        # close video file
+        video.release()
+
+        # return ImageFile from thumbnail
+        copy = self.copy()
+        copy['extension'] = '.jpg'
+
+        return ImageFile(copy, self.project_id)
+
+    def resize(self, maximum_width):
+        return self.__thumbnail.resize(maximum_width)

webui/src/components/media/annotated-image.vue

@@ -1,9 +1,28 @@
 <template>
   <div class="annotated-image" v-on="events">
-    <img alt="media" ref="image" draggable="false"
-         :src="mediaUrl" :srcset="mediaUrlSet" :sizes="mediaUrlSizes"/>
+    <img v-if="data.type === 'image'"
+         alt="media" ref="image" draggable="false"
+         :src="mediaUrl" :srcset="mediaUrlSet" :sizes="mediaUrlSizes"
+         v-on:load="resizeEvent" v-on:loadedmetadata="resizeEvent" v-on:loadeddata="resizeEvent"/>
 
-    <div style="position: absolute; top: 0.5rem; left: 0.5rem">{{ data }}</div>
+    <template v-if="data.type === 'video'">
+      <video ref="image" draggable="false"
+             preload="auto" :src="mediaUrl"
+             v-on:touchstart.prevent v-on:touchmove.prevent v-on:touchend.prevent
+             v-on:mousedown.prevent v-on:mousemove.prevent v-on:mouseup.prevent
+             v-on:click.prevent v-on:dragstart.prevent
+             v-on:load="resizeEvent" v-on:canplay="resizeEvent" v-on:loadeddata="resizeEvent"
+             v-on:timeupdate="videoProgress">
+      </video>
+
+      <video-control :video="video" :data="data"
+                     v-on:touchstart.prevent.stop v-on:touchmove.prevent.stop v-on:touchend.prevent.stop
+                     v-on:mousedown.prevent.stop v-on:mousemove.prevent.stop v-on:mouseup.prevent.stop
+                     v-on:click.prevent.stop v-on:dragstart.prevent.stop
+                     @play="videoPlay" @jump="videoJump"/>
+    </template>
+
+    <!-- <div style="position: absolute; top: 0.5rem; left: 0.5rem">{{ video }}</div> -->
 
     <annotation-box v-if="current"
                     :image="image"
@@ -27,37 +46,50 @@
 
 <script>
 import AnnotationBox from "@/components/media/annotation-box";
+import VideoControl from "@/components/media/video-control";
 
 export default {
   name: "annotated-image",
-  components: {AnnotationBox},
+  components: {VideoControl, AnnotationBox},
   props: ['data', 'project', 'socket'],
   mounted: function () {
     window.addEventListener("resize", this.resizeEvent);
+    this.resizeEvent();
 
-    if (this.$refs.image.complete)
-      this.resizeEvent();
-    else
-      this.$refs.image.addEventListener('load', this.resizeEvent);
+    // TODO dirty workaround
+    // this works around a bug which is probably caused by partly loaded
+    // videos and their received dimensions
+    this.watcher = setInterval(this.resizeEvent, 400);
   },
   destroyed() {
     window.removeEventListener("resize", this.resizeEvent);
+    clearInterval(this.watcher);
   },
   watch: {
     data: function () {
       this.current = false;
+      this.video = {
+        frame: 0,
+        play: false
+      };
+
       this.resizeEvent();
     }
   },
   data: function () {
     return {
       sizes: [600, 800, 1200, 1600, 2000, 3000],
+      watcher: 0,
       image: {
         left: 0,
         top: 0,
         width: 0,
         height: 0
       },
+      video: {
+        frame: 0,
+        play: false
+      },
       start: false,
       fixed: false,
       current: false,
@@ -75,9 +107,14 @@ export default {
       return this.sizes.map(e => '(max-width: ' + e + 'px) ' + e + 'px').join(',');
     },
     predictions: function () {
-      return Object.keys(this.data.predictionResults)
+      const predictions = Object.keys(this.data.predictionResults)
           .map(k => this.data.predictionResults[k])
           .filter(k => 'x' in k);
+
+      if (this.data.type === 'video')
+        return predictions.filter(k => k.frame === this.video.frame);
+      else
+        return predictions;
     },
     events: function () {
       if (this.project.model.supports.includes('bounding-boxes')
@@ -106,6 +143,22 @@ export default {
       this.image.width = element.width;
       this.image.height = element.height;
     },
+    videoPlay: function (value) {
+      this.video.play = value;
+      if (value)
+        this.$refs.image.play();
+      else
+        this.$refs.image.pause();
+    },
+    videoJump: function (value) {
+      value = Math.max(0, Math.min(this.data.frameCount, value));
+      const diff = value - this.video.frame;
+
+      this.$refs.image.currentTime += diff / this.data.fps;
+    },
+    videoProgress: function (event) {
+      this.video.frame = Math.floor(event.target.currentTime * this.data.fps);
+    },
     get: function (event) {
       if ('clientX' in event)
         return event;
@@ -137,12 +190,17 @@ export default {
         hy = this.fixed && 'hy' in this.fixed ? this.fixed.hy : Math.min(Math.max(y1, y2), 1);
       }
 
-      return {
+      const rectangle = {
         x: lx,
         y: ly,
         w: hx - lx,
         h: hy - ly
-      };
+      }
+
+      if (this.data.type === 'video')
+        rectangle.frame = this.video.frame;
+
+      return rectangle;
     },
     press: function (event) {
       this.start = {
@@ -222,8 +280,14 @@ export default {
 </script>
 
 <style scoped>
-img {
+img,
+video {
   max-width: 100%;
   max-height: 100%;
 }
+
+.video-control {
+  position: absolute;
+  bottom: 0;
+}
 </style>

webui/src/components/media/video-control.vue

@@ -0,0 +1,154 @@
+<template>
+  <div class="video-control"
+       v-on:touchstart="prevent" v-on:touchmove="prevent" v-on:touchend="prevent"
+       v-on:mousedown="prevent" v-on:mousemove="prevent" v-on:mouseup="prevent"
+       v-on:click="prevent" v-on:dragstart="prevent">
+    <div class="play">
+      <template v-if="!video.play">
+        <div>
+          Play
+        </div>
+
+        <button-input
+            @click.prevent.stop="playButton">&#x25b6;
+        </button-input>
+      </template>
+      <template v-if="video.play">
+        <div>
+          Pause
+        </div>
+
+        <button-input
+            @click.prevent.stop="pauseButton">||
+        </button-input>
+      </template>
+    </div>
+
+    <div class="progress">
+      <div>
+        Progress
+      </div>
+
+      <input type="range"
+             min="0" :max="data.frameCount" :value="video.frame"
+             @change="progress">
+    </div>
+
+    <div class="frame">
+      <div>
+        Frame
+      </div>
+
+      <button-row>
+        <button-input @click.prevent.stop="prevFrame">&lt;</button-input>
+        <button-input @click.prevent.stop="nextFrame">&gt;</button-input>
+      </button-row>
+    </div>
+
+    <div class="prediction">
+      <div>
+        Prediction
+      </div>
+
+      <button-row>
+        <button-input @click.prevent.stop="prevPrediction">&lt;</button-input>
+        <button-input @click.prevent.stop="nextPrediction">&gt;</button-input>
+      </button-row>
+    </div>
+  </div>
+</template>
+
+<script>
+import ButtonInput from "@/components/base/button-input";
+import ButtonRow from "@/components/base/button-row";
+
+export default {
+  name: "video-control",
+  components: {ButtonRow, ButtonInput},
+  props: ['video', 'data'],
+  computed: {
+    timeStep: function () {
+      return 1 / this.data.fps;
+    }
+  },
+  methods: {
+    prevent: function (e) {
+      e.stopPropagation();
+      // e.preventDefault();
+    },
+    playButton: function () {
+      this.$emit('play', true);
+    },
+    pauseButton: function () {
+      this.$emit('play', false);
+    },
+    prevFrame: function () {
+      this.$emit('jump', this.video.frame - 1);
+    },
+    nextFrame: function () {
+      this.$emit('jump', this.video.frame + 1);
+    },
+    prevPrediction: function () {
+      let targetFrame = -Infinity;
+
+      for (let key in this.data.predictionResults) {
+        const element = this.data.predictionResults[key];
+        if ('frame' in element
+            && element.frame < this.video.frame
+            && element.frame > targetFrame) {
+          targetFrame = element.frame;
+        }
+      }
+
+      if (targetFrame !== -Infinity) {
+        this.$emit('jump', targetFrame);
+      }
+    },
+    nextPrediction: function () {
+      let targetFrame = Infinity;
+
+      for (let key in this.data.predictionResults) {
+        const element = this.data.predictionResults[key];
+        if ('frame' in element
+            && element.frame > this.video.frame
+            && element.frame < targetFrame) {
+          targetFrame = element.frame;
+        }
+      }
+
+      if (targetFrame !== Infinity) {
+        this.$emit('jump', targetFrame);
+      }
+    },
+    progress: function (event) {
+      this.$emit('jump', event.target.value);
+    }
+  }
+}
+</script>
+
+<style scoped>
+.video-control {
+  display: flex;
+  flex-direction: row;
+  width: 100%;
+  padding: 0.5rem;
+  background-color: rgba(0, 0, 0, 0.4);
+  color: rgba(255, 255, 255, 0.6);
+}
+
+.video-control .progress {
+  flex-grow: 1;
+}
+
+.video-control > div {
+  display: flex;
+  flex-direction: column;
+  text-align: center;
+  font-size: 80%;
+}
+
+.video-control > div > :not(div) {
+  flex-grow: 1;
+}
+</style>