PyCS/webui/src/components/media/annotated-image.vue

<template>
  <div class="annotated-image" v-on="events">
    <img v-if="data.type === 'image'"
         alt="media" ref="image" draggable="false"
         :src="mediaUrl" :srcset="mediaUrlSet" :sizes="mediaUrlSizes"
         v-on:load="resizeEvent" v-on:loadedmetadata="resizeEvent" v-on:loadeddata="resizeEvent"/>

    <template v-if="data.type === 'video'">
      <video ref="image" draggable="false"
             preload="auto" :src="mediaUrl"
             v-on:touchstart.prevent v-on:touchmove.prevent v-on:touchend.prevent
             v-on:mousedown.prevent v-on:mousemove.prevent v-on:mouseup.prevent
             v-on:click.prevent v-on:dragstart.prevent
             v-on:load="resizeEvent" v-on:canplay="resizeEvent" v-on:loadeddata="resizeEvent"
             v-on:timeupdate="videoProgress">
      </video>

      <video-control :video="video" :data="data"
                     v-on:touchstart.prevent.stop v-on:touchmove.prevent.stop v-on:touchend.prevent.stop
                     v-on:mousedown.prevent.stop v-on:mousemove.prevent.stop v-on:mouseup.prevent.stop
                     v-on:click.prevent.stop v-on:dragstart.prevent.stop
                     @play="videoPlay" @jump="videoJump"/>
    </template>

    <!-- <div style="position: absolute; top: 0.5rem; left: 0.5rem">{{ extremeClicking }}</div> -->

    <annotation-box v-if="current"
                    :image="image"
                    :supports="project.model.supports"
                    :position="current"/>

    <annotation-box v-for="(result, index) in predictions"
                    :type="result.origin"
                    :key="index"
                    :id="result.id"
                    :image="image"
                    :position="result"
                    :socket="socket"
                    :box-url="mediaUrl + '/' + result.id"
                    :labels="project.labels"
                    :supports="project.model.supports"
                    :selected="selectedBoundingBox !== false && index === selectedBoundingBox"
                    @move="move"
                    @resize="resize"
                    @update="update"/>
  </div>
</template>

<script>
import AnnotationBox from "@/components/media/annotation-box";
import VideoControl from "@/components/media/video-control";

export default {
  name: "annotated-image",
  components: {VideoControl, AnnotationBox},
  props: ['data', 'project', 'socket', 'filter', 'extremeClicking'],
  mounted: function () {
    window.addEventListener('resize', this.resizeEvent);
    this.resizeEvent();

    // TODO dirty workaround
    // this works around a bug which is probably caused by partly loaded
    // videos and their received dimensions
    this.watcher = setInterval(this.resizeEvent, 400);
  },
  created: function () {
    window.addEventListener('keypress', this.keypressEvent);
  },
  destroyed() {
    window.removeEventListener('resize', this.resizeEvent);
    window.removeEventListener('keypress', this.keypressEvent);
    clearInterval(this.watcher);
  },
  watch: {
    data: function () {
      this.current = false;
      this.video = {
        frame: 0,
        play: false
      };

      this.resizeEvent();
    },
    extremeClicking: function (newValue) {
      if (!newValue && this.current) {
        this.release();
      }
    }
  },
  data: function () {
    return {
      sizes: [600, 800, 1200, 1600, 2000, 3000],
      watcher: 0,
      image: {
        left: 0,
        top: 0,
        width: 0,
        height: 0
      },
      video: {
        frame: 0,
        play: false
      },
      start: false,
      fixed: false,
      current: false,
      callback: false,
      selectedBoundingBox: false
    }
  },
  computed: {
    mediaUrl: function () {
      return this.socket.media(this.project.id, this.data.id);
    },
    mediaUrlSet: function () {
      return this.sizes.map(e => this.mediaUrl + '/' + e + ' ' + e + 'w').join(',');
    },
    mediaUrlSizes: function () {
      return this.sizes.map(e => '(max-width: ' + e + 'px) ' + e + 'px').join(',');
    },
    predictions: function () {
      const predictions = Object.keys(this.data.predictionResults)
          .map(k => this.data.predictionResults[k])
          .filter(k => 'x' in k)
          .filter(k => !this.filter || k.origin === this.filter);

      if (this.data.type === 'video')
        return predictions.filter(k => k.frame === this.video.frame);
      else
        return predictions;
    },
    events: function () {
      if (this.project.model.supports.includes('bounding-boxes')
          || this.project.model.supports.includes('labeled-bounding-boxes')) {
        return {
          'touchstart': this.press,
          'touchmove': this.track,
          'touchend': this.release,
          'mousedown': this.press,
          'mousemove': this.track,
          'mouseup': this.release,
          'dragstart': e => e.stopPropagation()
        }
      } else {
        return {}
      }
    }
  },
  methods: {
    resizeEvent: function () {
      if (!this.$refs.image)
        return;

      const element = this.$refs.image.getBoundingClientRect();
      const parent = this.$refs.image.parentElement.getBoundingClientRect();

      this.image.left = element.x - parent.x;
      this.image.top = element.y - parent.y;
      this.image.width = element.width;
      this.image.height = element.height;
    },
    keypressEvent: function (event) {
      if (event.key === 'w') {
        if (this.selectedBoundingBox === false || this.selectedBoundingBox === this.predictions.length - 1)
          this.selectedBoundingBox = -1;

        this.selectedBoundingBox += 1;
      } else if (event.key === 's') {
        if (this.selectedBoundingBox === false || this.selectedBoundingBox === 0)
          this.selectedBoundingBox = this.predictions.length;

        this.selectedBoundingBox -= 1;
      } else if (event.key === 'v') {
        navigator.clipboard.readText().then(text => {
          const obj = JSON.parse(text);
          if (obj.type === 'bounding-box' || obj.type === 'labeled-bounding-box') {
            if (this.data.type === 'video')
              obj.frame = this.video.frame;

            this.socket.post(this.mediaUrl, obj).then(this.update);
            console.log('paste', obj);
          }
        });
      }
    },
    update: function () {
      this.selectedBoundingBox = false;
      this.$emit('update', true);
    },
    videoPlay: function (value) {
      this.video.play = value;
      if (value)
        this.$refs.image.play();
      else
        this.$refs.image.pause();
    },
    videoJump: function (value) {
      // calculate difference
      value = Math.max(0, Math.min(this.data.frameCount, value));
      const diff = value - this.video.frame;

      // set timestamp
      this.$refs.image.currentTime += diff / this.data.fps;

      // get timestamp offset and correct if needed
      const currentFrame = this.$refs.image.currentTime * this.data.fps;

      if (diff < 0 && currentFrame < value) {
        this.$refs.image.currentTime += 0.5 / this.data.fps;
      } else if (diff > 0 && currentFrame > value + 0.5) {
        this.$refs.image.currentTime -= 0.5 / this.data.fps;
      }
    },
    videoProgress: function (event) {
      this.video.frame = Math.floor(event.target.currentTime * this.data.fps);
    },
    get: function (event) {
      if ('clientX' in event)
        return event;
      if ('touches' in event && event.touches.length > 0)
        return event.touches[0];
      if ('changedTouches' in event && event.changedTouches.length > 0)
        return event.changedTouches[0];
    },
    getX: function (event) {
      const bcr = this.$refs.image.getBoundingClientRect();
      return (this.get(event).clientX - bcr.left) / bcr.width;
    },
    getY: function (event) {
      const bcr = this.$refs.image.getBoundingClientRect();
      return (this.get(event).clientY - bcr.top) / bcr.height;
    },
    buildRectangle: function (x1, y1, x2, y2) {
      let lx, hx, ly, hy;

      if (this.fixed && 'offsetX' in this.fixed && 'offsetY' in this.fixed) {
        lx = Math.max(x2 + this.fixed.offsetX, 0);
        hx = Math.min(x2 + this.fixed.offsetX + this.fixed.w, 1);
        ly = Math.max(y2 + this.fixed.offsetY, 0);
        hy = Math.min(y2 + this.fixed.offsetY + this.fixed.h, 1);
      } else {
        lx = this.fixed && 'lx' in this.fixed ? this.fixed.lx : Math.max(Math.min(x1, x2), 0);
        hx = this.fixed && 'hx' in this.fixed ? this.fixed.hx : Math.min(Math.max(x1, x2), 1);
        ly = this.fixed && 'ly' in this.fixed ? this.fixed.ly : Math.max(Math.min(y1, y2), 0);
        hy = this.fixed && 'hy' in this.fixed ? this.fixed.hy : Math.min(Math.max(y1, y2), 1);
      }

      const rectangle = {
        x: lx,
        y: ly,
        w: hx - lx,
        h: hy - ly
      }

      if (this.data.type === 'video')
        rectangle.frame = this.video.frame;

      return rectangle;
    },
    press: function (event) {
      const x = this.getX(event);
      const y = this.getY(event);

      if (this.extremeClicking && this.start) {
        if (this.current) {
          const minX = Math.min(x, this.current.x);
          const maxX = Math.max(x, this.current.x + this.current.w);
          const minY = Math.min(y, this.current.y);
          const maxY = Math.max(y, this.current.y + this.current.h);

          this.current = this.buildRectangle(minX, minY, maxX, maxY);
        } else {
          this.current = this.buildRectangle(this.start.x, this.start.y, x, y);
        }
      } else {
        this.start = {
          x: x,
          y: y
        };
      }
    },
    track: function (event) {
      if (this.start && !this.extremeClicking) {
        const x = this.getX(event);
        const y = this.getY(event);

        this.current = this.buildRectangle(this.start.x, this.start.y, x, y);
      }
    },
    release: function () {
      if (this.start && !this.extremeClicking) {
        if (this.callback) {
          this.callback(this.current);
          this.update();
        } else {
          this.socket.post(this.mediaUrl, this.current)
              .then(this.update);
        }

        this.start = false;
        this.fixed = false;
        this.current = false;
        this.callback = false;
      }
    },
    move: function (event, position, callback) {
      this.callback = callback;

      const x = this.getX(event);
      const y = this.getY(event);

      this.start = {x: position.x, y: position.y};
      this.fixed = {w: position.w, h: position.h, offsetX: position.x - x, offsetY: position.y - y};
    },
    resize: function (position, mode, callback) {
      this.callback = callback;
      switch (mode) {
        case 'nw':
          this.start = {x: position.x + position.w, y: position.y + position.h};
          this.fixed = false;
          break;
        case 'ne':
          this.start = {x: position.x, y: position.y + position.h};
          this.fixed = false;
          break;
        case 'sw':
          this.start = {x: position.x + position.w, y: position.y};
          this.fixed = false;
          break;
        case 'se':
          this.start = {x: position.x, y: position.y};
          this.fixed = false;
          break;
        case 'nn':
          this.start = {x: position.x, y: position.y + position.h};
          this.fixed = {lx: position.x, hx: position.x + position.w};
          break;
        case 'ww':
          this.start = {x: position.x + position.w, y: position.y};
          this.fixed = {ly: position.y, hy: position.y + position.h};
          break;
        case 'ee':
          this.start = {x: position.x, y: position.y};
          this.fixed = {ly: position.y, hy: position.y + position.h};
          break;
        case 'ss':
          this.start = {x: position.x, y: position.y};
          this.fixed = {lx: position.x, hx: position.x + position.w};
          break;
      }
    }
  }
}
</script>

<style scoped>
img,
video {
  max-width: 100%;
  max-height: 100%;
}

.video-control {
  position: absolute;
  bottom: 0;
}
</style>