PyCS/webui/src/components/media/annotated-image.vue

<template>
  <div class="annotated-image" ref="root">
    <template v-if="src">
      <options-bar :file="current"
                   :interaction="interaction"
                   @interaction="interaction = $event"
                   :filter="filter"
                   @filter="filter = $event"
                   :label="label"
                   @label="label = $event"
                   :labels="labels"
                   :zoomBox="zoomBox"
                   :infoBox="infoBox !== false"
                   @infoBox="infoBox = $event; interaction=false"
                   @unzoom="zoomBox=false; interaction=false"
                   @prevzoom="$refs.overlay.prevZoom()"
                   @nextzoom="$refs.overlay.nextZoom()"/>

      <div class="media">
        <!-- image -->
        <img v-if="current.type === 'image'"
             ref="media" :src="src" alt="media"
             :style="cropPosition"
             v-on:load="change" v-on:loadedmetadata="change" v-on:loadeddata="change"
             v-on:transitionend="resize">

        <!-- video -->
        <template v-if="current.type === 'video'">
          <div class="video-player">
            <video ref="media" :src="src"
                   v-on:load="change" v-on:loadedmetadata="change" v-on:loadeddata="change" v-on:canplay="change"
                   v-on:timeupdate="videoProgress"/>
          </div>
          <video-control :file="current"
                         :video="video"
                         :results="results"
                         @play="videoPlay"
                         @jump="videoJump"/>
        </template>

        <!-- overlay -->
        <annotation-overlay ref="overlay"
                            :file="current"
                            :position="overlayPosition"
                            :size="image"
                            :interaction="interaction"
                            :filter="filter"
                            :label="label"
                            :video="video"
                            :results="results"
                            :labels="labels"
                            :crop="infoBox"
                            @crop="infoBox = $event"
                            :zoom="zoomBox"
                            @zoom="zoomBox = $event"/>
      </div>

      <cropped-image v-if="infoBox !== false"
                     :labels="labels"
                     :file="current"
                     :box="infoBox"
                     @close="infoBox=false"/>
    </template>
  </div>
</template>

<script>
import AnnotationOverlay from "@/components/media/annotation-overlay";
import VideoControl from "@/components/media/video-control";
import OptionsBar from "@/components/media/options-bar";
import CroppedImage from "@/components/media/cropped-image";

export default {
  name: "annotated-image",
  components: {OptionsBar, VideoControl, AnnotationOverlay, CroppedImage},
  props: ['current'],
  mounted: function () {
    // add resize listener
    window.addEventListener('resize', this.resize);
    this.interval = setInterval(this.resize, 1000);
    this.resize();

    // add result listener
    this.$root.socket.on('create-result', this.createResult);
    this.$root.socket.on('edit-result', this.editResult);
    this.$root.socket.on('remove-result', this.removeResult);

    // add label listener
    this.getLabels();
    this.$root.socket.on('connect', this.getLabels);
    this.$root.socket.on('create-label', this.addLabelToList);
    this.$root.socket.on('remove-label', this.removeLabelFromList);
    this.$root.socket.on('edit-label', this.editLabelInList);

    // get model
    this.$root.socket.get(`/projects/${this.$root.project.identifier}/model`)
      .then(response => response.json())
      .then(model => {
        this.supported.labeledImages = model.supports.includes('labeled-images');
        this.supported.labeledBoundingBoxes = model.supports.includes('labeled-bounding-boxes');
        this.supported.boundingBoxes = this.supported.labeledBoundingBoxes
          || model.supports.includes('bounding-boxes');
      });
  },
  destroyed: function () {
    window.removeEventListener('resize', this.resize);
    clearInterval(this.interval);

    this.$root.socket.off('create-result', this.createResult);
    this.$root.socket.off('edit-result', this.editResult);
    this.$root.socket.off('remove-result', this.removeResult);

    this.$root.socket.off('connect', this.getLabels);
    this.$root.socket.off('create-label', this.addLabelToList);
    this.$root.socket.off('remove-label', this.removeLabelFromList);
    this.$root.socket.off('edit-label', this.editLabelInList);
  },
  data: function () {
    return {
      interval: false,
      container: {
        top: 0,
        left: 0,
        width: 0,
        height: 0,
      },
      image: {
        top: 0,
        left: 0,
        width: 0,
        height: 0
      },
      video: {
        frame: 0,
        play: false
      },
      infoBox: false,
      zoomBox: false,
      supported: {
        labeledImages: false,
        boundingBoxes: false,
        labeledBoundingBoxes: false,
      },
      interaction: 'draw-box',
      filter: ['user', 'pipeline'],
      label: false,
      results: [],
      labels: []
    }
  },
  computed: {
    src: function () {
      if (!this.container.width || !this.container.height)
        return '';

      if (this.current.type === 'video')
        return this.$root.socket.media(this.current);

      const width = Math.ceil(this.container.width / 400) * 400;
      const height = Math.ceil(this.container.height / 400) * 400;
      return this.$root.socket.media(this.current, width, height);
    },
    overlayPosition: function () {
      return {
        top: (this.image.top - this.container.top) + 'px',
        left: (this.image.left - this.container.left) + 'px',
        width: this.image.width + 'px',
        height: this.image.height + 'px'
      }
    },
    cropPosition: function () {
      if (!this.zoomBox)
        return {
          transform: ``,
        };

      const posX = 0.5 - (this.zoomBox.x + this.zoomBox.w / 2);
      const posY = 0.5 - (this.zoomBox.y + this.zoomBox.h / 2);
      const factor = 0.75 / Math.max(this.zoomBox.w, this.zoomBox.h);

      // use a transition to use the transitionend event to recalculate box positions
      return {
        transform: `scale(${factor}) translateX(${posX * 100}%) translateY(${posY * 100}%)`
      };
    }
  },
  methods: {
    resize: function () {
      const rect = this.$refs.root.getBoundingClientRect();

      this.container.top = rect.top;
      this.container.left = rect.left;
      this.container.width = rect.width;
      this.container.height = rect.height;

      this.change();
    },
    change: function () {
      if (!this.$refs.media)
        return;

      const rect = this.$refs.media.getBoundingClientRect();

      this.image.top = rect.top;
      this.image.left = rect.left;
      this.image.width = rect.width;
      this.image.height = rect.height;
    },
    videoProgress: function (event) {
      this.video.frame = Math.floor(event.target.currentTime * this.current.fps);
    },
    videoPlay: function (value) {
      this.video.play = value;
      if (value)
        this.$refs.media.play();
      else
        this.$refs.media.pause();
    },
    videoJump: function (value) {
      // calculate difference
      value = Math.max(0, Math.min(this.current.frames, value));
      const diff = value - this.video.frame;

      // set timestamp
      this.$refs.media.currentTime += diff / this.current.fps;

      // get timestamp offset and correct if needed
      const currentFrame = this.$refs.media.currentTime * this.current.fps;

      if (diff < 0 && currentFrame < value) {
        this.$refs.media.currentTime += 0.5 / this.current.fps;
      } else if (diff > 0 && currentFrame > value + 0.5) {
        this.$refs.media.currentTime -= 0.5 / this.current.fps;
      }
    },
    createResult: function (result) {
      if (result['file_id'] !== this.current.identifier)
        return;

      for (let r of this.results)
        if (r.identifier === result.identifier)
          return;

      this.results.push(result);
    },
    removeResult: function (result) {
      for (let i = 0; i < this.results.length; i++) {
        if (this.results[i].identifier === result.identifier) {
          this.results.splice(i, 1);
          return;
        }
      }
    },
    editResult: function (result) {
      if (this.infoBox && result.identifier === this.infoBox.identifier)
        this.infoBox = result;

      for (let i = 0; i < this.results.length; i++) {
        if (this.results[i].identifier === result.identifier) {
          this.$set(this.results, i, result);
          return;
        }
      }
    },
    getLabels: function () {
      this.$root.socket.get(`/projects/${this.$root.project.identifier}/labels`)
        .then(response => response.json())
        .then(labels => {
          this.labels = [];
          labels.forEach(this.addLabelToList);
        });
    },
    addLabelToList: function (label) {
      if (label['project_id'] !== this.$root.project.identifier)
        return;

      for (let l of this.labels)
        if (l.identifier === label.identifier)
          return;

      this.labels.push(label);
    },
    removeLabelFromList: function (label) {
      for (let i = 0; i < this.labels.length; i++) {
        if (this.labels[i].identifier === label.identifier) {
          this.labels.splice(i, 1);
          return;
        }
      }
    },
    editLabelInList: function (label) {
      for (let i = 0; i < this.labels.length; i++) {
        if (this.labels[i].identifier === label.identifier) {
          this.$set(this.labels, i, label);
          return;
        }
      }
    }
  },
  watch: {
    current: {
      immediate: true,
      handler: function (newVal) {
        this.video.play = false;
        this.video.frame = 0;

        this.zoomBox = false;

        this.$root.socket.get(`/data/${newVal.identifier}/results`)
          .then(response => response.json())
          .then(results => {
            this.results = results;
          });
      }
    },
    infoBox: function () {
      setTimeout(this.resize, 1);
    }
  }
}
</script>

<style scoped>
.annotated-image {
  width: 100%;
  height: 100%;

  display: flex;
  flex-direction: row;
  justify-content: center;
  align-items: center;

  overflow: hidden;
}

.options-bar {
  height: 100%;
}

.media {
  width: 100%;
  height: 100%;
  flex-grow: 1;

  display: flex;
  flex-direction: column;
  justify-content: center;
  align-items: center;

  overflow: hidden;
}

.video-player {
  width: 100%;
  height: 100%;
  flex-grow: 1;
  overflow-y: auto;

  display: flex;
  justify-content: center;
  align-items: center;
}

img, video {
  max-width: 100%;
  max-height: 100%;
  transition: transform 0.01s;
}
</style>