aiforge/web_src/js/components/MinioUploader.vue

<template>
  <div class="dropzone-wrapper dataset-files">
    <div id="dataset" class="dropzone"></div>
    <p class="upload-info">
      {{ file_status_text }}
      <span class="success">{{ status }}</span>
    </p>
  </div>
</template>

<script>
// import Dropzone from 'dropzone/dist/dropzone.js';
// import 'dropzone/dist/dropzone.css'
import createDropzone from "../features/dropzone.js";
import SparkMD5 from 'spark-md5';
import axios from "axios";
import qs from "qs";

const { AppSubUrl, StaticUrlPrefix, csrf } = window.config;

export default {
  data() {
    return {
      dropzoneUploader: null,
      maxFiles: 1,
      maxFilesize: 1 * 1024 * 1024 * 1024 * 1024,
      acceptedFiles: "*/*",
      progress: 0,
      status: "",
      dropzoneParams: {},
      file_status_text: ""
    };
  },

  async mounted() {
    let previewTemplate = "";
    previewTemplate += '<div class="dz-preview dz-file-preview">\n ';
    previewTemplate += '    <div class="dz-details">\n ';
    previewTemplate += '        <div class="dz-filename">';
    previewTemplate +=
      "            <span data-dz-name data-dz-thumbnail></span>";
    previewTemplate += "        </div>\n    ";
    previewTemplate += '        <div class="dz-size" data-dz-size></div>\n  ';
    previewTemplate += "    </div>\n  ";
    previewTemplate += '    <div class="dz-progress ui active progress">';
    previewTemplate +=
      '        <div class="dz-upload bar" data-dz-uploadprogress><div class="progress"></div></div>\n  ';
    previewTemplate += "    </div>\n  ";
    previewTemplate += '    <div class="dz-success-mark">';
    previewTemplate += "        <span>上传成功</span>";
    previewTemplate += "    </div>\n  ";
    previewTemplate += '    <div class="dz-error-mark">';
    previewTemplate += "        <span>上传失败</span>";
    previewTemplate += "    </div>\n  ";
    previewTemplate += '    <div class="dz-error-message">';
    previewTemplate += "        <span data-dz-errormessage></span>";
    previewTemplate += "    </div>\n";
    previewTemplate += "</div>";

    this.dropzoneParams = $("div#minioUploader-params");
    this.file_status_text = this.dropzoneParams.data("file-status");
    this.status = this.dropzoneParams.data("file-init-status");
    const $dropzone = $("div#dataset");
    const dropzoneUploader = await createDropzone($dropzone[0], {
      url: "/todouploader",
      maxFiles: this.maxFiles,
      maxFilesize: this.maxFileSize,
      timeout: 0,
      autoQueue: false,
      dictDefaultMessage: this.dropzoneParams.data("default-message"),
      dictInvalidFileType: this.dropzoneParams.data("invalid-input-type"),
      dictFileTooBig: this.dropzoneParams.data("file-too-big"),
      dictRemoveFile: this.dropzoneParams.data("remove-file"),
      previewTemplate
    });
    dropzoneUploader.on("addedfile", file => {
      setTimeout(() => {
        file.accepted && this.onFileAdded(file);
      }, 200);
    });
    dropzoneUploader.on("maxfilesexceeded", function(file) {
      if (this.files[0].status !== "success") {
        alert(this.dropzoneParams.data("waitting-uploading"));
        this.removeFile(file);
        return;
      }
      this.removeAllFiles();
      this.addFile(file);
    });

    this.dropzoneUploader = dropzoneUploader;
  },
  methods: {
    resetStatus() {
      this.progress = 0;
      this.status = "";
    },
    updateProgress(file, progress) {
      file.previewTemplate.querySelector(
        ".dz-upload"
      ).style.width = `${progress}%`;
    },
    emitDropzoneSuccess(file) {
      file.status = "success";
      this.dropzoneUploader.emit("success", file);
      this.dropzoneUploader.emit("complete", file);
    },
    onFileAdded(file) {
      file.datasetId = document
        .getElementById("datasetId")
        .getAttribute("datasetId");
      this.resetStatus();
      this.computeMD5(file);
    },

    finishUpload(file) {
      this.emitDropzoneSuccess(file);
      setTimeout(() => {
        window.location.reload();
      }, 1000);
    },

    computeMD5(file) {
      this.resetStatus();
      let blobSlice =
          File.prototype.slice ||
          File.prototype.mozSlice ||
          File.prototype.webkitSlice,
        chunkSize = 1024 * 1024 * 64,
        chunks = Math.ceil(file.size / chunkSize),
        currentChunk = 0,
        spark = new SparkMD5.ArrayBuffer(),
        fileReader = new FileReader();
      let time = new Date().getTime();
      // console.log('计算MD5...')
      this.status = this.dropzoneParams.data("md5-computing");
      file.totalChunkCounts = chunks;
      loadNext();

      fileReader.onload = e => {
        fileLoaded.call(this, e);
      };
      fileReader.onerror = err => {
        console.warn("oops, something went wrong.", err);
        file.cancel();
      };

      function fileLoaded(e) {
        spark.append(e.target.result); // Append array buffer
        currentChunk++;
        if (currentChunk < chunks) {
          // console.log(`第${currentChunk}分片解析完成, 开始第${currentChunk +1}/${chunks}分片解析`);
          this.status = `${this.dropzoneParams.data("loading-file")} ${(
            (currentChunk / chunks) *
            100
          ).toFixed(2)}% (${currentChunk}/${chunks})`;
          this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
          loadNext();
          return;
        }

        let md5 = spark.end();
        console.log(
          `MD5计算完成：${file.name} \nMD5：${md5} \n分片：${chunks} 大小:${
            file.size
          } 用时：${(new Date().getTime() - time) / 1000} s`
        );
        spark.destroy(); //释放缓存
        file.uniqueIdentifier = md5; //将文件md5赋值给文件唯一标识
        file.cmd5 = false; //取消计算md5状态
        this.computeMD5Success(file);
      }

      function loadNext() {
        let start = currentChunk * chunkSize;
        let end =
          start + chunkSize >= file.size ? file.size : start + chunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }
    },

    async computeMD5Success(md5edFile) {
      const file = await this.getSuccessChunks(md5edFile);
      if (file.uploadID == "" || file.uuid == "") {
        //未上传过
        await this.newMultiUpload(file);
        if (file.uploadID != "" && file.uuid != "") {
          file.chunks = "";
          this.multipartUpload(file);
        } else {
          //失败如何处理
          return;
        }
        return;
      }

      if (file.uploaded == "1") {
        //已上传成功
        //秒传
        if (file.attachID == "0") {
          //删除数据集记录，未删除文件
          await addAttachment(file);
        }
        console.log("文件已上传完成");
        this.progress = 100;
        this.status = this.dropzoneParams.data("upload-complete");
        this.finishUpload(file);
      } else {
        //断点续传
        this.multipartUpload(file);
      }

      async function addAttachment(file) {
        return await axios.post(
          "/attachments/add",
          qs.stringify({
            uuid: file.uuid,
            file_name: file.name,
            size: file.size,
            dataset_id: file.datasetId,
            _csrf: csrf
          })
        );
      }
    },

    async getSuccessChunks(file) {
      const params = {
        params: {
          md5: file.uniqueIdentifier,
          _csrf: csrf
        }
      };
      try {
        const response = await axios.get("/attachments/get_chunks", params);
        file.uploadID = response.data.uploadID;
        file.uuid = response.data.uuid;
        file.uploaded = response.data.uploaded;
        file.chunks = response.data.chunks;
        file.attachID = response.data.attachID;
        return file;
      } catch (error) {
        console.log("getSuccessChunks catch: ", error);
        return null;
      }
    },

    async newMultiUpload(file) {
      const res = await axios.get("/attachments/new_multipart", {
        params: {
          totalChunkCounts: file.totalChunkCounts,
          md5: file.uniqueIdentifier,
          size: file.size,
          fileType: file.type,
          _csrf: csrf
        }
      });
      file.uploadID = res.data.uploadID;
      file.uuid = res.data.uuid;
    },

    multipartUpload(file) {
      let blobSlice =
          File.prototype.slice ||
          File.prototype.mozSlice ||
          File.prototype.webkitSlice,
        chunkSize = 1024 * 1024 * 64,
        chunks = Math.ceil(file.size / chunkSize),
        currentChunk = 0,
        fileReader = new FileReader(),
        time = new Date().getTime();

      function loadNext() {
        let start = currentChunk * chunkSize;
        let end =
          start + chunkSize >= file.size ? file.size : start + chunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }

      function checkSuccessChunks() {
        var index = successChunks.indexOf((currentChunk + 1).toString());
        if (index == -1) {
          return false;
        }
        return true;
      }

      async function getUploadChunkUrl(currentChunk, partSize) {
        const res = await axios.get("/attachments/get_multipart_url", {
          params: {
            uuid: file.uuid,
            uploadID: file.uploadID,
            size: partSize,
            chunkNumber: currentChunk + 1,
            _csrf: csrf
          }
        });
        console.log("getUploadChunkUrl: ", res);
        urls[currentChunk] = res.data.url;
      }

      async function uploadMinio(url, e) {
        const res = await axios.put(url, e.target.result);
        etags[currentChunk] = res.headers.etag;
      }

      async function updateChunk(currentChunk) {
        await axios.post(
          "/attachments/update_chunk",
          qs.stringify({
            uuid: file.uuid,
            chunkNumber: currentChunk + 1,
            etag: etags[currentChunk],
            _csrf: csrf
          })
        );
      }
      async function uploadChunk(e) {
        if (!checkSuccessChunks()) {
          let start = currentChunk * chunkSize;
          let partSize =
            start + chunkSize >= file.size ? file.size - start : chunkSize;
          //获取分片上传url
          await getUploadChunkUrl(currentChunk, partSize);
          if (urls[currentChunk] != "") {
            //上传到minio
            await uploadMinio(urls[currentChunk], e);
            if (etags[currentChunk] != "") {
              //更新数据库：分片上传结果
              await updateChunk(currentChunk);
            } else {
              return;
            }
          } else {
            return;
          }
        }
      }

      async function completeUpload() {
        return await axios.post(
          "/attachments/complete_multipart",
          qs.stringify({
            uuid: file.uuid,
            uploadID: file.uploadID,
            file_name: file.name,
            size: file.size,
            dataset_id: file.datasetId,
            _csrf: csrf
          })
        );
      }

      var successChunks = new Array();
      var successParts = new Array();
      successParts = file.chunks.split(",");
      for (let i = 0; i < successParts.length; i++) {
        successChunks[i] = successParts[i].split("-")[0].split('"')[1];
      }
      var urls = new Array();
      var etags = new Array();
      console.log("上传分片...");
      this.status = this.dropzoneParams.data("uploading");
      loadNext();
      fileReader.onload = async e => {
        await uploadChunk(e);
        currentChunk++;
        if (currentChunk < chunks) {
          console.log(
            `第${currentChunk}个分片上传完成, 开始第${currentChunk +
              1}/${chunks}个分片上传`
          );
          this.progress = Math.ceil((currentChunk / chunks) * 100);
          this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
          this.status = `${this.dropzoneParams.data("uploading")} ${(
            (currentChunk / chunks) *
            100
          ).toFixed(2)}%`;
          await loadNext();
        } else {
          await completeUpload();
          console.log(
            `文件上传完成：${file.name} \n分片：${chunks} 大小:${
              file.size
            } 用时：${(new Date().getTime() - time) / 1000} s`
          );
          this.progress = 100;
          this.status = this.dropzoneParams.data("upload-complete");
          this.finishUpload(file);
        }
      };
    }
  }
};
</script>

<style>
.dropzone-wrapper {
  margin: 2em auto;
}
.ui .dropzone {
  border: 2px dashed #0087f5;
  box-shadow: none !important;
  padding: 0;
  min-height: 5rem;
  border-radius: 4px;
}
.dataset .dataset-files #dataset .dz-preview.dz-file-preview,
.dataset .dataset-files #dataset .dz-preview.dz-processing {
  display: flex;
  align-items: center;
}
.dataset .dataset-files #dataset .dz-preview {
  border-bottom: 1px solid #dadce0;
  min-height: 0;
}
</style>