aiforge/web_src/js/components/MinioUploader.vue

<template>
  <div class="dropzone-wrapper dataset-files">
    <div id="dataset" class="dropzone">
      <div
        class="maxfilesize ui red message"
        style="display: none; margin: 2.5rem"
      ></div>
    </div>
    <el-button
      style="background-color: #21ba45; margin-top: 2rem"
      type="success"
      :disabled="btnFlag"
      @click="startUpload"
      >{{ upload }}</el-button
    >
    <el-button type="info" @click="cancelDataset">{{ cancel }}</el-button>
    <div style="margin-top: 2rem; position: relative">
      <label
        class="el-form-item__label"
        style="width: 140px; position: absolute; left: -140px"
        >{{ upload_status }}:</label
      >
      <div v-for="(item, index) in uploadFiles" class="datast-upload-progress">
        <span class="dataset-name nowrap" :title="item.name">{{
          item.name
        }}</span>
        <div class="dataset-progress">
          <el-progress
            :text-inside="true"
            :stroke-width="14"
            :percentage="uploadProgressList[index].progress"
          ></el-progress>
        </div>
        <div class="dataset-status nowrap">
          <div class="status-flex">
            <i
              v-if="
                uploadProgressList[index].infoCode === 1 ||
                uploadProgressList[index].infoCode === 2
              "
              class="ri-close-circle-line failed"
            ></i>
            <i
              v-if="uploadProgressList[index].infoCode === 0"
              class="ri-checkbox-circle-line success"
            >
            </i>
            <span>{{ uploadProgressList[index].status }}</span>
            <el-tooltip
              v-if="uploadProgressList[index].infoCode === 1"
              class="item"
              effect="dark"
              placement="top"
            >
              <div slot="content">
                {{ uploadProgressList[index].failedInfo }}
              </div>
              <i
                style="font-size: 16px; margin-left: 0.5rem; cursor: pointer"
                class="ri-question-fill"
              ></i>
            </el-tooltip>
          </div>
        </div>
      </div>
    </div>
  </div>
</template>

<script>
import SparkMD5 from "spark-md5";
import axios from "axios";
import qs from "qs";
import createDropzone from "../features/dropzone.js";

const { _AppSubUrl, _StaticUrlPrefix, csrf } = window.config;
const chunkSize = 1024 * 1024 * 64;
const md5ChunkSize = 1024 * 1024 * 1;

export default {
  props: {
    uploadtype: {
      type: Number,
      required: true,
    },
    desc: {
      type: String,
      default: "",
    },
  },
  data() {
    return {
      dropzoneUploader: null,
      maxFiles: 10,
      maxFilesize: 200,
      acceptedFiles: "*/*",
      progress: 0,
      status: "",
      dropzoneParams: {},
      file_status_text: "",
      file: {},
      repoPath: "",
      btnFlag: false,
      cancel: "",
      upload: "",
      upload_status: "",
      uploadFiles: [],
      uploadFilesAddId: [],
      // allUploadFiles: [],
      uploadLength: 0,
      allUploadLength: 0,
      uploadProgressList: [],
    };
  },

  async mounted() {
    this.dropzoneParams = $("div#minioUploader-params");
    this.file_status_text = this.dropzoneParams.data("file-status");
    this.status = this.dropzoneParams.data("file-init-status");
    this.repoPath = this.dropzoneParams.data("repopath");
    this.cancel = this.dropzoneParams.data("cancel");
    this.upload = this.dropzoneParams.data("upload");
    this.upload_status = this.dropzoneParams.data("upload-status");
    let previewTemplate = `
        <div class="dz-preview dz-file-preview"> 
          <div class="dz-image"> 
            <img data-dz-thumbnail /> 
          </div> 
          <div class="dz-details"> 
            <div class="dz-size"><span data-dz-size></span></div> 
            <div class="dz-filename"><span data-dz-name></span></div> 
          </div> 

          <div style="opacity:0" class="dz-progress"><span class="dz-upload" data-dz-uploadprogress></span></div> 
          <div class="dz-error-message" style="line-height: 1.5;"><span data-dz-errormessage></span></div> 
          <div class="dz-success-mark"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="54" height="54"><path fill="none" d="M0 0h24v24H0z"/><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16zm-.997-4L6.76 11.757l1.414-1.414 2.829 2.829 5.656-5.657 1.415 1.414L11.003 16z" fill="rgba(47,204,113,1)"/></svg></div> 
          <div class="dz-error-mark"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="54" height="54"><path fill="none" d="M0 0h24v24H0z"/><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16zm0-9.414l2.828-2.829 1.415 1.415L13.414 12l2.829 2.828-1.415 1.415L12 13.414l-2.828 2.829-1.415-1.415L10.586 12 7.757 9.172l1.415-1.415L12 10.586z" fill="rgba(231,76,60,1)"/></svg></div> 
        </div> `;
    const $dropzone = $("div#dataset");
    const dropzoneUploader = await createDropzone($dropzone[0], {
      url: "/todouploader",
      maxFiles: this.maxFiles,
      maxFilesize: 1024 * 200,
      filesizeBase: 1024,
      parallelUploads: this.maxFiles,
      timeout: 0,
      addRemoveLinks: true,
      // autoQueue: false,
      autoProcessQueue: false, //自动上传
      dictDefaultMessage: this.dropzoneParams.data("default-message"),
      dictInvalidFileType: this.dropzoneParams.data("invalid-input-type"),
      dictFileTooBig: this.dropzoneParams.data("file-too-big"),
      dictRemoveFile: this.dropzoneParams.data("remove-file"),
      previewTemplate: previewTemplate,
    });
    dropzoneUploader.on("addedfile", (file) => {
      if (file.size / (1024 * 1024) > dropzoneUploader.options.maxFilesize) {
        dropzoneUploader.removeFile(file);
        $(".maxfilesize.ui.red.message").text(
          "单次最多上传10个文件，单个文件不超过200G"
        );
        $(".maxfilesize.ui.red.message").css("display", "block");
      } else {
        this.file = file;
        $(".maxfilesize.ui.red.message").css("display", "none");
      }
    });
    dropzoneUploader.on("removedfile", (file) => {
      $(".maxfilesize.ui.red.message").css("display", "none");
    });
    dropzoneUploader.on("maxfilesexceeded", function (file) {
      dropzoneUploader.removeFile(file);
      $(".maxfilesize.ui.red.message").text(
        "单次最多上传10个文件，单个文件不超过200G"
      );
      $(".maxfilesize.ui.red.message").css("display", "block");
    });
    this.dropzoneUploader = dropzoneUploader;
  },
  watch: {
    allUploadLength(len) {
      if (len === this.uploadFiles.length) {
        setTimeout(() => {
          this.dropzoneUploader.removeAllFiles(true);
          this.btnFlag = false;
          this.$emit("setcluster", this.btnFlag);
        }, 2000);
      }
    },
  },
  methods: {
    startUpload() {
      this.uploadFiles = this.dropzoneUploader.getQueuedFiles();
      if (this.uploadFiles.length === 0) {
        return;
      }
      this.resetStatus();
      $(".dz-remove").remove();
      $(".maxfilesize.ui.red.message").css("display", "none");
      this.btnFlag = true;
      this.$emit("setcluster", this.btnFlag);
      this.uploadFiles.forEach((element) => {
        element.datasetId = document
          .getElementById("datasetId")
          .getAttribute("datasetId");
        this.computeMD5(element);
      });
    },
    cancelDataset() {
      location.href = this.repoPath;
      this.dropzoneUploader.removeAllFiles(true);
    },
    resetStatus() {
      this.uploadLength = 0;
      this.allUploadLength = 0;
      // this.allUploadFiles = [];
      this.uploadProgressList = [];
    },
    updateProgress(file, status, progress, infoCode, failedInfo = "") {
      console.log("this.uploadProgressList:", this.uploadProgressList);
      // file.previewTemplate.querySelector(
      //   ".dz-upload"
      // ).style.width = `${progress}%`;
      // file.previewTemplate.querySelector(".dz-upload").style.background =
      //   "#409eff";
      // file.previewTemplate.querySelector(".dz-upload").style.opacity = 0;
      this.uploadProgressList.forEach((item, index) => {
        if (item.name === file.name) {
          this.uploadProgressList[index].status = status;
          this.uploadProgressList[index].progress = progress;
          this.uploadProgressList[index].infoCode = infoCode;
          this.uploadProgressList[index].failedInfo = failedInfo;
        }
      });
    },
    uploadError(file, info) {
      file.previewTemplate.querySelector(".dz-error-mark").style.opacity = 1;
      file.previewTemplate.querySelector(".dz-progress").style.opacity = 0;
      file.previewTemplate.querySelector(".dz-error-message span").innerHTML =
        info;
      file.previewTemplate.querySelector(".dz-error-message").style.display =
        "block";
      file.previewTemplate.querySelector(".dz-details").onmouseover =
        function () {
          file.previewTemplate.querySelector(
            ".dz-error-message"
          ).style.opacity = 1;
        };
      file.previewTemplate.querySelector(".dz-details").onmouseout =
        function () {
          file.previewTemplate.querySelector(
            ".dz-error-message"
          ).style.opacity = 0;
        };
    },
    emitDropzoneSuccess(file) {
      file.status = "success";
      this.dropzoneUploader.emit("success", file);
      this.dropzoneUploader.emit("complete", file);
    },
    emitDropzoneFailed(file) {
      this.status = this.dropzoneParams.data("falied");
      file.status = "error";
      this.dropzoneUploader.emit("error", file);
      // this.dropzoneUploader.emit('complete', file);
    },
    finishUpload(file) {
      console.log("finish", file);
      file.previewTemplate.querySelector(".dz-success-mark").style.opacity = 1;
      file.previewTemplate.querySelector(".dz-progress").style.opacity = 0;
      if (this.uploadLength === this.uploadFiles.length) {
        setTimeout(() => {
          window.location.href = this.repoPath;
        }, 1000);
      }
    },

    computeMD5(file) {
      const blobSlice =
          File.prototype.slice ||
          File.prototype.mozSlice ||
          File.prototype.webkitSlice,
        chunks = Math.ceil(file.size / chunkSize),
        spark = new SparkMD5.ArrayBuffer(),
        fileReader = new FileReader();
      let currentChunk = 0;
      const time = new Date().getTime();
      this.status = this.dropzoneParams.data("md5-computing");
      this.uploadProgressList.push({
        name: file.name,
        status: this.dropzoneParams.data("md5-computing"),
        progress: 0,
        infoCode: 3,
      });
      file.totalChunkCounts = chunks;
      if (file.size == 0) {
        file.totalChunkCounts = 1;
      }
      loadMd5Next();

      fileReader.onload = (e) => {
        fileLoaded.call(this, e);
      };
      fileReader.onerror = (err) => {
        console.warn("oops, something went wrong.", err);
        file.cancel();
      };

      function fileLoaded(e) {
        spark.append(e.target.result); // Append array buffer
        currentChunk++;
        if (currentChunk < chunks) {
          this.status = `${this.dropzoneParams.data("loading-file")} ${(
            (currentChunk / chunks) *
            100
          ).toFixed(2)}% (${currentChunk}/${chunks})`;
          console.log("-=-=-=-=--=", this.status);
          // this.updateProgress(
          //   file,
          //   this.dropzoneParams.data("md5-computing"),
          //   Number(((currentChunk / chunks) * 100).toFixed(1))
          // );
          loadMd5Next();
          return;
        }

        const md5 = spark.end();
        console.log(
          `MD5计算完成：${file.name} \nMD5：${md5} \n分片：${chunks} 大小:${
            file.size
          } 用时：${(new Date().getTime() - time) / 1000} s`
        );
        // this.updateProgress(
        //   file,
        //   this.dropzoneParams.data("md5-computing"),
        //   100
        // );
        spark.destroy(); // 释放缓存
        file.uniqueIdentifier = md5; // 将文件md5赋值给文件唯一标识
        file.cmd5 = false; // 取消计算md5状态
        this.computeMD5Success(file);
      }

      function loadNext() {
        const start = currentChunk * chunkSize;
        const end =
          start + chunkSize >= file.size ? file.size : start + chunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }

      function loadMd5Next() {
        const start = currentChunk * chunkSize;
        const end =
          start + md5ChunkSize >= file.size ? file.size : start + md5ChunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }
    },

    async computeMD5Success(md5edFile) {
      const file = await this.getSuccessChunks(md5edFile);
      console.log(file);
      try {
        if (file.uploadID == "" || file.uuid == "") {
          // 未上传过
          await this.newMultiUpload(file);
          if (file.uploadID != "" && file.uuid != "") {
            file.chunks = "";
            this.multipartUpload(file);
          } else {
            // 失败如何处理
            let info = "上传失败";
            this.allUploadLength++;
            this.uploadError(file, info);
            this.updateProgress(file, "上传失败", 0, 2);
            // this.allUploadFiles.push({
            //   name: file.name,
            //   status: 2,
            //   info: info,
            // });
            return;
          }
          return;
        }

        if (file.uploaded == "1") {
          // 已上传成功
          // 秒传
          if (file.attachID == "0") {
            // 删除数据集记录，未删除文件
            await addAttachment(file);
          }
          //不同数据集上传同一个文件
          if (file.datasetID != "") {
            if (file.datasetName != "" && file.realName != "") {
              let info = `该文件已上传在数据集: ${file.datasetName}`;
              this.uploadError(file, info);
              this.allUploadLength++;
              this.updateProgress(file, "上传失败", 0, 1, info);
              // this.allUploadFiles.push({
              //   name: file.name,
              //   status: 1,
              //   info: info,
              // });
            }
          }
          console.log("文件已上传完成");
          this.progress = 100;
          this.status = this.dropzoneParams.data("upload-complete");
          // this.finishUpload(file);
        } else {
          // 断点续传
          this.multipartUpload(file);
        }
      } catch (error) {
        this.emitDropzoneFailed(file);
        console.log(error);
      }

      async function addAttachment(file) {
        return await axios.post(
          "/attachments/add",
          qs.stringify({
            uuid: file.uuid,
            file_name: file.name,
            size: file.size,
            dataset_id: file.datasetId,
            type: this.uploadtype,
            _csrf: csrf,
          })
        );
      }
    },

    async getSuccessChunks(file) {
      const params = {
        params: {
          md5: file.uniqueIdentifier,
          type: this.uploadtype,
          file_name: file.name,
          _csrf: csrf,
        },
      };
      try {
        const response = await axios.get("/attachments/get_chunks", params);
        file.uploadID = response.data.uploadID;
        file.uuid = response.data.uuid;
        file.uploaded = response.data.uploaded;
        file.chunks = response.data.chunks;
        file.attachID = response.data.attachID;
        file.datasetID = response.data.datasetID;
        file.datasetName = response.data.datasetName;
        file.realName = response.data.fileName;
        return file;
      } catch (error) {
        this.emitDropzoneFailed(file);
        console.log("getSuccessChunks catch: ", error);
        return null;
      }
    },

    async newMultiUpload(file) {
      const res = await axios.get("/attachments/new_multipart", {
        params: {
          totalChunkCounts: file.totalChunkCounts,
          md5: file.uniqueIdentifier,
          size: file.size,
          fileType: file.type,
          type: this.uploadtype,
          file_name: file.name,
          _csrf: csrf,
        },
      });
      file.uploadID = res.data.uploadID;
      file.uuid = res.data.uuid;
    },

    multipartUpload(file) {
      const blobSlice =
          File.prototype.slice ||
          File.prototype.mozSlice ||
          File.prototype.webkitSlice,
        chunks = Math.ceil(file.size / chunkSize),
        fileReader = new FileReader(),
        time = new Date().getTime();
      let currentChunk = 0;
      let _this = this;

      function loadNext() {
        const start = currentChunk * chunkSize;
        const end =
          start + chunkSize >= file.size ? file.size : start + chunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }

      function checkSuccessChunks() {
        const index = successChunks.indexOf((currentChunk + 1).toString());
        if (index == -1) {
          return false;
        }
        return true;
      }

      async function getUploadChunkUrl(currentChunk, partSize) {
        const res = await axios.get("/attachments/get_multipart_url", {
          params: {
            uuid: file.uuid,
            uploadID: file.uploadID,
            size: partSize,
            chunkNumber: currentChunk + 1,
            type: _this.uploadtype,
            file_name: file.name,
            _csrf: csrf,
          },
        });
        urls[currentChunk] = res.data.url;
      }

      async function uploadMinio(url, e) {
        const res = await axios.put(url, e.target.result);
        delete e.target.result;
        etags[currentChunk] = res.headers.etag;
      }

      async function uploadMinioNewMethod(url, e) {
        var xhr = new XMLHttpRequest();
        xhr.open("PUT", url, false);
        if (_this.uploadtype === 0) {
          xhr.setRequestHeader("Content-Type", "text/plain");
          xhr.send(e.target.result);

          var etagValue = xhr.getResponseHeader("etag");
          etags[currentChunk] = etagValue;
        } else if (_this.uploadtype === 1) {
          xhr.setRequestHeader("Content-Type", "");
          xhr.send(e.target.result);
          var etagValue = xhr.getResponseHeader("ETag");
          //console.log(etagValue);
          etags[currentChunk] = etagValue;
        }
      }

      async function updateChunk(currentChunk) {
        await axios.post(
          "/attachments/update_chunk",
          qs.stringify({
            uuid: file.uuid,
            chunkNumber: currentChunk + 1,
            etag: etags[currentChunk],
            _csrf: csrf,
          })
        );
      }
      async function uploadChunk(e) {
        try {
          if (!checkSuccessChunks()) {
            const start = currentChunk * chunkSize;
            const partSize =
              start + chunkSize >= file.size ? file.size - start : chunkSize;
            // 获取分片上传url
            await getUploadChunkUrl(currentChunk, partSize);
            if (urls[currentChunk] != "") {
              // 上传到minio
              //await uploadMinio(urls[currentChunk], e);
              await uploadMinioNewMethod(urls[currentChunk], e);

              if (etags[currentChunk] != "") {
                // 更新数据库：分片上传结果
                //await updateChunk(currentChunk);
              } else {
                console.log(
                  "上传到minio uploadChunk etags[currentChunk] == ''"
                ); // TODO
              }
            } else {
              console.log("uploadChunk urls[currentChunk] != ''"); // TODO
            }
          }
        } catch (error) {
          // this.emitDropzoneFailed(file);
          console.log(error);
        }
      }

      async function completeUpload() {
        return await axios.post(
          "/attachments/complete_multipart",
          qs.stringify({
            uuid: file.uuid,
            uploadID: file.uploadID,
            file_name: file.name,
            size: file.size,
            dataset_id: file.datasetId,
            type: _this.uploadtype,
            _csrf: csrf,
            description: _this.desc,
          })
        );
      }

      const successChunks = [];
      let successParts = [];
      successParts = file.chunks.split(",");
      for (let i = 0; i < successParts.length; i++) {
        successChunks[i] = successParts[i].split("-")[0];
      }
      const urls = []; // TODO const ?
      const etags = [];
      console.log("上传分片...");
      this.status = this.dropzoneParams.data("uploading");
      loadNext();
      fileReader.onload = async (e) => {
        try {
          await uploadChunk(e);
        } catch (err) {
          console.log(err);
        }

        fileReader.abort();
        currentChunk++;
        if (currentChunk < chunks) {
          console.log(
            `第${currentChunk}个分片上传完成, 开始第${
              currentChunk + 1
            }/${chunks}个分片上传`
          );
          this.progress = Math.ceil((currentChunk / chunks) * 100);
          this.updateProgress(
            file,
            this.dropzoneParams.data("uploading"),
            Number(((currentChunk / chunks) * 100).toFixed(2)),
            3
          );
          this.status = `${this.dropzoneParams.data("uploading")} ${(
            (currentChunk / chunks) *
            100
          ).toFixed(2)}%`;
          await loadNext();
        } else {
          try {
            await completeUpload();
          } catch (err) {
            let info = "上传失败";
            this.allUploadLength++;
            this.uploadError(file, info);
            this.updateProgress(
              file,
              info,
              Number(((currentChunk / chunks) * 100).toFixed(2)) - 1,
              2
            );
            // this.allUploadFiles.push({
            //   name: file.name,
            //   status: 2,
            //   info: info,
            // });
            if (err) {
              return;
            }
          }

          console.log(
            `文件上传完成：${file.name} \n分片：${chunks} 大小:${
              file.size
            } 用时：${(new Date().getTime() - time) / 1000} s`
          );
          this.uploadLength++;
          this.allUploadLength++;
          // this.allUploadFiles.push({
          //   name: file.name,
          //   status: 0,
          //   info: "上传成功",
          // });

          this.updateProgress(file, "上传成功", 100, 0);
          this.progress = 100;
          this.status = this.dropzoneParams.data("upload-complete");
          this.finishUpload(file);
        }
      };
    },
  },
};
</script>

<style scoped>
.dropzone-wrapper {
  margin: 0;
}
.ui .dropzone {
  border: 2px dashed #0087f5;
  box-shadow: none !important;
  padding: 0;
  min-height: 5rem;
  border-radius: 4px;
}
.dataset .dataset-files #dataset .dz-preview.dz-file-preview,
.dataset .dataset-files #dataset .dz-preview.dz-processing {
  display: flex;
  align-items: center;
}
.dataset .dataset-files #dataset .dz-preview {
  border-bottom: 1px solid #dadce0;
  min-height: 0;
}
.upload-info {
  margin-top: 1em;
  margin-bottom: 3em;
}
.success {
  color: #21ba45;
  font-size: 16px;
  margin-right: 0.5rem;
}
.failed {
  color: red;
  font-size: 16px;
  margin-right: 0.5rem;
}
.datast-upload-progress {
  display: flex;
  align-items: center;
}
.datast-upload-progress .dataset-name {
  text-align: right;
  width: 200px;
  margin-right: 1rem;
}
.datast-upload-progress .dataset-progress {
  flex: 1;
}
.datast-upload-progress .dataset-status {
  width: 100px;
  margin-left: 1rem;
}
.datast-upload-progress .dataset-status .status-flex {
  display: flex;
  align-items: center;
}
/deep/ .el-progress-bar__inner {
  background-color: #21ba45;
}
</style>