Browse Source

feat: improve

master
colorfulberry 4 years ago
parent
commit
ea4cde73d9
4 changed files with 1576 additions and 971 deletions
  1. +1
    -1
      templates/base/head_navbar.tmpl
  2. +194
    -133
      web_src/js/components/MinioUploader.vue
  3. +1225
    -684
      web_src/js/index.js
  4. +156
    -153
      web_src/less/_dataset.less

+ 1
- 1
templates/base/head_navbar.tmpl View File

@@ -9,7 +9,7 @@
</div>

{{if .IsSigned}}
<a class="item {{if .PageIsDashboard}}active{{end}}" href="/">{{.i18n.Tr "dashboard"}}</a>
<a class="item {{if .PageIsDashboard}}active{{end}}" href="/dashboard">{{.i18n.Tr "dashboard"}}</a>
<div class="ui dropdown item">
{{.i18n.Tr "custom.head.community"}}
<i class="dropdown icon"></i>


+ 194
- 133
web_src/js/components/MinioUploader.vue View File

@@ -1,24 +1,22 @@
<template>
<div class="dropzone-wrapper">
<div id="dropzone" class="dropzone">
</div>
<p>{{file_status_text}} {{ status }}</p>
<div class="dropzone-wrapper dataset-files">
<div id="dataset" class="dropzone"></div>
<p class="upload-info">
{{ file_status_text }}
<span class="success">{{ status }}</span>
</p>
</div>
</template>

<script>
// import Dropzone from 'dropzone/dist/dropzone.js';
// import 'dropzone/dist/dropzone.css'
import createDropzone from '../features/dropzone.js';
import createDropzone from "../features/dropzone.js";
import SparkMD5 from 'spark-md5';
import axios from 'axios';
import qs from 'qs';
import axios from "axios";
import qs from "qs";

const {
AppSubUrl,
StaticUrlPrefix,
csrf
} = window.config;
const { AppSubUrl, StaticUrlPrefix, csrf } = window.config;

export default {
data() {
@@ -26,55 +24,81 @@ export default {
dropzoneUploader: null,
maxFiles: 1,
maxFilesize: 1 * 1024 * 1024 * 1024 * 1024,
acceptedFiles: '*/*',
acceptedFiles: "*/*",
progress: 0,
status: '',
status: "",
dropzoneParams: {},
file_status_text: '',
}
file_status_text: ""
};
},

async mounted() {
this.dropzoneParams = $('div#minioUploader-params')
this.file_status_text = this.dropzoneParams.data('file-status')
this.status = this.dropzoneParams.data('file-init-status')
const $dropzone = $('div#dropzone')
let previewTemplate = "";
previewTemplate += '<div class="dz-preview dz-file-preview">\n ';
previewTemplate += ' <div class="dz-details">\n ';
previewTemplate += ' <div class="dz-filename">';
previewTemplate +=
" <span data-dz-name data-dz-thumbnail></span>";
previewTemplate += " </div>\n ";
previewTemplate += ' <div class="dz-size" data-dz-size></div>\n ';
previewTemplate += " </div>\n ";
previewTemplate += ' <div class="dz-progress ui active progress">';
previewTemplate +=
' <div class="dz-upload bar" data-dz-uploadprogress><div class="progress"></div></div>\n ';
previewTemplate += " </div>\n ";
previewTemplate += ' <div class="dz-success-mark">';
previewTemplate += " <span>上传成功</span>";
previewTemplate += " </div>\n ";
previewTemplate += ' <div class="dz-error-mark">';
previewTemplate += " <span>上传失败</span>";
previewTemplate += " </div>\n ";
previewTemplate += ' <div class="dz-error-message">';
previewTemplate += " <span data-dz-errormessage></span>";
previewTemplate += " </div>\n";
previewTemplate += "</div>";

this.dropzoneParams = $("div#minioUploader-params");
this.file_status_text = this.dropzoneParams.data("file-status");
this.status = this.dropzoneParams.data("file-init-status");
const $dropzone = $("div#dataset");
const dropzoneUploader = await createDropzone($dropzone[0], {
url: '/todouploader',
url: "/todouploader",
maxFiles: this.maxFiles,
maxFilesize: this.maxFileSize,
timeout: 0,
autoQueue: false,
dictDefaultMessage: this.dropzoneParams.data('default-message'),
dictInvalidFileType: this.dropzoneParams.data('invalid-input-type'),
dictFileTooBig: this.dropzoneParams.data('file-too-big'),
dictRemoveFile: this.dropzoneParams.data('remove-file'),
})
dropzoneUploader.on("addedfile", (file) => {
dictDefaultMessage: this.dropzoneParams.data("default-message"),
dictInvalidFileType: this.dropzoneParams.data("invalid-input-type"),
dictFileTooBig: this.dropzoneParams.data("file-too-big"),
dictRemoveFile: this.dropzoneParams.data("remove-file"),
previewTemplate
});
dropzoneUploader.on("addedfile", file => {
setTimeout(() => {
file.accepted && this.onFileAdded(file);
}, 200);
});
dropzoneUploader.on("maxfilesexceeded", function(file) {
if (this.files[0].status !== 'success') {
alert(this.dropzoneParams.data('waitting-uploading'))
this.removeFile(file)
return
if (this.files[0].status !== "success") {
alert(this.dropzoneParams.data("waitting-uploading"));
this.removeFile(file);
return;
}
this.removeAllFiles();
this.addFile(file);
});

this.dropzoneUploader = dropzoneUploader

this.dropzoneUploader = dropzoneUploader;
},
methods: {
resetStatus() {
this.progress = 0
this.status = ''
this.progress = 0;
this.status = "";
},
updateProgress(file, progress) {
file.previewTemplate.querySelector(".dz-upload").style.width = `${progress}%`;
file.previewTemplate.querySelector(
".dz-upload"
).style.width = `${progress}%`;
},
emitDropzoneSuccess(file) {
file.status = "success";
@@ -82,54 +106,64 @@ export default {
this.dropzoneUploader.emit("complete", file);
},
onFileAdded(file) {
file.datasetId = document.getElementById("datasetId").getAttribute("datasetId");
this.resetStatus()
file.datasetId = document
.getElementById("datasetId")
.getAttribute("datasetId");
this.resetStatus();
this.computeMD5(file);
},

finishUpload(file) {
this.emitDropzoneSuccess(file)
this.emitDropzoneSuccess(file);
setTimeout(() => {
window.location.reload();
}, 1000);
},

computeMD5(file) {
this.resetStatus()
let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
chunkSize = 1024 * 1024 * 64,
chunks = Math.ceil(file.size / chunkSize),
currentChunk = 0,
spark = new SparkMD5.ArrayBuffer(),
fileReader = new FileReader();
this.resetStatus();
let blobSlice =
File.prototype.slice ||
File.prototype.mozSlice ||
File.prototype.webkitSlice,
chunkSize = 1024 * 1024 * 64,
chunks = Math.ceil(file.size / chunkSize),
currentChunk = 0,
spark = new SparkMD5.ArrayBuffer(),
fileReader = new FileReader();
let time = new Date().getTime();
// console.log('计算MD5...')
this.status = this.dropzoneParams.data('md5-computing')
this.status = this.dropzoneParams.data("md5-computing");
file.totalChunkCounts = chunks;
loadNext();

fileReader.onload = (e) => {
fileLoaded.call(this, e)
fileReader.onload = e => {
fileLoaded.call(this, e);
};
fileReader.onerror = (err) => {
console.warn('oops, something went wrong.', err);
fileReader.onerror = err => {
console.warn("oops, something went wrong.", err);
file.cancel();
};

function fileLoaded(e){
function fileLoaded(e) {
spark.append(e.target.result); // Append array buffer
currentChunk++;
if (currentChunk < chunks) {
// console.log(`第${currentChunk}分片解析完成, 开始第${currentChunk +1}/${chunks}分片解析`);
this.status = `${this.dropzoneParams.data('loading-file')} ${(currentChunk/chunks*100).toFixed(2)}% (${currentChunk}/${chunks})`;
this.updateProgress(file, (currentChunk/chunks*100).toFixed(2))
this.status = `${this.dropzoneParams.data("loading-file")} ${(
(currentChunk / chunks) *
100
).toFixed(2)}% (${currentChunk}/${chunks})`;
this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
loadNext();
return
return;
}

let md5 = spark.end();
console.log(
`MD5计算完成:${file.name} \nMD5:${md5} \n分片:${chunks} 大小:${file.size} 用时:${(new Date().getTime() - time)/1000} s`
`MD5计算完成:${file.name} \nMD5:${md5} \n分片:${chunks} 大小:${
file.size
} 用时:${(new Date().getTime() - time) / 1000} s`
);
spark.destroy(); //释放缓存
file.uniqueIdentifier = md5; //将文件md5赋值给文件唯一标识
@@ -139,14 +173,16 @@ export default {

function loadNext() {
let start = currentChunk * chunkSize;
let end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize;
let end =
start + chunkSize >= file.size ? file.size : start + chunkSize;
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
}
},

async computeMD5Success(md5edFile) {
const file = await this.getSuccessChunks(md5edFile);
if (file.uploadID == "" || file.uuid == "") { //未上传过
if (file.uploadID == "" || file.uuid == "") {
//未上传过
await this.newMultiUpload(file);
if (file.uploadID != "" && file.uuid != "") {
file.chunks = "";
@@ -155,86 +191,94 @@ export default {
//失败如何处理
return;
}
return
return;
}

if (file.uploaded == "1") { //已上传成功
if (file.uploaded == "1") {
//已上传成功
//秒传
if (file.attachID == "0") { //删除数据集记录,未删除文件
if (file.attachID == "0") {
//删除数据集记录,未删除文件
await addAttachment(file);
}
console.log("文件已上传完成");
this.progress = 100;
this.status = this.dropzoneParams.data('upload-complete')
this.finishUpload(file)
this.status = this.dropzoneParams.data("upload-complete");
this.finishUpload(file);
} else {
//断点续传
this.multipartUpload(file);
}


async function addAttachment(file) {
return await axios.post('/attachments/add', qs.stringify({
return await axios.post(
"/attachments/add",
qs.stringify({
uuid: file.uuid,
file_name: file.name,
size: file.size,
dataset_id: file.datasetId,
_csrf: csrf
}))
})
);
}
},

async getSuccessChunks(file) {
const params = {
params: {
md5: file.uniqueIdentifier,
_csrf: csrf
}
params: {
md5: file.uniqueIdentifier,
_csrf: csrf
}
};
try {
const response = await axios.get('/attachments/get_chunks', params)
const response = await axios.get("/attachments/get_chunks", params);
file.uploadID = response.data.uploadID;
file.uuid = response.data.uuid;
file.uploaded = response.data.uploaded;
file.chunks = response.data.chunks;
file.attachID = response.data.attachID;
return file
} catch(error) {
return file;
} catch (error) {
console.log("getSuccessChunks catch: ", error);
return null
return null;
}
},

async newMultiUpload(file) {
const res = await axios.get('/attachments/new_multipart', {
params: {
totalChunkCounts: file.totalChunkCounts,
md5: file.uniqueIdentifier,
size: file.size,
fileType: file.type,
_csrf: csrf
}
})
const res = await axios.get("/attachments/new_multipart", {
params: {
totalChunkCounts: file.totalChunkCounts,
md5: file.uniqueIdentifier,
size: file.size,
fileType: file.type,
_csrf: csrf
}
});
file.uploadID = res.data.uploadID;
file.uuid = res.data.uuid;
},

multipartUpload(file) {
let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
chunkSize = 1024 * 1024 * 64,
chunks = Math.ceil(file.size / chunkSize),
currentChunk = 0,
fileReader = new FileReader(),
time = new Date().getTime();
let blobSlice =
File.prototype.slice ||
File.prototype.mozSlice ||
File.prototype.webkitSlice,
chunkSize = 1024 * 1024 * 64,
chunks = Math.ceil(file.size / chunkSize),
currentChunk = 0,
fileReader = new FileReader(),
time = new Date().getTime();

function loadNext() {
let start = currentChunk * chunkSize;
let end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize;
let end =
start + chunkSize >= file.size ? file.size : start + chunkSize;
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
}

function checkSuccessChunks() {
var index = successChunks.indexOf((currentChunk + 1).toString())
var index = successChunks.indexOf((currentChunk + 1).toString());
if (index == -1) {
return false;
}
@@ -242,36 +286,40 @@ export default {
}

async function getUploadChunkUrl(currentChunk, partSize) {
const res = await axios.get('/attachments/get_multipart_url', {
params: {
uuid: file.uuid,
uploadID: file.uploadID,
size: partSize,
chunkNumber: currentChunk + 1,
_csrf: csrf
}
})
console.log("getUploadChunkUrl: ", res)
urls[currentChunk] = res.data.url
const res = await axios.get("/attachments/get_multipart_url", {
params: {
uuid: file.uuid,
uploadID: file.uploadID,
size: partSize,
chunkNumber: currentChunk + 1,
_csrf: csrf
}
});
console.log("getUploadChunkUrl: ", res);
urls[currentChunk] = res.data.url;
}

async function uploadMinio(url, e) {
const res = await axios.put(url, e.target.result)
const res = await axios.put(url, e.target.result);
etags[currentChunk] = res.headers.etag;
}

async function updateChunk(currentChunk) {
await axios.post('/attachments/update_chunk', qs.stringify({
await axios.post(
"/attachments/update_chunk",
qs.stringify({
uuid: file.uuid,
chunkNumber: currentChunk + 1,
etag: etags[currentChunk],
_csrf: csrf
}))
})
);
}
async function uploadChunk(e) {
if (!checkSuccessChunks()) {
let start = currentChunk * chunkSize;
let partSize = ((start + chunkSize) >= file.size) ? file.size - start : chunkSize;
let partSize =
start + chunkSize >= file.size ? file.size - start : chunkSize;
//获取分片上传url
await getUploadChunkUrl(currentChunk, partSize);
if (urls[currentChunk] != "") {
@@ -287,70 +335,83 @@ export default {
return;
}
}
};
}

async function completeUpload() {
return await axios.post('/attachments/complete_multipart', qs.stringify({
return await axios.post(
"/attachments/complete_multipart",
qs.stringify({
uuid: file.uuid,
uploadID: file.uploadID,
file_name: file.name,
size: file.size,
dataset_id: file.datasetId,
_csrf: csrf
}))
})
);
}

var successChunks = new Array();
var successParts = new Array();
successParts = file.chunks.split(",");
for (let i = 0; i < successParts.length; i++) {
successChunks[i] = successParts[i].split("-")[0].split("\"")[1];
successChunks[i] = successParts[i].split("-")[0].split('"')[1];
}
var urls = new Array();
var etags = new Array();
console.log('上传分片...');
this.status = this.dropzoneParams.data('uploading')
console.log("上传分片...");
this.status = this.dropzoneParams.data("uploading");
loadNext();
fileReader.onload = async (e) => {
fileReader.onload = async e => {
await uploadChunk(e);
currentChunk++;
if (currentChunk < chunks) {
console.log(`第${currentChunk}个分片上传完成, 开始第${currentChunk +1}/${chunks}个分片上传`);
console.log(
`第${currentChunk}个分片上传完成, 开始第${currentChunk +
1}/${chunks}个分片上传`
);
this.progress = Math.ceil((currentChunk / chunks) * 100);
this.updateProgress(file, (currentChunk/chunks*100).toFixed(2))
this.status = `${this.dropzoneParams.data('uploading')} ${(currentChunk/chunks*100).toFixed(2)}%`
this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
this.status = `${this.dropzoneParams.data("uploading")} ${(
(currentChunk / chunks) *
100
).toFixed(2)}%`;
await loadNext();
} else {
await completeUpload();
console.log(`文件上传完成:${file.name} \n分片:${chunks} 大小:${file.size} 用时:${(new Date().getTime() - time)/1000} s`);
console.log(
`文件上传完成:${file.name} \n分片:${chunks} 大小:${
file.size
} 用时:${(new Date().getTime() - time) / 1000} s`
);
this.progress = 100;
this.status = this.dropzoneParams.data('upload-complete')
this.finishUpload(file)
this.status = this.dropzoneParams.data("upload-complete");
this.finishUpload(file);
}
};
},

}
}
}
};
</script>

<style>
.dropzone-wrapper {
margin: 2em auto ;
margin: 2em auto;
}
.ui .dropzone {
border: 2px dashed #0087f5;
box-shadow: none !important;
padding: 0;
min-height: 5rem;
border-radius: 4px;
border: 2px dashed #0087f5;
box-shadow: none !important;
padding: 0;
min-height: 5rem;
border-radius: 4px;
}
.dataset .dataset-files #dataset .dz-preview.dz-file-preview, .dataset .dataset-files #dataset .dz-preview.dz-processing {
display: flex;
align-items: center;
.dataset .dataset-files #dataset .dz-preview.dz-file-preview,
.dataset .dataset-files #dataset .dz-preview.dz-processing {
display: flex;
align-items: center;
}
.dataset .dataset-files #dataset .dz-preview {
border-bottom: 1px solid #dadce0;
min-height: 0;
border-bottom: 1px solid #dadce0;
min-height: 0;
}
</style>

+ 1225
- 684
web_src/js/index.js
File diff suppressed because it is too large
View File


+ 156
- 153
web_src/less/_dataset.less View File

@@ -1,162 +1,165 @@
.dataset {
padding-top: 15px;
.dataset-files {
margin: 0 30px !important;
#dataset {
.dz-preview {
border-bottom: 1px solid rgb(218, 220, 224);
min-height: 0;
}
.dz-preview.dz-file-preview,
.dz-preview.dz-processing {
display: flex;
align-items: center;
.dz-details {
z-index: 1;
opacity: 1 !important;
display: flex;
padding: 0 0 10px 0;
min-width: 60%;
max-width: 60%;
position: relative;
}
.dz-success-mark {
z-index: 0;
position: relative;
top: 0;
left: auto;
color: green;
margin: 0 16px;
}
.dz-error-mark {
z-index: 0;
position: relative;
top: 0;
left: auto;
color: red;
margin: 0 16px;
}
.bar {
background: #21ba45;
padding-top: 15px;
.dataset-files {
margin: 30px 0 !important;
#dataset {
.dz-preview {
border-bottom: 1px solid rgb(218, 220, 224);
min-height: 0;
}
.dz-preview.dz-file-preview,
.dz-preview.dz-processing {
display: flex;
align-items: center;
.dz-details {
z-index: 1;
opacity: 1 !important;
display: flex;
padding: 0 0 10px 0;
min-width: 60%;
max-width: 60%;
position: relative;
}
.dz-success-mark {
z-index: 0;
position: relative;
top: 0;
left: auto;
color: green;
margin: 0 16px;
}
.dz-error-mark {
z-index: 0;
position: relative;
top: 0;
left: auto;
color: red;
margin: 0 16px;
}
.bar {
background: #21ba45;
}
.dz-progress {
left: 4px;
background: grey;
width: 70%;
top: 20px;
margin: 0;
}
.dz-file {
margin-bottom: auto;
}
}
.dz-preview .dz-error-message {
top: 30px;
}
}
.dz-progress {
left: 4px;
background: grey;
width: 70%;
top: 20px;
margin: 0;
.upload-info {
padding-top: 8px;
}
.dz-file {
margin-bottom: auto;
}
#dataset-list {
border-top: 1px solid #dddddd;
margin-top: 20px;
padding-top: 15px;

> li {
list-style: none;

.meta,
.detail {
padding-top: 30px;
padding-bottom: 40px;
}

.meta {
text-align: right;
position: relative;

.tag:not(.icon) {
display: block;
margin-top: 15px;
}

.commit {
display: block;
margin-top: 10px;
}
}

.detail {
border-left: 1px solid #dddddd;

.author {
img {
margin-bottom: -3px;
}
}

.download {
margin-top: 20px;

> a {
.svg {
margin-left: 5px;
margin-right: 5px;
}
}

.list {
padding-left: 0;
border-top: 1px solid #eeeeee;

li {
list-style: none;
display: block;
padding-top: 8px;
padding-bottom: 8px;
border-bottom: 1px solid #eeeeee;

a > .text.right {
margin-right: 5px;
}
}
}
}

.dot {
width: 9px;
height: 9px;
background-color: #cccccc;
z-index: 999;
position: absolute;
display: block;
left: -5px;
top: 40px;
border-radius: 6px;
border: 1px solid #ffffff;
}
}
}
}
.dz-preview .dz-error-message {
top: 30px;
}
}
}
#dataset-list {
border-top: 1px solid #dddddd;
margin-top: 20px;
padding-top: 15px;

> li {
list-style: none;

.meta,
.detail {
padding-top: 30px;
padding-bottom: 40px;
}

.meta {
text-align: right;
position: relative;

.tag:not(.icon) {
display: block;
margin-top: 15px;
}

.commit {
display: block;
margin-top: 10px;
}
}

.detail {
border-left: 1px solid #dddddd;

.author {
img {
margin-bottom: -3px;
}
}

.download {
margin-top: 20px;

> a {
.svg {
margin-left: 5px;
margin-right: 5px;
}
}

.list {
padding-left: 0;
border-top: 1px solid #eeeeee;

li {
list-style: none;
display: block;
padding-top: 8px;
padding-bottom: 8px;
border-bottom: 1px solid #eeeeee;

a > .text.right {
margin-right: 5px;
}
}
}
}

.dot {
width: 9px;
height: 9px;
background-color: #cccccc;
z-index: 999;
position: absolute;
display: block;
left: -5px;
top: 40px;
border-radius: 6px;
border: 1px solid #ffffff;
}
}
}
}
.item {
padding-top: 15px;
padding-bottom: 10px;
border-bottom: 1px dashed #aaaaaa;
}
.ui.grid>.row {
align-items: center;
}
.title {
font-size: 16px;
font-weight: bold;
margin: 0 6px;
}
.item {
padding-top: 15px;
padding-bottom: 10px;
border-bottom: 1px dashed #aaaaaa;
}
.ui.grid > .row {
align-items: center;
}
.title {
font-size: 16px;
font-weight: bold;
margin: 0 6px;
}
}
.dataset-list {
.octicon-check {
padding-right: 8px;;
}
.hide {
display: none;
}
.octicon-check {
padding-right: 8px;
}
.hide {
display: none;
}
}
.ui.dataset.list {
.item {
@@ -200,4 +203,4 @@
height: 24px;
}
}
}
}

Loading…
Cancel
Save