aiforge/web_src/js/components/MinioUploader.vue

<template>
  <div class="dropzone-wrapper dataset-files">
    <div id="dataset" class="dropzone">
        <div class="maxfilesize ui red message" style="display: none;margin: 2.5rem;"></div>
    </div>
    <!-- <p class="upload-info">
      {{ file_status_text }}
      <strong class="success text red">{{ status }}</strong>
    </p> -->
    <el-button style="background-color: #21ba45;margin-top: 2rem;" type="success" :disabled="btnFlag" @click="startUpload">{{upload}}</el-button>
    <el-button type="info" @click="cancelDataset">{{cancel}}</el-button>
    <div style="margin-top: 2rem;position: relative;">
      <label class="el-form-item__label" style="width: 140px;position: absolute;left: -140px;">上传状态:</label>
      <div v-for="item in allUploadFiles" style="display:flex;padding: 0.8rem 0;border-bottom: 1px solid #e8e8e8;line-height: 1;" >
          <span style="flex:4 1 0%;display: flex;"><i :class="[item.status===0?'ri-checkbox-circle-line success':'ri-close-circle-line failed']" style="margin-right: 0.5rem;"></i>{{item.name}}</span>
          
            <span style="flex:1" v-if="item.status===0"><span style="color: #21ba45;">上传成功</span></span>
          
          <span style="flex:1" v-else-if="item.status===1">
            <el-tooltip class="item" effect="dark" placement="top">
              <div slot="content">{{item.info}}</div>
              <span style="color: red;cursor: pointer;">上传失败<span>(重复上传)</span></span>
            </el-tooltip>
          </span>
          <span style="flex:1" v-else><span style="color: red;">上传失败</span></span>
    
        </div>
    </div>
    
    <!-- <p>说明：<br>
      - 只有zip格式的数据集才能发起云脑任务；<br>
      - 云脑1提供 <span class="text blue">CPU / GPU</span> 资源，云脑2提供 <span class="text blue">Ascend NPU</span> 资源；调试使用的数据集也需要上传到对应的环境。</p> -->
  </div>
</template>

<script>
import SparkMD5 from 'spark-md5';
import axios from 'axios';
import qs from 'qs';
import createDropzone from '../features/dropzone.js';

const {_AppSubUrl, _StaticUrlPrefix, csrf} = window.config;
const chunkSize = 1024 * 1024 * 64;
const md5ChunkSize = 1024 * 1024 * 1;

export default {
  props:{
    uploadtype:{
      type:Number,
      required:true
    },
    desc:{
      type:String,
      default:''
    }
  },
  data() {
    return {
      dropzoneUploader: null,
      maxFiles: 10,
      maxFilesize: 200 ,
      acceptedFiles: '*/*',
      progress: 0,
      status: '',
      dropzoneParams: {},
      file_status_text: '',
      file:{},
      repoPath:'',
      btnFlag:false,
      cancel:'',
      upload:'',
      uploadFiles:[],
      uploadFilesAddId:[],
      allUploadFiles:[],
      uploadLength:0,
      allUploadLength:0,
    };
  },

  async mounted() {
    this.dropzoneParams = $('div#minioUploader-params');
    this.file_status_text = this.dropzoneParams.data('file-status');
    this.status = this.dropzoneParams.data('file-init-status');
    this.repoPath = this.dropzoneParams.data('repopath');
    this.cancel = this.dropzoneParams.data('cancel');
    this.upload = this.dropzoneParams.data('upload');
    let previewTemplate = `
        <div class="dz-preview dz-file-preview"> 
          <div class="dz-image"> 
            <img data-dz-thumbnail /> 
          </div> 
          <div class="dz-details"> 
            <div class="dz-size"><span data-dz-size></span></div> 
            <div class="dz-filename"><span data-dz-name></span></div> 
          </div> 

          <div class="dz-progress"><span class="dz-upload" data-dz-uploadprogress></span></div> 
          <div class="dz-error-message" style="line-height: 1.5;"><span data-dz-errormessage></span></div> 
          <div class="dz-success-mark"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="54" height="54"><path fill="none" d="M0 0h24v24H0z"/><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16zm-.997-4L6.76 11.757l1.414-1.414 2.829 2.829 5.656-5.657 1.415 1.414L11.003 16z" fill="rgba(47,204,113,1)"/></svg></div> 
          <div class="dz-error-mark"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="54" height="54"><path fill="none" d="M0 0h24v24H0z"/><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16zm0-9.414l2.828-2.829 1.415 1.415L13.414 12l2.829 2.828-1.415 1.415L12 13.414l-2.828 2.829-1.415-1.415L10.586 12 7.757 9.172l1.415-1.415L12 10.586z" fill="rgba(231,76,60,1)"/></svg></div> 
        </div> `
    const $dropzone = $('div#dataset');
    const dropzoneUploader = await createDropzone($dropzone[0], {
      url: '/todouploader',
      maxFiles: this.maxFiles,
      maxFilesize: 1024*10,
      filesizeBase:1024,
      parallelUploads: this.maxFiles,
      timeout: 0,
      addRemoveLinks:true,
      // autoQueue: false,
      autoProcessQueue: false, //自动上传
      dictDefaultMessage: this.dropzoneParams.data('default-message'),
      dictInvalidFileType: this.dropzoneParams.data('invalid-input-type'),
      dictFileTooBig: this.dropzoneParams.data('file-too-big'),
      dictRemoveFile: this.dropzoneParams.data('remove-file'),
      previewTemplate:previewTemplate
    });
    dropzoneUploader.on('addedfile', (file) => {
      if(file.size/(1024*1024)>dropzoneUploader.options.maxFilesize){
        dropzoneUploader.removeFile(file)
        $('.maxfilesize.ui.red.message').text("单次最多上传10个文件，单个文件不超过200G")
        $('.maxfilesize.ui.red.message').css('display','block')
        
      }else{
        this.file = file
        $('.maxfilesize.ui.red.message').css('display','none')
        
      }
    });
    dropzoneUploader.on("removedfile",(file)=>{
      $('.maxfilesize.ui.red.message').css('display','none')
    })
    dropzoneUploader.on('maxfilesexceeded', function (file) {
      dropzoneUploader.removeFile(file)
      $('.maxfilesize.ui.red.message').text("单次最多上传10个文件，单个文件不超过200G")
      $('.maxfilesize.ui.red.message').css('display','block')

    });
    this.dropzoneUploader = dropzoneUploader;
  },
  watch:{
    allUploadLength(len){
        if(len===this.uploadFiles.length){
          setTimeout(() => {
          this.dropzoneUploader.removeAllFiles(true)
          this.btnFlag = false
          this.$emit('setcluster',this.btnFlag)
        }, 2000);
        }
    }
  },
  methods: {
    startUpload(){
      this.uploadFiles = this.dropzoneUploader.getQueuedFiles()
      if(this.uploadFiles.length===0){
        return
      }
      this.resetStatus()
      $('.dz-remove').remove()
      $('.maxfilesize.ui.red.message').css('display','none')
      this.btnFlag = true
      this.$emit('setcluster',this.btnFlag)
      this.uploadFiles.forEach(element => {
        element.datasetId = document.getElementById('datasetId').getAttribute('datasetId')
        this.computeMD5(element)
      });
    },
    cancelDataset(){
      location.href = this.repoPath
      this.dropzoneUploader.removeAllFiles(true)
    },
    resetStatus() {
      this.uploadLength = 0
      this.allUploadLength = 0
      this.allUploadFiles = []
    },
    updateProgress(file, progress) {
      file.previewTemplate.querySelector(
        '.dz-upload'
      ).style.width = `${progress}%`
      file.previewTemplate.querySelector(
        '.dz-upload'
      ).style.background = '#409eff';
    },
    uploadError(file,info){
      file.previewTemplate.querySelector(
        '.dz-error-mark'
      ).style.opacity = 1
      file.previewTemplate.querySelector(
        '.dz-progress'
      ).style.opacity = 0
      file.previewTemplate.querySelector(
        '.dz-error-message span'
      ).innerHTML = info
      file.previewTemplate.querySelector(
        '.dz-error-message'
      ).style.display = 'block'
      file.previewTemplate.querySelector(
        '.dz-details'
      ).onmouseover = function(){file.previewTemplate.querySelector('.dz-error-message').style.opacity = 1 }
      file.previewTemplate.querySelector(
        '.dz-details'
      ).onmouseout = function(){file.previewTemplate.querySelector('.dz-error-message').style.opacity = 0 }
    },
    emitDropzoneSuccess(file) {
      file.status = 'success';
      this.dropzoneUploader.emit('success', file);
      this.dropzoneUploader.emit('complete', file);
    },
    emitDropzoneFailed(file) {
      this.status = this.dropzoneParams.data('falied');
      file.status = 'error';
      this.dropzoneUploader.emit('error', file);
      // this.dropzoneUploader.emit('complete', file);
    },
    finishUpload(file) {
      file.previewTemplate.querySelector(
        '.dz-success-mark'
      ).style.opacity = 1
      file.previewTemplate.querySelector(
        '.dz-progress'
      ).style.opacity = 0
      if(this.uploadLength === this.uploadFiles.length){
        setTimeout(() => {
        window.location.href = this.repoPath
        }, 1000);
      }
    },

    computeMD5(file) {
      const blobSlice =
          File.prototype.slice ||
          File.prototype.mozSlice ||
          File.prototype.webkitSlice,
        chunks = Math.ceil(file.size / chunkSize),
        spark = new SparkMD5.ArrayBuffer(),
        fileReader = new FileReader();
      let currentChunk = 0;
      console.log("computeMD5-chunks:",chunks)
      const time = new Date().getTime();
      this.status = this.dropzoneParams.data('md5-computing');
      file.totalChunkCounts = chunks;
      if (file.size==0) {
        file.totalChunkCounts = 1
      }
      loadMd5Next();

      fileReader.onload = (e) => {
        fileLoaded.call(this, e);
      };
      fileReader.onerror = (err) => {
        console.warn('oops, something went wrong.', err);
        file.cancel();
      };

      function fileLoaded(e) {
        spark.append(e.target.result); // Append array buffer
        currentChunk++;
        if (currentChunk < chunks) {
          this.status = `${this.dropzoneParams.data('loading-file')} ${(
            (currentChunk / chunks) *
            100
          ).toFixed(2)}% (${currentChunk}/${chunks})`;
          this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
          loadMd5Next();
          return;
        }

        const md5 = spark.end();
        console.log(
          `MD5计算完成：${file.name} \nMD5：${md5} \n分片：${chunks} 大小:${
            file.size
          } 用时：${(new Date().getTime() - time) / 1000} s`
        );
        this.updateProgress(file,100)
        spark.destroy(); // 释放缓存
        file.uniqueIdentifier = md5; // 将文件md5赋值给文件唯一标识
        file.cmd5 = false; // 取消计算md5状态
        this.computeMD5Success(file);
      }

      function loadNext() {
        const start = currentChunk * chunkSize;
        const end =
          start + chunkSize >= file.size ? file.size : start + chunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }

      function loadMd5Next() {
        const start = currentChunk * chunkSize;
        const end =
          start + md5ChunkSize >= file.size ? file.size : start + md5ChunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }
    },

    async computeMD5Success(md5edFile) {
      const file = await this.getSuccessChunks(md5edFile);
      try {
        if (file.uploadID == '' || file.uuid == '') {
        // 未上传过
          await this.newMultiUpload(file);
          if (file.uploadID != '' && file.uuid != '') {
            file.chunks = '';
            this.multipartUpload(file);
          } else {
          // 失败如何处理
          let info = "上传失败"
          this.allUploadLength++
          this.uploadError(file,info)
          this.allUploadFiles.push({name:file.name,status:2,info:info})
            return;
          }
          return;
        }

        if (file.uploaded == '1') {
        // 已上传成功
        // 秒传
          if (file.attachID == '0') {
          // 删除数据集记录，未删除文件
            await addAttachment(file);
          }
          //不同数据集上传同一个文件
          if (file.datasetID != '') {
            if (file.datasetName != "" && file.realName != "") {
              // var info = "该文件已上传，对应数据集（" + file.datasetName + "）-文件（" + file.realName + ")";
              // window.location.reload();
              let info = `该文件已上传在数据集: ${file.datasetName}`
              this.uploadError(file,info)
              this.allUploadLength++
              this.allUploadFiles.push({name:file.name,status:1,info:info})
            }
          }
          console.log('文件已上传完成');
          this.progress = 100;
          this.status = this.dropzoneParams.data('upload-complete');
          // this.finishUpload(file);
        } else {
        // 断点续传
          this.multipartUpload(file);
        }
      } catch (error) {
        this.emitDropzoneFailed(file);
        console.log(error);
      }

      async function addAttachment(file) {
        return await axios.post(
          '/attachments/add',
          qs.stringify({
            uuid: file.uuid,
            file_name: file.name,
            size: file.size,
            dataset_id: file.datasetId,
            type: this.uploadtype,
            _csrf: csrf
          })
        );
      }
    },

    async getSuccessChunks(file) {
      const params = {
        params: {
          md5: file.uniqueIdentifier,
          type: this.uploadtype,
          file_name: file.name,
          _csrf: csrf
        }
      };
      try {
        const response = await axios.get('/attachments/get_chunks', params);
        file.uploadID = response.data.uploadID;
        file.uuid = response.data.uuid;
        file.uploaded = response.data.uploaded;
        file.chunks = response.data.chunks;
        file.attachID = response.data.attachID;
        file.datasetID = response.data.datasetID;
        file.datasetName = response.data.datasetName;
        file.realName = response.data.fileName;
        return file;
      } catch (error) {
        this.emitDropzoneFailed(file);
        console.log('getSuccessChunks catch: ', error);
        return null;
      }
    },

    async newMultiUpload(file) {
      const res = await axios.get('/attachments/new_multipart', {
        params: {
          totalChunkCounts: file.totalChunkCounts,
          md5: file.uniqueIdentifier,
          size: file.size,
          fileType: file.type,
          type: this.uploadtype,
          file_name: file.name,
          _csrf: csrf
        }
      });
      file.uploadID = res.data.uploadID;
      file.uuid = res.data.uuid;
    },

    multipartUpload(file) {
      const blobSlice =
          File.prototype.slice ||
          File.prototype.mozSlice ||
          File.prototype.webkitSlice,
        chunks = Math.ceil(file.size / chunkSize),
        fileReader = new FileReader(),
        time = new Date().getTime();
      let currentChunk = 0;
      let _this = this

      function loadNext() {
        const start = currentChunk * chunkSize;
        const end =
          start + chunkSize >= file.size ? file.size : start + chunkSize;
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
      }

      function checkSuccessChunks() {
        const index = successChunks.indexOf((currentChunk + 1).toString());
        if (index == -1) {
          return false;
        }
        return true;
      }

      async function getUploadChunkUrl(currentChunk, partSize) {
        const res = await axios.get('/attachments/get_multipart_url', {
          params: {
            uuid: file.uuid,
            uploadID: file.uploadID,
            size: partSize,
            chunkNumber: currentChunk + 1,
            type: _this.uploadtype,
            file_name: file.name,
            _csrf: csrf
          }
        });
        urls[currentChunk] = res.data.url;
      }

      async function uploadMinio(url, e) {
        const res = await axios.put(url, e.target.result);
        delete e.target.result
        etags[currentChunk] = res.headers.etag;
       
      }
      
    async function uploadMinioNewMethod(url,e){
    
		 var xhr = new XMLHttpRequest();
     xhr.open('PUT', url, false);
     if(_this.uploadtype===0){
      xhr.setRequestHeader('Content-Type', 'text/plain')
      xhr.send(e.target.result);
     
      var etagValue = xhr.getResponseHeader('etag');
      etags[currentChunk] = etagValue;
     }
     else if(_this.uploadtype===1){
      xhr.setRequestHeader('Content-Type', '')
      xhr.send(e.target.result);
      var etagValue = xhr.getResponseHeader('ETag');
      //console.log(etagValue);
      etags[currentChunk] = etagValue;
     }
		 
     
		 
      }

      async function updateChunk(currentChunk) {
        await axios.post(
          '/attachments/update_chunk',
          qs.stringify({
            uuid: file.uuid,
            chunkNumber: currentChunk + 1,
            etag: etags[currentChunk],
            _csrf: csrf
          })
        );
      }
      async function uploadChunk(e) {
        try {
          if (!checkSuccessChunks()) {
            const start = currentChunk * chunkSize;
            const partSize =
            start + chunkSize >= file.size ? file.size - start : chunkSize;
            // 获取分片上传url
            await getUploadChunkUrl(currentChunk, partSize);
            if (urls[currentChunk] != '') {
              // 上传到minio
              //await uploadMinio(urls[currentChunk], e);
              await uploadMinioNewMethod(urls[currentChunk], e);
              
              if (etags[currentChunk] != '') {
              // 更新数据库：分片上传结果
                //await updateChunk(currentChunk);
              } else {
                console.log("上传到minio uploadChunk etags[currentChunk] == ''");// TODO
              }
            } else {
              console.log("uploadChunk urls[currentChunk] != ''");// TODO
            }
          }
        } catch (error) {
          console.log(error);
          //this.emitDropzoneFailed(file);
          //console.log(error);
        }
      }

      async function completeUpload() {
        return await axios.post(
          '/attachments/complete_multipart',
          qs.stringify({
            uuid: file.uuid,
            uploadID: file.uploadID,
            file_name: file.name,
            size: file.size,
            dataset_id: file.datasetId,
            type: _this.uploadtype,
            _csrf: csrf,
            description:_this.desc
          })
        );
      }

      const successChunks = [];
      let successParts = [];
      successParts = file.chunks.split(',');
      for (let i = 0; i < successParts.length; i++) {
        successChunks[i] = successParts[i].split('-')[0];
      }
      const urls = []; // TODO const ?
      const etags = [];
      console.log('上传分片...');
      this.status = this.dropzoneParams.data('uploading');
      loadNext();
      fileReader.onload = async (e) => {
        try{
          await uploadChunk(e);
        }catch(err){
          console.log(err)
        }
        
        fileReader.abort();
        currentChunk++;
        if (currentChunk < chunks) {
          console.log(
            `第${currentChunk}个分片上传完成, 开始第${currentChunk +
              1}/${chunks}个分片上传`
          );
          this.progress = Math.ceil((currentChunk / chunks) * 100);
          this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
          this.status = `${this.dropzoneParams.data('uploading')} ${(
            (currentChunk / chunks) *
            100
          ).toFixed(2)}%`;
          await loadNext();
        } else {
          try{
            await completeUpload();
          }catch(err){
            let info = "上传失败"
            this.allUploadLength++
            this.uploadError(file,info)
            this.allUploadFiles.push({name:file.name,status:2,info:info})
            if(err){
              return
            }
          }
          
          
          console.log(
            `文件上传完成：${file.name} \n分片：${chunks} 大小:${
              file.size
            } 用时：${(new Date().getTime() - time) / 1000} s`
          );
          this.uploadLength++
          this.allUploadLength++
          this.allUploadFiles.push({name:file.name,status:0,info:'上传成功'})
          this.updateProgress(file, 100);
          this.progress = 100;
          this.status = this.dropzoneParams.data('upload-complete');
          this.finishUpload(file);
        }
      };
    }
  }
};
</script>

<style>
.dropzone-wrapper {
  margin: 0;
}
.ui .dropzone {
  border: 2px dashed #0087f5;
  box-shadow: none !important;
  padding: 0;
  min-height: 5rem;
  border-radius: 4px;
}
.dataset .dataset-files #dataset .dz-preview.dz-file-preview,
.dataset .dataset-files #dataset .dz-preview.dz-processing {
  display: flex;
  align-items: center;
}
.dataset .dataset-files #dataset .dz-preview {
  border-bottom: 1px solid #dadce0;
  min-height: 0;
}
.upload-info{
  margin-top: 1em;
  margin-bottom: 3em;
}
.success{
  color: #21ba45;
}
.failed{
  color: red;
}
</style>