Merge pull request '修复npu环境上传数据集解压失败的问题' (#1929) from fix-1912 into V20220415

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/1929
Reviewed-by: zouap <zouap@pcl.ac.cn>

modules/storage/obs.go

@@ -30,6 +30,8 @@ type FileInfo struct {
 }
 type FileInfoList []FileInfo
 
+const MAX_LIST_PARTS = 1000
+
 func (ulist FileInfoList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] }
 func (ulist FileInfoList) Len() int      { return len(ulist) }
 func (ulist FileInfoList) Less(i, j int) bool {
@@ -97,29 +99,48 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error {
 	input.Bucket = setting.Bucket
 	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
 	input.UploadId = uploadID
-	output, err := ObsCli.ListParts(&obs.ListPartsInput{
-		Bucket:   setting.Bucket,
-		Key:      input.Key,
-		UploadId: uploadID,
-	})
-	if err != nil {
-		log.Error("ListParts failed:", err.Error())
-		return err
-	}
 
-	for _, partInfo := range output.Parts {
-		input.Parts = append(input.Parts, obs.Part{
-			PartNumber: partInfo.PartNumber,
-			ETag:       partInfo.ETag,
+	partNumberMarker := 0
+	for {
+		output, err := ObsCli.ListParts(&obs.ListPartsInput{
+			Bucket:           setting.Bucket,
+			Key:              input.Key,
+			UploadId:         uploadID,
+			MaxParts:         MAX_LIST_PARTS,
+			PartNumberMarker: partNumberMarker,
 		})
+		if err != nil {
+			log.Error("ListParts failed:", err.Error())
+			return err
+		}
+
+		partNumberMarker = output.NextPartNumberMarker
+		log.Info("uuid:%s, MaxParts:%d, PartNumberMarker:%d, NextPartNumberMarker:%d, len:%d", uuid, output.MaxParts, output.PartNumberMarker, output.NextPartNumberMarker, len(output.Parts))
+
+		for _, partInfo := range output.Parts {
+			input.Parts = append(input.Parts, obs.Part{
+				PartNumber: partInfo.PartNumber,
+				ETag:       partInfo.ETag,
+			})
+		}
+
+		if len(output.Parts) < output.MaxParts {
+			break
+		} else {
+			continue
+		}
+
+		break
 	}
 
-	_, err = ObsCli.CompleteMultipartUpload(input)
+	output, err := ObsCli.CompleteMultipartUpload(input)
 	if err != nil {
 		log.Error("CompleteMultipartUpload failed:", err.Error())
 		return err
 	}
 
+	log.Info("uuid:%s, RequestId:%s", uuid, output.RequestId)
+
 	return nil
 }
 

web_src/js/components/MinioUploader.vue

@@ -26,7 +26,7 @@ import qs from 'qs';
 import createDropzone from '../features/dropzone.js';
 
 const {_AppSubUrl, _StaticUrlPrefix, csrf} = window.config;
-// const uploadtype = 0;
+const chunkSize = 1024 * 1024 * 64;
 
 export default {
   props:{
@@ -137,7 +137,6 @@ export default {
     resetStatus() {
       this.progress = 0;
       this.status = '';
-      console.log(this.uploadtype)
     },
     updateProgress(file, progress) {
       console.log("progress---",progress)
@@ -165,7 +164,6 @@ export default {
         .getElementById('datasetId')
         .getAttribute('datasetId');
       this.resetStatus();
-      console.log(this.file,!this.file?.upload)
       if(!this.file?.upload){
         this.btnFlag = false
         return 
@@ -186,7 +184,6 @@ export default {
           File.prototype.slice ||
           File.prototype.mozSlice ||
           File.prototype.webkitSlice,
-        chunkSize = 1024 * 1024 * 64,
         chunks = Math.ceil(file.size / chunkSize),
         spark = new SparkMD5.ArrayBuffer(),
         fileReader = new FileReader();
@@ -327,7 +324,6 @@ export default {
     },
 
     async newMultiUpload(file) {
-      console.log(this.uploadtype,this)
       const res = await axios.get('/attachments/new_multipart', {
         params: {
           totalChunkCounts: file.totalChunkCounts,
@@ -348,7 +344,6 @@ export default {
           File.prototype.slice ||
           File.prototype.mozSlice ||
           File.prototype.webkitSlice,
-        chunkSize = 1024 * 1024 * 32,
         chunks = Math.ceil(file.size / chunkSize),
         fileReader = new FileReader(),
         time = new Date().getTime();
@@ -457,7 +452,6 @@ export default {
       }
 
       async function completeUpload() {
-        console.log(_this.uploadtype)
         return await axios.post(
           '/attachments/complete_multipart',
           qs.stringify({
@@ -494,7 +488,6 @@ export default {
               1}/${chunks}个分片上传`
           );
           this.progress = Math.ceil((currentChunk / chunks) * 100);
-          console.log("((currentChunk / chunks) * 100).toFixed(2)",((currentChunk / chunks) * 100).toFixed(2))
           this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
           this.status = `${this.dropzoneParams.data('uploading')} ${(
             (currentChunk / chunks) *