Merge branch 'multi-dataset' of git.openi.org.cn:OpenI/aiforge into multi-dataset

routers/repo/modelarts.go

@@ -991,19 +991,6 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 	FlavorName := form.FlavorName
 	VersionCount := modelarts.VersionCount
 	EngineName := form.EngineName
-	if IsDatasetUseCountExceed(uuid) {
-		log.Error("DatasetUseCount is Exceed:%v")
-		trainJobErrorNewDataPrepare(ctx, form)
-		ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobNew, &form)
-		return
-	}
-	_, datasetNames, err := models.GetDatasetInfo(uuid)
-	if err != nil {
-		log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"])
-		trainJobErrorNewDataPrepare(ctx, form)
-		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplModelArtsTrainJobNew, &form)
-		return
-	}
 
 	count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID)
 	if err != nil {
@@ -1114,15 +1101,15 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 			Value: modelarts.Ascend,
 		})
 	}
-	DatasUrlList, dataUrl, isMultiDataset, err := GetDatasUrlListByUUIDS(uuid)
+	datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid)
 	if err != nil {
-		log.Error("Failed to GetDatasUrlListByUUIDS: %v", err)
+		log.Error("Failed to getDatasUrlListByUUIDS: %v", err)
 		trainJobErrorNewDataPrepare(ctx, form)
-		ctx.RenderWithErr("Failed to GetDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobNew, &form)
+		ctx.RenderWithErr("Failed to getDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobNew, &form)
 		return
 	}
 	dataPath := dataUrl
-	jsondatas, err := json.Marshal(DatasUrlList)
+	jsondatas, err := json.Marshal(datasUrlList)
 	if err != nil {
 		log.Error("Failed to Marshal: %v", err)
 		trainJobErrorNewDataPrepare(ctx, form)
@@ -1270,20 +1257,6 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	EngineName := form.EngineName
 	isLatestVersion := modelarts.IsLatestVersion
 
-	if IsDatasetUseCountExceed(uuid) {
-		log.Error("DatasetUseCount is Exceed:%v")
-		versionErrorDataPrepare(ctx, form)
-		ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobVersionNew, &form)
-		return
-	}
-	_, datasetNames, err := models.GetDatasetInfo(uuid)
-	if err != nil {
-		log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"])
-		versionErrorDataPrepare(ctx, form)
-		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplModelArtsTrainJobVersionNew, &form)
-		return
-	}
-
 	canNewJob, _ := canUserCreateTrainJobVersion(ctx, latestTask.UserID)
 	if !canNewJob {
 		ctx.RenderWithErr("user cann't new trainjob", tplModelArtsTrainJobVersionNew, &form)
@@ -1368,15 +1341,16 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 			Value: modelarts.Ascend,
 		})
 	}
-	DatasUrlList, dataUrl, isMultiDataset, err := GetDatasUrlListByUUIDS(uuid)
+
+	datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid)
 	if err != nil {
-		log.Error("Failed to GetDatasUrlListByUUIDS: %v", err)
+		log.Error("Failed to getDatasUrlListByUUIDS: %v", err)
 		versionErrorDataPrepare(ctx, form)
-		ctx.RenderWithErr("Failed to GetDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
+		ctx.RenderWithErr("Failed to getDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
 	dataPath := dataUrl
-	jsondatas, err := json.Marshal(DatasUrlList)
+	jsondatas, err := json.Marshal(datasUrlList)
 	if err != nil {
 		log.Error("Failed to Marshal: %v", err)
 		versionErrorDataPrepare(ctx, form)
@@ -2508,43 +2482,50 @@ func TrainJobDownloadLogFile(ctx *context.Context) {
 	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
-func GetDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, bool, error) {
+func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) {
+	var isMultiDataset bool
 	var dataUrl string
-	var DatasUrlList []models.Datasurl
-	uuidList := strings.Split(uuidStr, ";")
-	isMultiDataset := false
-
-	if len(uuidList) >= 1 {
-		for _, uuid := range uuidList {
-			attach, err := models.GetAttachmentByUUID(uuid)
-			if err != nil {
-				log.Error("GetAttachmentByUUID failed: %v", err)
-				return nil, "", isMultiDataset, err
+	var datasetNames string
+	var datasUrlList []models.Datasurl
+	uuids := strings.Split(uuidStr, ";")
+	if len(uuids) > setting.MaxDatasetNum {
+		log.Error("the dataset count(%d) exceed the limit", len(uuids))
+		return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("the dataset count exceed the limit")
+	}
+
+	datasetInfos := make(map[string]models.DatasetInfo)
+	attachs, err := models.GetAttachmentsByUUIDs(uuids)
+	if err != nil {
+		log.Error("GetAttachmentsByUUIDs failed: %v", err)
+		return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("GetAttachmentsByUUIDs failed")
+	}
+	for i, attach := range attachs {
+		fileName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(attach.Name, ".zip"), ".tar.gz"), ".tgz")
+		for _, datasetInfo := range datasetInfos {
+			if fileName == datasetInfo.Name {
+				log.Error("the dataset name is same: %v", attach.Name)
+				return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("the dataset name is same")
 			}
-			datasetName := attach.Name
-			index := strings.LastIndex(datasetName, ".")
-			if index <= 0 {
-				index = 0
-			}
-			datasetNameHead := datasetName[:index]
-			datasetUrl := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
-			DatasUrlList = append(DatasUrlList, models.Datasurl{
+		}
+		if len(attachs) <= 1 {
+			dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + attach.UUID + "/"
+			isMultiDataset = false
+		} else {
+			dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(attachs[0].UUID[0:1], attachs[0].UUID[1:2]) + "/" + attachs[0].UUID + attachs[0].UUID + "/"
+			datasetUrl := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + attach.UUID + "/"
+			datasUrlList = append(datasUrlList, models.Datasurl{
 				DatasetUrl:  datasetUrl,
-				DatasetName: datasetNameHead,
+				DatasetName: fileName,
 			})
+			isMultiDataset = true
+		}
+
+		if i == 0 {
+			datasetNames = attach.Name
+		} else {
+			datasetNames += ";" + attach.Name
 		}
-		firstDataset := uuidList[0]
-		dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(firstDataset[0:1], firstDataset[1:2]) + "/" + firstDataset + firstDataset + "/"
-		isMultiDataset = true
-		return DatasUrlList, dataUrl, isMultiDataset, nil
-	}
-	return nil, "", isMultiDataset, nil
-}
-func IsDatasetUseCountExceed(uuid string) bool {
-	uuidList := strings.Split(uuid, ";")
-	if len(uuidList) > setting.MaxDatasetNum {
-		return true
-	} else {
-		return false
 	}
+
+	return datasUrlList, dataUrl, datasetNames, isMultiDataset, nil
 }

templates/repo/cloudbrain/trainjob/new.tmpl

@@ -30,7 +30,7 @@
         margin-left: 10.5rem !important;
         align-items: center;
     }
-
+    
     .width81 {
         margin-left: 1.5rem !important;
         width: 81% !important;
@@ -62,6 +62,7 @@
         color: rgba(16, 16, 16, 100) !important;
         font-size: 14px !important;
     }
+
 </style>
 <!-- <div class="ui page dimmer">
     <div class="ui text loader">{{.i18n.Tr "loading"}}</div>
@@ -280,7 +281,11 @@
 
     $('.menu .item')
         .tab();
-
+        $(document).keydown(function(event){
+        switch(event.keyCode){
+            case 13:return false; 
+        }
+    });
     let sever_num = $('#trainjob_work_server_num')
     $('.add').click(function () {
         sever_num.val(parseInt(sever_num.val()) + 1)

templates/repo/grampus/trainjob/gpu/new.tmpl

@@ -24,7 +24,8 @@
 }
 .width85{
     width: 85% !important;
-    margin-left: 4.5rem !important;
+    margin-left: 10.5rem !important;
+    align-items: center;
 }
 .width81{
     width: 81% !important;
@@ -120,7 +121,7 @@
                 </div>
                 
                 <div class="min_title inline field">
-                    <label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}&nbsp;&nbsp;</label>
+                    <label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
                     <textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
                 </div>
                 <div class="ui divider"></div>

templates/repo/grampus/trainjob/npu/new.tmpl

@@ -19,7 +19,8 @@
 }
 .width85{
     width: 85% !important;
-    margin-left: 4.5rem !important;
+    margin-left: 10.5rem !important;
+    align-items: center;
 }
 .width81{
     width: 81% !important;
@@ -115,7 +116,7 @@
                 </div>
 
                 <div class="min_title inline field">
-                    <label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}&nbsp;&nbsp;</label>
+                    <label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
                     <textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
                 </div>
                 <div class="ui divider"></div>

templates/repo/modelarts/trainjob/new.tmpl

@@ -294,7 +294,11 @@
 
     $('.menu .item')
         .tab();
-
+    $(document).keydown(function(event){
+        switch(event.keyCode){
+            case 13:return false; 
+        }
+    });
     // let sever_num = $("#trainjob_work_server_num_select .text").text() //$('#trainjob_work_server_num')
     // console.log("sever_num:",sever_num)
     // $('.add').click(function(){

web_src/js/components/dataset/selectDataset.vue

@@ -23,7 +23,7 @@
       <input type="hidden" name="attachment" :value="confirmDatasetList" />
       <div class="multi-dataset-box">
         <span
-          v-for="(item, index) in checkList"
+          v-for="(item, index) in confirmChecklist"
           :key="index"
           class="select-dataset-label"
           :title="item"
@@ -581,6 +581,7 @@ export default {
       repoLink: "",
       selectDatasetArray: [],
       checkList: [],
+      confirmChecklist: [],
       confirmDatasetList: "",
       confirmFlag: false,
 
@@ -643,18 +644,12 @@ export default {
 
     //tree 勾选触发事件
     onCheck(data, checkedInfo) {
-      console.log("--ref------", this.$refs[data.ref]);
-      console.log("---------oncheck-----", data, checkedInfo);
-
-      console.log("start array", this.selectDatasetArray);
       if (
         this.selectDatasetArray.length === 0 ||
         this.selectDatasetArray.every((item) => item.id !== data.id)
       ) {
-        console.log("111111111111");
         if (
           this.selectDatasetArray.some((item) => {
-            console.log(item.label.split(".")[0], data.label.split(".")[0]);
             return item.label.split(".")[0] === data.label.split(".")[0];
           })
         ) {
@@ -664,18 +659,13 @@ export default {
           this.$refs[data.ref].setChecked(data.id, false, false);
           this.$message.error("最多不超过五个文件");
         } else {
-          console.log("push");
           this.selectDatasetArray.push(data);
         }
       } else {
-        console.log("55555555555");
         let index = this.selectDatasetArray.findIndex((item) => {
-          console.log(item.id, data.id);
           return item.id === data.id;
         });
-        console.log("index", index);
         this.selectDatasetArray.splice(index, 1);
-        console.log(this.selectDatasetArray);
       }
       this.checkList = this.selectDatasetArray.map((item) => {
         return item.label;
@@ -683,16 +673,10 @@ export default {
       this.saveStatusList = this.selectDatasetArray.map((item) => {
         return item.UUID;
       });
-      this.confirmDatasetList = this.saveStatusList.join(";");
-      console.log(this.selectDatasetArray, this.checkList);
-      console.log("this.saveStatusList", this.saveStatusList);
-      console.log("this.confirmDatasetList", this.confirmDatasetList);
-
-      console.log("======================================");
+      //   this.confirmDatasetList = this.saveStatusList.join(";");
     },
     //已选择数据集checkbox group 勾选事件
     changeCheckbox(checked, data) {
-      console.log(checked, data);
       this.$refs.currentTree.setChecked(data.id, false, false);
       this.$refs.myTree.setChecked(data.id, false, false);
       this.$refs.publicTree.setChecked(data.id, false, false);
@@ -700,15 +684,9 @@ export default {
       let index = this.selectDatasetArray.findIndex((item) => {
         return item.id === data.id;
       });
-      console.log("index", index);
       this.selectDatasetArray.splice(index, 1);
       this.saveStatusList.splice(index, 1);
-      console.log(this.selectDatasetArray);
-      this.confirmDatasetList = this.saveStatusList.join(";");
-      console.log("selectDatasetArray--:", this.selectDatasetArray);
-      console.log("saveStatusList----:", this.saveStatusList);
-      console.log("confirmDatasetList----:", this.confirmDatasetList);
-      console.log("==================================================");
+      //   this.confirmDatasetList = this.saveStatusList.join(";");
     },
     tableHeaderStyle({ row, column, rowIndex, columnIndex }) {
       if (rowIndex === 0) {
@@ -729,7 +707,6 @@ export default {
       this.getPublicDataset();
     },
     favoriteChangePage(val) {
-      console.log(val);
       this.paramsFavorite.page = val;
       this.getMyFavoriteDataset();
     },
@@ -744,19 +721,14 @@ export default {
         })
         .then((res) => {
           this.loadingCurrent = false;
-          console.log(res);
           let data = JSON.parse(res.data.data);
-          console.log(data);
           this.currentDatasetList = this.transformeTreeData(
             data,
             "currentTree",
             this.paramsCurrent.page
           );
           this.initCurrentTreeNode = [this.currentDatasetList[0].id];
-          console.log("this.initCurrentTreeNode", this.initCurrentTreeNode);
           this.totalNumCurrent = parseInt(res.data.count);
-          console.log(this.selectDatasetArray);
-          console.log("this.currentDatasetList:", this.currentDatasetList);
           let setCheckedKeysList = this.currentDatasetList.reduce(
             (pre, cur) => {
               cur.Attachments.forEach((item) => {
@@ -768,7 +740,6 @@ export default {
             },
             []
           );
-          console.log("setCheckedKeysList", setCheckedKeysList);
           this.$refs.currentTree.setCheckedKeys(setCheckedKeysList);
         })
         .catch(function (error) {
@@ -786,9 +757,7 @@ export default {
         })
         .then((res) => {
           this.loadingMy = false;
-          console.log(res);
           let data = JSON.parse(res.data.data);
-          console.log(data);
           this.myDatasetList = this.transformeTreeData(
             data,
             "myTree",
@@ -796,7 +765,6 @@ export default {
           );
           this.initMyTreeNode = [this.myDatasetList[0].id];
           this.totalNumMy = parseInt(res.data.count);
-          console.log("this.myDatasetList:", this.myDatasetList);
           let setCheckedKeysList = this.myDatasetList.reduce((pre, cur) => {
             cur.Attachments.forEach((item) => {
               if (this.saveStatusList.includes(item.id)) {
@@ -805,7 +773,6 @@ export default {
             });
             return pre;
           }, []);
-          console.log("setCheckedKeysList", setCheckedKeysList);
           this.$refs.myTree.setCheckedKeys(setCheckedKeysList);
         })
         .catch(function (error) {
@@ -823,9 +790,7 @@ export default {
         })
         .then((res) => {
           this.loadingPublic = false;
-          console.log(res);
           let data = JSON.parse(res.data.data);
-          console.log(data);
           this.publicDatasetList = this.transformeTreeData(
             data,
             "publicTree",
@@ -833,8 +798,6 @@ export default {
           );
           this.initPublicTreeNode = [this.publicDatasetList[0].id];
           this.totalNumPublic = parseInt(res.data.count);
-          console.log("this.publicDatasetList:", this.publicDatasetList);
-
           let setCheckedKeysList = this.publicDatasetList.reduce((pre, cur) => {
             cur.Attachments.forEach((item) => {
               if (this.saveStatusList.includes(item.id)) {
@@ -843,7 +806,6 @@ export default {
             });
             return pre;
           }, []);
-          console.log("setCheckedKeysList", setCheckedKeysList);
           this.$refs.publicTree.setCheckedKeys(setCheckedKeysList);
         })
         .catch(function (error) {
@@ -862,9 +824,7 @@ export default {
         })
         .then((res) => {
           this.loadingFavorite = false;
-          console.log(res);
           let data = JSON.parse(res.data.data);
-          console.log(data);
           this.MyFavoriteDatasetList = this.transformeTreeData(
             data,
             "favoriteTree",
@@ -872,11 +832,6 @@ export default {
           );
           this.initFavoriteTreeNode = [this.MyFavoriteDatasetList[0].id];
           this.totalNumFavorite = parseInt(res.data.count);
-          console.log(
-            "this.MyFavoriteDatasetList:",
-            this.MyFavoriteDatasetList
-          );
-
           let setCheckedKeysList = this.MyFavoriteDatasetList.reduce(
             (pre, cur) => {
               cur.Attachments.forEach((item) => {
@@ -888,7 +843,6 @@ export default {
             },
             []
           );
-          console.log("setCheckedKeysList", setCheckedKeysList);
           this.$refs.favoriteTree.setCheckedKeys(setCheckedKeysList);
         })
         .catch(function (error) {
@@ -941,8 +895,11 @@ export default {
         this.paramsFavorite.page = 1;
         this.getMyFavoriteDataset();
       }
+      return false;
     },
     confirmDataset() {
+      this.confirmDatasetList = this.saveStatusList.join(";");
+      this.confirmChecklist = this.checkList;
       this.dialogVisible = false;
       this.confirmFlag = true;
     },
@@ -992,7 +949,6 @@ export default {
   mounted() {
     this.type = $(".cloudbrain-type").data("cloudbrain-type");
     this.repoLink = $(".cloudbrain-type").data("repo-link");
-    console.log(this.type, this.repoLink);
     if (
       location.href.indexOf("benchmark") !== -1 ||
       location.href.indexOf("train-job") !== -1

web_src/less/openi.less

@@ -1204,3 +1204,15 @@ i.SUCCEEDED {
     display: flex;
     align-items: center;
 }
+.inline.min_title.field:not(.required) .label-fix-width:after {
+    margin: -0.2em 0 0 0.2em;
+    content: "*";
+    max-width: 6.38px;
+    visibility: hidden;
+}
+.inline.min_title.fields.required .label-fix-width:after {
+    margin: -0.2em 0 0 0.2em;
+    content: "*";
+    max-width: 6.38px;
+    visibility: hidden;
+}