Merge remote-tracking branch 'origin/V20220830' into zouap

go.sum

@@ -713,12 +713,14 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
 github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=

modules/cloudbrain/resty.go

@@ -24,10 +24,11 @@ var (
 )
 
 const (
-	JobHasBeenStopped = "S410"
-	Public            = "public"
-	Custom            = "custom"
-	LogPageSize		  = 500
+	JobHasBeenStopped   = "S410"
+	errInvalidToken     = "S401"
+	Public              = "public"
+	Custom              = "custom"
+	LogPageSize         = 500
 	LogPageTokenExpired = "5m"
 	pageSize            = 15
 	QueuesDetailUrl     = "/rest-server/api/v2/queuesdetail"
@@ -59,7 +60,7 @@ func loginCloudbrain() error {
 
 	res, err := client.R().
 		SetHeader("Content-Type", "application/json").
-		SetBody(map[string]interface{}{"username": username, "password": password, "expiration": "604800"}).
+		SetBody(map[string]interface{}{"username": username, "password": password, "expiration": conf.Expiration}).
 		SetResult(&loginResult).
 		Post(HOST + "/rest-server/api/v1/token")
 	if err != nil {
@@ -122,16 +123,12 @@ sendjob:
 		Post(HOST + "/rest-server/api/v1/jobs/")
 
 	if err != nil {
-		if res != nil {
-			var response models.CloudBrainResult
-			json.Unmarshal(res.Body(), &response)
-			log.Error("code(%s), msg(%s)", response.Code, response.Msg)
-			return nil, fmt.Errorf(response.Msg)
-		}
 		return nil, fmt.Errorf("resty create job: %s", err)
 	}
 
-	if jobResult.Code == "S401" && retry < 1 {
+	var response models.CloudBrainResult
+	json.Unmarshal(res.Body(), &response)
+	if response.Code == errInvalidToken && retry < 1 {
 		retry++
 		_ = loginCloudbrain()
 		goto sendjob
@@ -163,7 +160,9 @@ sendjob:
 		return nil, fmt.Errorf("resty GetJob: %v", err)
 	}
 
-	if getJobResult.Code == "S401" && retry < 1 {
+	var response models.CloudBrainResult
+	json.Unmarshal(res.Body(), &response)
+	if response.Code == errInvalidToken && retry < 1 {
 		retry++
 		_ = loginCloudbrain()
 		goto sendjob
@@ -196,13 +195,8 @@ sendjob:
 	}
 
 	var response models.CloudBrainResult
-	err = json.Unmarshal(res.Body(), &response)
-	if err != nil {
-		log.Error("json.Unmarshal failed: %s", err.Error())
-		return &getImagesResult, fmt.Errorf("json.Unmarshal failed: %s", err.Error())
-	}
-
-	if response.Code == "S401" && retry < 1 {
+	json.Unmarshal(res.Body(), &response)
+	if response.Code == errInvalidToken && retry < 1 {
 		retry++
 		_ = loginCloudbrain()
 		goto sendjob
@@ -290,7 +284,9 @@ sendjob:
 		return fmt.Errorf("resty CommitImage: %v", err)
 	}
 
-	if result.Code == "S401" && retry < 1 {
+	var response models.CloudBrainResult
+	json.Unmarshal(res.Body(), &response)
+	if response.Code == errInvalidToken && retry < 1 {
 		retry++
 		_ = loginCloudbrain()
 		goto sendjob
@@ -442,7 +438,9 @@ sendjob:
 		return fmt.Errorf("resty StopJob: %v", err)
 	}
 
-	if result.Code == "S401" && retry < 1 {
+	var response models.CloudBrainResult
+	json.Unmarshal(res.Body(), &response)
+	if response.Code == errInvalidToken && retry < 1 {
 		retry++
 		_ = loginCloudbrain()
 		goto sendjob
@@ -498,7 +496,7 @@ func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) {
 	client := getRestyClient()
 	var result models.GetJobLogResult
 	req := models.GetAllJobLogParams{
-		Scroll: LogPageTokenExpired,
+		Scroll:   LogPageTokenExpired,
 		ScrollID: scrollID,
 	}
 
@@ -522,7 +520,7 @@ func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) {
 	return &result, nil
 }
 
-func DeleteJobLogToken(scrollID string) (error) {
+func DeleteJobLogToken(scrollID string) error {
 	checkSetting()
 	client := getRestyClient()
 	var result models.DeleteJobLogTokenResult

modules/setting/cloudbrain.go

@@ -5,6 +5,7 @@ type CloudbrainLoginConfig struct {
 	Password       string
 	Host           string
 	ImageURLPrefix string
+	Expiration     string
 }
 
 var (
@@ -17,5 +18,6 @@ func GetCloudbrainConfig() CloudbrainLoginConfig {
 	Cloudbrain.Password = cloudbrainSec.Key("PASSWORD").MustString("")
 	Cloudbrain.Host = cloudbrainSec.Key("REST_SERVER_HOST").MustString("")
 	Cloudbrain.ImageURLPrefix = cloudbrainSec.Key("IMAGE_URL_PREFIX").MustString("")
+	Cloudbrain.Expiration = cloudbrainSec.Key("EXPIRATION").MustString("604800")
 	return Cloudbrain
 }

options/locale/locale_en-US.ini

@@ -3123,7 +3123,8 @@ wrong_specification=You cannot use this specification, please choose another ite
 
 
 job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-).
-dataset_path_rule = The dataset location is stored in the environment variable <strong style="color:#010101">data_url</strong>, and the output path is stored in the environment variable <strong style="color:#010101">train_url</strong>.
+train_dataset_path_rule = The dataset location is stored in the environment variable <strong style="color:#010101">data_url</strong>, and the output path is stored in the environment variable <strong style="color:#010101">train_url</strong>.
+infer_dataset_path_rule = The dataset location is stored in the environment variable <strong style="color:#010101">data_url</strong>, and the output path is stored in the environment variable <strong style="color:#010101">train_url</strong>.
 view_sample = View sample
 inference_output_path_rule = The inference output path is stored in the environment variable result_url.
 model_file_path_rule=The model file location is stored in the environment variable ckpt_url

options/locale/locale_zh-CN.ini

@@ -3137,7 +3137,8 @@ card_type = 卡类型
 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。
 
 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。
-dataset_path_rule = 数据集位置存储在环境变量<strong style="color:#010101">data_url</strong>中，输出路径存储在环境变量<strong style="color:#010101">train_url</strong>中。
+train_dataset_path_rule = 数据集位置存储在环境变量<strong style="color:#010101">data_url</strong>中，训练输出路径存储在环境变量<strong style="color:#010101">train_url</strong>中。
+infer_dataset_path_rule = 数据集位置存储在环境变量<strong style="color:#010101">data_url</strong>中，推理输出路径存储在环境变量<strong style="color:#010101">train_url</strong>中。
 view_sample = 查看样例
 inference_output_path_rule = 推理输出路径存储在环境变量result_url中。
 model_file_path_rule = 模型文件位置存储在环境变量ckpt_url中。

routers/api/v1/repo/modelarts.go

@@ -6,13 +6,14 @@
 package repo
 
 import (
-	"code.gitea.io/gitea/modules/notification"
 	"encoding/json"
 	"net/http"
 	"path"
 	"strconv"
 	"strings"
 
+	"code.gitea.io/gitea/modules/notification"
+
 	"code.gitea.io/gitea/modules/grampus"
 	"code.gitea.io/gitea/modules/setting"
 
@@ -352,6 +353,14 @@ func DelTrainJobVersion(ctx *context.APIContext) {
 		return
 	}
 
+	if task.Status != string(models.ModelArtsTrainJobImageFailed) && task.Status != string(models.ModelArtsTrainJobSubmitFailed) && task.Status != string(models.ModelArtsTrainJobDeleteFailed) &&
+		task.Status != string(models.ModelArtsTrainJobCompleted) && task.Status != string(models.ModelArtsTrainJobFailed) &&
+		task.Status != string(models.ModelArtsTrainJobKilled) && task.Status != string(models.ModelArtsTrainJobCanceled) && task.Status != string(models.ModelArtsTrainJobLost) {
+		log.Error("the job(%s) version has not been stopped", task.JobName)
+		ctx.NotFound(err)
+		return
+	}
+
 	//删除modelarts上的记录
 	_, err = modelarts.DelTrainJobVersion(jobID, strconv.FormatInt(task.VersionID, 10))
 	if err != nil {

routers/repo/modelarts.go

@@ -1845,6 +1845,16 @@ func TrainJobDel(ctx *context.Context) {
 		return
 	}
 
+	for _, task := range VersionListTasks {
+		if task.Status != string(models.ModelArtsTrainJobImageFailed) && task.Status != string(models.ModelArtsTrainJobSubmitFailed) && task.Status != string(models.ModelArtsTrainJobDeleteFailed) &&
+			task.Status != string(models.ModelArtsTrainJobCompleted) && task.Status != string(models.ModelArtsTrainJobFailed) &&
+			task.Status != string(models.ModelArtsTrainJobKilled) && task.Status != string(models.ModelArtsTrainJobCanceled) && task.Status != string(models.ModelArtsTrainJobLost) {
+			log.Error("the job(%s) version has not been stopped", task.JobName)
+			ctx.RenderWithErr("the job version has not been stopped", tplModelArtsTrainJobIndex, nil)
+			return
+		}
+	}
+
 	//删除modelarts上的任务记录
 	_, err = modelarts.DelTrainJob(jobID)
 	if err != nil {

templates/custom/wait_count.tmpl

@@ -13,9 +13,9 @@
 	{{ end }}
 	<i
 		class="ri-error-warning-line"
-		style="margin-right: 0.5rem; font-size: 16px"
+		style="margin-right: 0.5rem; font-size: 14px"
 	></i>
-	<span id="gpu-nums"
+	<span id="gpu-nums" style="font-size: 12px"
 		>{{.i18n.Tr "repo.wait_count_start"}}
 		{{if .QueuesDetail}}
 		{{ $gpuQueue }}

templates/custom/wait_count_train.tmpl

@@ -14,9 +14,9 @@
 	{{ end }}
 	<i
 		class="ri-error-warning-line"
-		style="margin-right: 0.5rem; font-size: 16px"
+		style="margin-right: 0.5rem; font-size: 14px"
 	></i>
-	<span id="gpu-nums"
+	<span id="gpu-nums" style="font-size: 12px"
 		>{{.ctx.i18n.Tr "repo.wait_count_start"}}
 		{{if .type}}
 		{{ $gpuQueue }}

templates/repo/modelarts/inferencejob/new.tmpl

@@ -86,7 +86,7 @@
                     {{template "custom/wait_count_train" Dict "ctx" $}}
                     <div style="display: flex;align-items: center;margin-left: 155px;margin-top: 0.5rem;">
                         <i class="ri-error-warning-line" style="color: #f2711c;margin-right: 0.5rem;"></i>
-                        <span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.dataset_path_rule" | Safe}}</span>
+                        <span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.infer_dataset_path_rule" | Safe}}</span>
                     </div>
                 </div>
                 <div class="required min_title inline field">

templates/repo/modelarts/trainjob/new.tmpl

@@ -113,7 +113,7 @@
                     {{template "custom/wait_count_train" Dict "ctx" $}}
                     <div style="display: flex;align-items: center;margin-left: 155px;margin-top: 0.5rem;">
                         <i class="ri-error-warning-line" style="color: #f2711c;margin-right: 0.5rem;"></i>
-                        <span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.dataset_path_rule" | Safe}}</span>
+                        <span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.train_dataset_path_rule" | Safe}}</span>
                     </div>
                 </div>
                 <div class="required inline min_title field">

templates/repo/modelarts/trainjob/show.tmpl

@@ -274,10 +274,14 @@
 
 
                                     {{if .CanDel}}
-                                    <a class="ti-action-menu-item" onclick="deleteVersion({{.VersionName}})"
+                                    <a class='ti-action-menu-item delete-show-version {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING" "RUNNING" "KILLING" "INIT"}}disabled {{end}}' 
+                                        id="{{.VersionName}}-delete"
+                                        data-jobid="{{.JobID}}"
+                                        data-repopath="{{$.RepoRelPath}}/modelarts/train-job"
+                                        data-version = "{{.VersionName}}"
                                         style="color: #FF4D4F;">{{$.i18n.Tr "repo.delete"}}</a>
                                     {{else}}
-                                    <a class="ti-action-menu-item disabled" onclick="deleteVersion({{.VersionName}})"
+                                    <a class="ti-action-menu-item disabled"
                                         style="color: #FF4D4F;">{{$.i18n.Tr "repo.delete"}}</a>
                                     {{end}}
                                 </div>
@@ -899,35 +903,7 @@
         $('#name').val(modelName)
         $('#version').val("0.0.1")
     }
-    function deleteVersion(version_name) {
-        stopBubbling(arguments.callee.caller.arguments[0])
-        let flag = 1;
-        $('.ui.basic.modal').modal({
-            onDeny: function () {
-                flag = false
-            },
-            onApprove: function () {
-                $.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/del_version`, { version_name: version_name }, (data) => {
-                    if (data.VersionListCount === 0) {
-                        location.href = `/${userName}/${repoPath}/modelarts/train-job`
-                    } else {
-                        $('#accordion' + version_name).remove()
-                    }
-
-                }).fail(function (err) {
-                    console.log(err);
-                });
-                flag = true
-            },
-            onHidden: function () {
-                if (flag == false) {
-                    $('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
-                }
-            }
-        })
-            .modal('show')
-
-    }
+    
 
    
 

templates/repo/modelarts/trainjob/version_new.tmpl

@@ -73,6 +73,11 @@
                 <input type="hidden" id="ai_engine_name" name="engine_names" value="">
                 <input type="hidden" id="ai_flaver_name" name="flaver_names" value="">
                 <input type="hidden" id="display_job_name" name="display_job_name" value="{{.display_job_name}}">
+                {{template "custom/wait_count_train" Dict "ctx" $}}
+                <div style="display: flex;align-items: center;margin-left: 155px;margin-top: 0.5rem;">
+                    <i class="ri-error-warning-line" style="color: #f2711c;margin-right: 0.5rem;"></i>
+                    <span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.train_dataset_path_rule" | Safe}}</span>
+                </div>
                 <h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
                 <div class="required unite min_title inline field">
                     <label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
@@ -152,26 +157,9 @@
                         </span>
                         <a  href="https://git.openi.org.cn/OpenIOSSG/MINIST_Example" target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
                 </div>
-                
-                <!-- <div class="required unite min_title inline field">
-                    <label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.dataset"}}</label>
-                    <select class="ui dropdown width80" id="trainjob_datasets" name="attachment" placeholder="{{.i18n.Tr "cloudbrain.select_dataset"}}">
-                        {{if .dataset_name}}
-                            <option name="attachment" value="{{.uuid}}">{{.dataset_name}}</option>
-                        {{end}}
-                        {{range .attachments}}
-                            <option value="">{{$.i18n.Tr "cloudbrain.select_dataset"}}</option>
-                            {{if ne $.uuid .UUID}}
-                            <option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
-                            {{end}}
-                        {{end}}
-                    </select>
-                    <span class="tooltips">{{.i18n.Tr "cloudbrain.dataset_path_rule"}}</span>
-                </div> -->
                 <div id="select-multi-dataset">
 
                 </div>
-                <span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 1rem;">{{.i18n.Tr "cloudbrain.dataset_path_rule"}}</span>
                 <div class="inline unite min_title field">
                     <label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
                     <span id="add_run_para" style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>

web_src/js/features/cloudbrainShow.js

@@ -213,6 +213,35 @@ export default async function initCloudrainSow() {
     });
     e.stopPropagation();
   });
+  $(".delete-show-version").click(function (e) {
+    const ID = this.dataset.jobid;
+    const repoPath = this.dataset.repopath;
+    const version_name = this.dataset.version;
+    const url = `/api/v1/repos/${repoPath}/${ID}/del_version`;
+    $(".ui.basic.modal")
+      .modal({
+        onApprove: function () {
+          $.post(url, { version_name: version_name }, (data) => {
+            console.log(data);
+            if (data.StatusOK === 0) {
+              if (data.VersionListCount === 0) {
+                location.href = `/${repoPath}`;
+              } else {
+                $("#accordion" + version_name).remove();
+              }
+              refreshStatusShow(version_name, ID, repoPath);
+            } else {
+              return;
+            }
+          }).fail(function (err) {
+            console.log(err);
+          });
+        },
+      })
+      .modal("show");
+
+    e.stopPropagation();
+  });
   function refreshStatusShow(version_name, ID, repoPath) {
     $.get(
       `/api/v1/repos/${repoPath}/${ID}?version_name=${version_name}`,

web_src/js/features/cloudrbanin.js

@@ -182,6 +182,15 @@ export default async function initCloudrain() {
         "CREATE_FAILED",
         "STOPPED",
       ];
+      let deleteArray = [
+        "KILLED",
+        "FAILED",
+        "START_FAILED",
+        "COMPLETED",
+        "SUCCEEDED",
+        "CREATE_FAILED",
+        "STOPPED",
+      ];
       $.get(
         `/api/v1/repos/${repoPath}/${jobID}?version_name=${versionname}`,
         (data) => {
@@ -194,6 +203,10 @@ export default async function initCloudrain() {
           if (stopArray.includes(data.JobStatus)) {
             $("#" + versionname + "-stop").addClass("disabled");
           }
+          if (deleteArray.includes(data.JobStatus)) {
+            $(`#${versionname}-delete`).removeClass("disabled");
+            $(`#${versionname}-delete`).addClass("blue");
+          }
           if (data.JobStatus === "COMPLETED") {
             $("#" + versionname + "-create-model")
               .removeClass("disabled")

web_src/js/index.js

@@ -3520,7 +3520,7 @@ function initVueComponents() {
               `${self.reposFilter}:${self.archivedFilter}:${self.privateFilter}`,
               count
             );
-            self.finalPage = Math.floor(count / self.searchLimit) + 1;
+            self.finalPage = Math.ceil(count / self.searchLimit);
             self.updateHistory();
           }
         }).always(() => {