#1654

fix bug

models/cloudbrain.go

@@ -169,6 +169,14 @@ func IsTrainJobTerminal(status string) bool {
 	return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled)
 }
 
+func IsModelArtsDebugJobTerminal(status string) bool {
+	return status == string(ModelArtsStopped)
+}
+
+func IsCloudBrainOneDebugJobTerminal(status string) bool {
+	return status == string(JobStopped) || status == string(JobFailed)
+}
+
 type CloudbrainInfo struct {
 	Cloudbrain `xorm:"extends"`
 	User       `xorm:"extends"`

routers/api/v1/repo/cloudbrain.go

@@ -85,6 +85,10 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 	}
 
 	if result.JobStatus.State != string(models.JobWaiting) {
+		if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) {
+			job.EndTime = timeutil.TimeStampNow()
+			job.Duration = job.ComputeDuration()
+		}
 		err = models.UpdateJob(job)
 		if err != nil {
 			log.Error("UpdateJob failed:", err)

routers/api/v1/repo/modelarts.go

@@ -72,7 +72,7 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {
 		job.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 	}
 	job.Status = result.Status
-	if job.EndTime == 0 && job.Status == string(models.ModelArtsStopped) {
+	if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) {
 		job.EndTime = timeutil.TimeStampNow()
 		job.Duration = job.ComputeDuration()
 	}
@@ -154,7 +154,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		job.TrainJobDuration = "00:00:00"
 	}
 
-	if models.IsTrainJobTerminal(job.Status) {
+	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
 	}
 
@@ -392,7 +392,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 	} else {
 		job.TrainJobDuration = "00:00:00"
 	}
-	if models.IsTrainJobTerminal(job.Status) {
+	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
 	}
 

routers/repo/cloudbrain.go

@@ -493,8 +493,10 @@ func CloudBrainStop(ctx *context.Context) {
 		}
 
 		task.Status = string(models.JobStopped)
-		task.EndTime = timeutil.TimeStampNow()
-		task.Duration = task.ComputeDuration()
+		if task.EndTime == 0 {
+			task.EndTime = timeutil.TimeStampNow()
+			task.Duration = task.ComputeDuration()
+		}
 		err = models.UpdateJob(task)
 		if err != nil {
 			log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
@@ -588,8 +590,10 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
 		log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
 	} else {
 		taskInfo.Status = string(models.JobStopped)
-		taskInfo.EndTime = timeutil.TimeStampNow()
-		taskInfo.Duration = taskInfo.ComputeDuration()
+		if taskInfo.EndTime == 0 {
+			taskInfo.EndTime = timeutil.TimeStampNow()
+			taskInfo.Duration = taskInfo.ComputeDuration()
+		}
 		err = models.UpdateJob(taskInfo)
 		if err != nil {
 			log.Warn("UpdateJob failed", err)
@@ -964,10 +968,10 @@ func SyncCloudbrainStatus() {
 					if task.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() {
 						task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
 					}
-					//if task.Status == string(models.JobFailed) {
-					//	task.EndTime = timeutil.TimeStampNow()
-					//	task.Duration = task.ComputeDuration()
-					//}
+					if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) {
+						task.EndTime = timeutil.TimeStampNow()
+						task.Duration = task.ComputeDuration()
+					}
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -988,8 +992,10 @@ func SyncCloudbrainStatus() {
 							continue
 						}
 						task.Status = string(models.JobStopped)
-						task.EndTime = timeutil.TimeStampNow()
-						task.Duration = task.ComputeDuration()
+						if task.EndTime == 0 {
+							task.EndTime = timeutil.TimeStampNow()
+							task.Duration = task.ComputeDuration()
+						}
 						err = models.UpdateJob(task)
 						if err != nil {
 							log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -1011,7 +1017,7 @@ func SyncCloudbrainStatus() {
 					if task.StartTime == 0 && result.Lease.CreateTime > 0 {
 						task.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 					}
-					if task.EndTime == 0 && task.Status == string(models.ModelArtsStopped) {
+					if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
 						task.EndTime = timeutil.TimeStampNow()
 						task.Duration = task.ComputeDuration()
 					}
@@ -1042,7 +1048,7 @@ func SyncCloudbrainStatus() {
 					} else {
 						task.TrainJobDuration = "00:00:00"
 					}
-					if models.IsTrainJobTerminal(task.Status) {
+					if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
 						task.EndTime = task.StartTime.Add(task.Duration)
 					}