| @@ -1,6 +1,7 @@ | |||
| package models | |||
| import ( | |||
| "code.gitea.io/gitea/modules/util" | |||
| "encoding/json" | |||
| "fmt" | |||
| "strconv" | |||
| @@ -102,15 +103,15 @@ type Cloudbrain struct { | |||
| ContainerIp string | |||
| CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` | |||
| UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | |||
| Duration int64 //运行时长 单位秒 | |||
| TrainJobDuration string | |||
| Image string //镜像名称 | |||
| GpuQueue string //GPU类型即GPU队列 | |||
| ResourceSpecId int //GPU规格id | |||
| DeletedAt time.Time `xorm:"deleted"` | |||
| CanDebug bool `xorm:"-"` | |||
| CanDel bool `xorm:"-"` | |||
| CanModify bool `xorm:"-"` | |||
| Duration int64 `xorm:"DEFAULT 0"` //运行时长 单位秒 | |||
| TrainJobDuration string `xorm:"DEFAULT '00:00:00'"` | |||
| Image string //镜像名称 | |||
| GpuQueue string //GPU类型即GPU队列 | |||
| ResourceSpecId int //GPU规格id | |||
| DeletedAt time.Time `xorm:"deleted"` | |||
| CanDebug bool `xorm:"-"` | |||
| CanDel bool `xorm:"-"` | |||
| CanModify bool `xorm:"-"` | |||
| Type int | |||
| BenchmarkTypeID int | |||
| BenchmarkChildTypeID int | |||
| @@ -154,15 +155,28 @@ type Cloudbrain struct { | |||
| EndTime timeutil.TimeStamp | |||
| } | |||
| func (task *Cloudbrain) ComputeDuration() int64 { | |||
| if task.StartTime == 0 || task.EndTime == 0 { | |||
| return 0 | |||
| func (task *Cloudbrain) ComputeAndSetDuration() { | |||
| var d int64 | |||
| if task.StartTime == 0 { | |||
| d = 0 | |||
| } else if task.EndTime == 0 { | |||
| d = time.Now().Unix() - task.StartTime.AsTime().Unix() | |||
| } else { | |||
| d = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() | |||
| } | |||
| d := task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() | |||
| if d < 0 { | |||
| d = 0 | |||
| } | |||
| return d | |||
| task.Duration = d | |||
| task.TrainJobDuration = ConvertDurationToStr(d) | |||
| } | |||
| func ConvertDurationToStr(duration int64) string { | |||
| if duration == 0 { | |||
| return "00:00:00" | |||
| } | |||
| return util.AddZero(duration/3600) + ":" + util.AddZero(duration%3600/60) + ":" + util.AddZero(duration%60) | |||
| } | |||
| func IsTrainJobTerminal(status string) bool { | |||
| @@ -174,7 +188,7 @@ func IsModelArtsDebugJobTerminal(status string) bool { | |||
| } | |||
| func IsCloudBrainOneDebugJobTerminal(status string) bool { | |||
| return status == string(JobStopped) || status == string(JobFailed) | |||
| return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) | |||
| } | |||
| type CloudbrainInfo struct { | |||
| @@ -87,8 +87,8 @@ func GetCloudbrainTask(ctx *context.APIContext) { | |||
| if result.JobStatus.State != string(models.JobWaiting) { | |||
| if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) { | |||
| job.EndTime = timeutil.TimeStampNow() | |||
| job.Duration = job.ComputeDuration() | |||
| } | |||
| job.ComputeAndSetDuration() | |||
| err = models.UpdateJob(job) | |||
| if err != nil { | |||
| log.Error("UpdateJob failed:", err) | |||
| @@ -11,8 +11,6 @@ import ( | |||
| "strconv" | |||
| "strings" | |||
| "code.gitea.io/gitea/modules/util" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/context" | |||
| "code.gitea.io/gitea/modules/log" | |||
| @@ -74,8 +72,8 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { | |||
| job.Status = result.Status | |||
| if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) { | |||
| job.EndTime = timeutil.TimeStampNow() | |||
| job.Duration = job.ComputeDuration() | |||
| } | |||
| job.ComputeAndSetDuration() | |||
| err = models.UpdateJob(job) | |||
| if err != nil { | |||
| log.Error("UpdateJob failed:", err) | |||
| @@ -147,12 +145,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { | |||
| job.Duration = result.Duration / 1000 | |||
| job.TrainJobDuration = result.TrainJobDuration | |||
| if result.Duration != 0 { | |||
| job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60) | |||
| } else { | |||
| job.TrainJobDuration = "00:00:00" | |||
| } | |||
| job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) | |||
| if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { | |||
| job.EndTime = job.StartTime.Add(job.Duration) | |||
| @@ -386,12 +379,8 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) { | |||
| job.Duration = result.Duration / 1000 | |||
| job.TrainJobDuration = result.TrainJobDuration | |||
| if result.Duration != 0 { | |||
| job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60) | |||
| job.TrainJobDuration = models.ConvertDurationToStr(result.Duration) | |||
| } else { | |||
| job.TrainJobDuration = "00:00:00" | |||
| } | |||
| if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { | |||
| job.EndTime = job.StartTime.Add(job.Duration) | |||
| } | |||
| @@ -488,8 +488,8 @@ func CloudBrainStop(ctx *context.Context) { | |||
| task.Status = string(models.JobStopped) | |||
| if task.EndTime == 0 { | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| task.Duration = task.ComputeDuration() | |||
| } | |||
| task.ComputeAndSetDuration() | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) | |||
| @@ -585,8 +585,8 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { | |||
| taskInfo.Status = string(models.JobStopped) | |||
| if taskInfo.EndTime == 0 { | |||
| taskInfo.EndTime = timeutil.TimeStampNow() | |||
| taskInfo.Duration = taskInfo.ComputeDuration() | |||
| } | |||
| taskInfo.ComputeAndSetDuration() | |||
| err = models.UpdateJob(taskInfo) | |||
| if err != nil { | |||
| log.Warn("UpdateJob failed", err) | |||
| @@ -963,8 +963,8 @@ func SyncCloudbrainStatus() { | |||
| } | |||
| if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) { | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| task.Duration = task.ComputeDuration() | |||
| } | |||
| task.ComputeAndSetDuration() | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | |||
| @@ -987,8 +987,8 @@ func SyncCloudbrainStatus() { | |||
| task.Status = string(models.JobStopped) | |||
| if task.EndTime == 0 { | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| task.Duration = task.ComputeDuration() | |||
| } | |||
| task.ComputeAndSetDuration() | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | |||
| @@ -1012,8 +1012,8 @@ func SyncCloudbrainStatus() { | |||
| } | |||
| if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| task.Duration = task.ComputeDuration() | |||
| } | |||
| task.ComputeAndSetDuration() | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | |||
| @@ -1035,12 +1035,7 @@ func SyncCloudbrainStatus() { | |||
| if task.StartTime == 0 && result.StartTime > 0 { | |||
| task.StartTime = timeutil.TimeStamp(result.StartTime / 1000) | |||
| } | |||
| if result.Duration != 0 { | |||
| task.TrainJobDuration = util.AddZero(task.Duration/3600) + ":" + util.AddZero(task.Duration%3600/60) + ":" + util.AddZero(task.Duration%60) | |||
| } else { | |||
| task.TrainJobDuration = "00:00:00" | |||
| } | |||
| task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) | |||
| if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { | |||
| task.EndTime = task.StartTime.Add(task.Duration) | |||
| } | |||
| @@ -1090,13 +1085,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { | |||
| for i, task := range ciTasks { | |||
| ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) | |||
| ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource | |||
| var duration int64 | |||
| if task.Status == string(models.JobRunning) { | |||
| duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) | |||
| } else { | |||
| duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) | |||
| if ciTasks[i].TrainJobDuration == "" { | |||
| var duration int64 | |||
| if task.Status == string(models.JobRunning) { | |||
| duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) | |||
| } else { | |||
| duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) | |||
| } | |||
| ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(duration) | |||
| } | |||
| ciTasks[i].TrainJobDuration = util.AddZero(duration/3600000) + ":" + util.AddZero(duration%3600000/60000) + ":" + util.AddZero(duration%60000/1000) | |||
| ciTasks[i].BenchmarkTypeName = "" | |||
| if task.BenchmarkTypeID > 0 { | |||
| for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { | |||
| @@ -411,8 +411,8 @@ func NotebookManage(ctx *context.Context) { | |||
| task.Status = res.Status | |||
| if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| task.Duration = task.ComputeDuration() | |||
| } | |||
| task.ComputeAndSetDuration() | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) | |||