| @@ -1,6 +1,7 @@ | |||||
| package models | package models | ||||
| import ( | import ( | ||||
| "code.gitea.io/gitea/modules/util" | |||||
| "encoding/json" | "encoding/json" | ||||
| "fmt" | "fmt" | ||||
| "strconv" | "strconv" | ||||
| @@ -102,15 +103,15 @@ type Cloudbrain struct { | |||||
| ContainerIp string | ContainerIp string | ||||
| CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` | CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` | ||||
| UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | ||||
| Duration int64 //运行时长 单位秒 | |||||
| TrainJobDuration string | |||||
| Image string //镜像名称 | |||||
| GpuQueue string //GPU类型即GPU队列 | |||||
| ResourceSpecId int //GPU规格id | |||||
| DeletedAt time.Time `xorm:"deleted"` | |||||
| CanDebug bool `xorm:"-"` | |||||
| CanDel bool `xorm:"-"` | |||||
| CanModify bool `xorm:"-"` | |||||
| Duration int64 `xorm:"DEFAULT 0"` //运行时长 单位秒 | |||||
| TrainJobDuration string `xorm:"DEFAULT '00:00:00'"` | |||||
| Image string //镜像名称 | |||||
| GpuQueue string //GPU类型即GPU队列 | |||||
| ResourceSpecId int //GPU规格id | |||||
| DeletedAt time.Time `xorm:"deleted"` | |||||
| CanDebug bool `xorm:"-"` | |||||
| CanDel bool `xorm:"-"` | |||||
| CanModify bool `xorm:"-"` | |||||
| Type int | Type int | ||||
| BenchmarkTypeID int | BenchmarkTypeID int | ||||
| BenchmarkChildTypeID int | BenchmarkChildTypeID int | ||||
| @@ -154,15 +155,28 @@ type Cloudbrain struct { | |||||
| EndTime timeutil.TimeStamp | EndTime timeutil.TimeStamp | ||||
| } | } | ||||
| func (task *Cloudbrain) ComputeDuration() int64 { | |||||
| if task.StartTime == 0 || task.EndTime == 0 { | |||||
| return 0 | |||||
| func (task *Cloudbrain) ComputeAndSetDuration() { | |||||
| var d int64 | |||||
| if task.StartTime == 0 { | |||||
| d = 0 | |||||
| } else if task.EndTime == 0 { | |||||
| d = time.Now().Unix() - task.StartTime.AsTime().Unix() | |||||
| } else { | |||||
| d = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() | |||||
| } | } | ||||
| d := task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() | |||||
| if d < 0 { | if d < 0 { | ||||
| d = 0 | d = 0 | ||||
| } | } | ||||
| return d | |||||
| task.Duration = d | |||||
| task.TrainJobDuration = ConvertDurationToStr(d) | |||||
| } | |||||
| func ConvertDurationToStr(duration int64) string { | |||||
| if duration == 0 { | |||||
| return "00:00:00" | |||||
| } | |||||
| return util.AddZero(duration/3600) + ":" + util.AddZero(duration%3600/60) + ":" + util.AddZero(duration%60) | |||||
| } | } | ||||
| func IsTrainJobTerminal(status string) bool { | func IsTrainJobTerminal(status string) bool { | ||||
| @@ -174,7 +188,7 @@ func IsModelArtsDebugJobTerminal(status string) bool { | |||||
| } | } | ||||
| func IsCloudBrainOneDebugJobTerminal(status string) bool { | func IsCloudBrainOneDebugJobTerminal(status string) bool { | ||||
| return status == string(JobStopped) || status == string(JobFailed) | |||||
| return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) | |||||
| } | } | ||||
| type CloudbrainInfo struct { | type CloudbrainInfo struct { | ||||
| @@ -87,8 +87,8 @@ func GetCloudbrainTask(ctx *context.APIContext) { | |||||
| if result.JobStatus.State != string(models.JobWaiting) { | if result.JobStatus.State != string(models.JobWaiting) { | ||||
| if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) { | if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) { | ||||
| job.EndTime = timeutil.TimeStampNow() | job.EndTime = timeutil.TimeStampNow() | ||||
| job.Duration = job.ComputeDuration() | |||||
| } | } | ||||
| job.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(job) | err = models.UpdateJob(job) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob failed:", err) | log.Error("UpdateJob failed:", err) | ||||
| @@ -11,8 +11,6 @@ import ( | |||||
| "strconv" | "strconv" | ||||
| "strings" | "strings" | ||||
| "code.gitea.io/gitea/modules/util" | |||||
| "code.gitea.io/gitea/models" | "code.gitea.io/gitea/models" | ||||
| "code.gitea.io/gitea/modules/context" | "code.gitea.io/gitea/modules/context" | ||||
| "code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
| @@ -74,8 +72,8 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { | |||||
| job.Status = result.Status | job.Status = result.Status | ||||
| if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) { | if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) { | ||||
| job.EndTime = timeutil.TimeStampNow() | job.EndTime = timeutil.TimeStampNow() | ||||
| job.Duration = job.ComputeDuration() | |||||
| } | } | ||||
| job.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(job) | err = models.UpdateJob(job) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob failed:", err) | log.Error("UpdateJob failed:", err) | ||||
| @@ -147,12 +145,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { | |||||
| job.Duration = result.Duration / 1000 | job.Duration = result.Duration / 1000 | ||||
| job.TrainJobDuration = result.TrainJobDuration | job.TrainJobDuration = result.TrainJobDuration | ||||
| if result.Duration != 0 { | |||||
| job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60) | |||||
| } else { | |||||
| job.TrainJobDuration = "00:00:00" | |||||
| } | |||||
| job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) | |||||
| if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { | if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { | ||||
| job.EndTime = job.StartTime.Add(job.Duration) | job.EndTime = job.StartTime.Add(job.Duration) | ||||
| @@ -386,12 +379,8 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) { | |||||
| job.Duration = result.Duration / 1000 | job.Duration = result.Duration / 1000 | ||||
| job.TrainJobDuration = result.TrainJobDuration | job.TrainJobDuration = result.TrainJobDuration | ||||
| if result.Duration != 0 { | |||||
| job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60) | |||||
| job.TrainJobDuration = models.ConvertDurationToStr(result.Duration) | |||||
| } else { | |||||
| job.TrainJobDuration = "00:00:00" | |||||
| } | |||||
| if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { | if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { | ||||
| job.EndTime = job.StartTime.Add(job.Duration) | job.EndTime = job.StartTime.Add(job.Duration) | ||||
| } | } | ||||
| @@ -488,8 +488,8 @@ func CloudBrainStop(ctx *context.Context) { | |||||
| task.Status = string(models.JobStopped) | task.Status = string(models.JobStopped) | ||||
| if task.EndTime == 0 { | if task.EndTime == 0 { | ||||
| task.EndTime = timeutil.TimeStampNow() | task.EndTime = timeutil.TimeStampNow() | ||||
| task.Duration = task.ComputeDuration() | |||||
| } | } | ||||
| task.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(task) | err = models.UpdateJob(task) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) | ||||
| @@ -585,8 +585,8 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { | |||||
| taskInfo.Status = string(models.JobStopped) | taskInfo.Status = string(models.JobStopped) | ||||
| if taskInfo.EndTime == 0 { | if taskInfo.EndTime == 0 { | ||||
| taskInfo.EndTime = timeutil.TimeStampNow() | taskInfo.EndTime = timeutil.TimeStampNow() | ||||
| taskInfo.Duration = taskInfo.ComputeDuration() | |||||
| } | } | ||||
| taskInfo.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(taskInfo) | err = models.UpdateJob(taskInfo) | ||||
| if err != nil { | if err != nil { | ||||
| log.Warn("UpdateJob failed", err) | log.Warn("UpdateJob failed", err) | ||||
| @@ -963,8 +963,8 @@ func SyncCloudbrainStatus() { | |||||
| } | } | ||||
| if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) { | if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) { | ||||
| task.EndTime = timeutil.TimeStampNow() | task.EndTime = timeutil.TimeStampNow() | ||||
| task.Duration = task.ComputeDuration() | |||||
| } | } | ||||
| task.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(task) | err = models.UpdateJob(task) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| @@ -987,8 +987,8 @@ func SyncCloudbrainStatus() { | |||||
| task.Status = string(models.JobStopped) | task.Status = string(models.JobStopped) | ||||
| if task.EndTime == 0 { | if task.EndTime == 0 { | ||||
| task.EndTime = timeutil.TimeStampNow() | task.EndTime = timeutil.TimeStampNow() | ||||
| task.Duration = task.ComputeDuration() | |||||
| } | } | ||||
| task.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(task) | err = models.UpdateJob(task) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| @@ -1012,8 +1012,8 @@ func SyncCloudbrainStatus() { | |||||
| } | } | ||||
| if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { | if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { | ||||
| task.EndTime = timeutil.TimeStampNow() | task.EndTime = timeutil.TimeStampNow() | ||||
| task.Duration = task.ComputeDuration() | |||||
| } | } | ||||
| task.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(task) | err = models.UpdateJob(task) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| @@ -1035,12 +1035,7 @@ func SyncCloudbrainStatus() { | |||||
| if task.StartTime == 0 && result.StartTime > 0 { | if task.StartTime == 0 && result.StartTime > 0 { | ||||
| task.StartTime = timeutil.TimeStamp(result.StartTime / 1000) | task.StartTime = timeutil.TimeStamp(result.StartTime / 1000) | ||||
| } | } | ||||
| if result.Duration != 0 { | |||||
| task.TrainJobDuration = util.AddZero(task.Duration/3600) + ":" + util.AddZero(task.Duration%3600/60) + ":" + util.AddZero(task.Duration%60) | |||||
| } else { | |||||
| task.TrainJobDuration = "00:00:00" | |||||
| } | |||||
| task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) | |||||
| if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { | if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { | ||||
| task.EndTime = task.StartTime.Add(task.Duration) | task.EndTime = task.StartTime.Add(task.Duration) | ||||
| } | } | ||||
| @@ -1090,13 +1085,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { | |||||
| for i, task := range ciTasks { | for i, task := range ciTasks { | ||||
| ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) | ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) | ||||
| ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource | ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource | ||||
| var duration int64 | |||||
| if task.Status == string(models.JobRunning) { | |||||
| duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) | |||||
| } else { | |||||
| duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) | |||||
| if ciTasks[i].TrainJobDuration == "" { | |||||
| var duration int64 | |||||
| if task.Status == string(models.JobRunning) { | |||||
| duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) | |||||
| } else { | |||||
| duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) | |||||
| } | |||||
| ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(duration) | |||||
| } | } | ||||
| ciTasks[i].TrainJobDuration = util.AddZero(duration/3600000) + ":" + util.AddZero(duration%3600000/60000) + ":" + util.AddZero(duration%60000/1000) | |||||
| ciTasks[i].BenchmarkTypeName = "" | ciTasks[i].BenchmarkTypeName = "" | ||||
| if task.BenchmarkTypeID > 0 { | if task.BenchmarkTypeID > 0 { | ||||
| for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { | for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { | ||||
| @@ -411,8 +411,8 @@ func NotebookManage(ctx *context.Context) { | |||||
| task.Status = res.Status | task.Status = res.Status | ||||
| if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { | if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { | ||||
| task.EndTime = timeutil.TimeStampNow() | task.EndTime = timeutil.TimeStampNow() | ||||
| task.Duration = task.ComputeDuration() | |||||
| } | } | ||||
| task.ComputeAndSetDuration() | |||||
| err = models.UpdateJob(task) | err = models.UpdateJob(task) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) | ||||