| @@ -1895,44 +1895,48 @@ func SyncCloudbrainStatus() { | |||
| log.Error("task.JobType(%s) is error:%s", task.DisplayJobName, task.JobType) | |||
| } | |||
| } else if task.Type == models.TypeC2Net { | |||
| result, err := grampus.GetJob(task.JobID) | |||
| if err != nil { | |||
| log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err) | |||
| continue | |||
| } | |||
| if result != nil { | |||
| if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { | |||
| task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] | |||
| if task.JobType == string(models.JobTypeDebug) { | |||
| cloudbrainTask.SyncGrampusNotebookStatus(task) | |||
| } else { | |||
| result, err := grampus.GetJob(task.JobID) | |||
| if err != nil { | |||
| log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err) | |||
| continue | |||
| } | |||
| oldStatus := task.Status | |||
| task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) | |||
| task.Duration = result.JobInfo.RunSec | |||
| if task.Duration < 0 { | |||
| task.Duration = 0 | |||
| } | |||
| task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) | |||
| if result != nil { | |||
| if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { | |||
| task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] | |||
| } | |||
| oldStatus := task.Status | |||
| task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) | |||
| task.Duration = result.JobInfo.RunSec | |||
| if task.StartTime == 0 && result.JobInfo.StartedAt > 0 { | |||
| task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt) | |||
| } | |||
| if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { | |||
| task.EndTime = task.StartTime.Add(task.Duration) | |||
| } | |||
| task.CorrectCreateUnix() | |||
| if oldStatus != task.Status { | |||
| notification.NotifyChangeCloudbrainStatus(task, oldStatus) | |||
| if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource { | |||
| if len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
| urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID)) | |||
| if task.Duration < 0 { | |||
| task.Duration = 0 | |||
| } | |||
| task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) | |||
| if task.StartTime == 0 && result.JobInfo.StartedAt > 0 { | |||
| task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt) | |||
| } | |||
| if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { | |||
| task.EndTime = task.StartTime.Add(task.Duration) | |||
| } | |||
| task.CorrectCreateUnix() | |||
| if oldStatus != task.Status { | |||
| notification.NotifyChangeCloudbrainStatus(task, oldStatus) | |||
| if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource { | |||
| if len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
| urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID)) | |||
| } | |||
| } | |||
| } | |||
| } | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | |||
| continue | |||
| err = models.UpdateJob(task) | |||
| if err != nil { | |||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | |||
| continue | |||
| } | |||
| } | |||
| } | |||
| } else { | |||