@@ -63,62 +63,87 @@ func CloudBrainAiSafetyCreateTest(ctx *context.Context) {
func GetAiSafetyTask(ctx *context.Context) {
func GetAiSafetyTask(ctx *context.Context) {
var ID = ctx.Params(":jobid")
var ID = ctx.Params(":jobid")
getAiSafetyTaskStatusFromCloudbrain(ID)
}
func getAiSafetyTaskStatusFromCloudbrain(ID string) {
job, err := models.GetCloudbrainByJobIDWithDeleted(ID)
job, err := models.GetCloudbrainByJobIDWithDeleted(ID)
if err != nil {
if err != nil {
log.Error("GetCloudbrainByJobID failed:" + err.Error())
log.Error("GetCloudbrainByJobID failed:" + err.Error())
return
return
}
}
syncAiSafetyTaskStatus(job)
}
func syncAiSafetyTaskStatus(job *models.Cloudbrain) {
if job.Type == models.TypeCloudBrainTwo {
if job.Type == models.TypeCloudBrainTwo {
} else if job.Type == models.TypeCloudBrainOne {
} else if job.Type == models.TypeCloudBrainOne {
if isTaskNotFinished(job.Status) {
if isTaskNotFinished(job.Status) {
log.Info("The task not finished,name=" + job.DisplayJobName)
jobResult, err := cloudbrain.GetJob(job.JobID)
result, err := models.ConvertToJobResultPayload(jobResult.Payload)
if err != nil {
log.Error("ConvertToJobResultPayload failed:", err)
return
}
job.Status = result.JobStatus.State
if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
taskRoles := result.TaskRoles
taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
job.Status = taskRes.TaskStatuses[0].State
}
if result.JobStatus.State != string(models.JobSucceeded) {
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}
queryTaskStatusFromCloudbrain(job)
} else {
if job.Status == string(models.ModelSafetyTesting) {
queryTaskStatusFromModelSafetyTestServer(job)
} else {
} else {
//
job.Status = string(models.ModelSafetyTesting)
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}
//send msg to beihang
sendGpuInferenceResultToTest(job)
log.Info("The job is finished. status=" + job.Status)
}
}
}
}
}
func TimerHandleModelSafetyTestTask() {
tasks, err := models.GetModelSafetyTestTask()
if err == nil {
if tasks != nil && len(tasks) > 0 {
for _, job := range tasks {
syncAiSafetyTaskStatus(job)
}
} else {
} else {
if job.Status == string(models.ModelSafetyTesting) {
//
result, err := aisafety.GetTaskStatus(job.PreVersionName)
if err == nil {
if result["code"] != nil {
log.Info("query running model safety test task 0.")
}
} else {
log.Info("query running model safety test task err." + err.Error())
}
}
}
}
}
func queryTaskStatusFromCloudbrain(job *models.Cloudbrain) {
log.Info("The task not finished,name=" + job.DisplayJobName)
jobResult, err := cloudbrain.GetJob(job.JobID)
result, err := models.ConvertToJobResultPayload(jobResult.Payload)
if err != nil {
log.Error("ConvertToJobResultPayload failed:", err)
return
}
job.Status = result.JobStatus.State
if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
taskRoles := result.TaskRoles
taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
job.Status = taskRes.TaskStatuses[0].State
}
if result.JobStatus.State != string(models.JobSucceeded) {
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}
} else {
//
job.Status = string(models.ModelSafetyTesting)
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}
}
//send msg to beihang
sendGpuInferenceResultToTest(job)
}
}
func queryTaskStatusFromModelSafetyTestServer(job *models.Cloudbrain) {
result, err := aisafety.GetTaskStatus(job.PreVersionName)
if err == nil {
if result["code"] != nil {
if result["code"].(string) == "0" {
}
}
}
}
}
}
@@ -272,6 +297,15 @@ func AiSafetyCreateForPost(ctx *context.Context) {
return
return
}
}
}
}
BootFile := ctx.Query("BootFile")
bootFileExist, err := ctx.Repo.FileExists(BootFile, cloudbrain.DefaultBranchName)
if err != nil || !bootFileExist {
log.Error("Get bootfile error:", err)
modelSafetyNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsInferenceJobNew, nil)
return
}
if taskType == models.TypeCloudBrainTwo {
if taskType == models.TypeCloudBrainTwo {
createForNPU(ctx, jobName)
createForNPU(ctx, jobName)
} else if taskType == models.TypeCloudBrainOne {
} else if taskType == models.TypeCloudBrainOne {
@@ -280,35 +314,6 @@ func AiSafetyCreateForPost(ctx *context.Context) {
//ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
//ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
}
}
func TimerHandleModelSafetyTestTask() {
tasks, err := models.GetRunningModelSafetyTestTask()
if err == nil {
if tasks != nil && len(tasks) > 0 {
for _, task := range tasks {
if task.Type == models.TypeCloudBrainOne {
if task.Status == string(models.JobRunning) {
//调用云脑任务状态接口
} else if task.Status == string(models.ModelSafetyTesting) {
//调用模型评测查询状态接口
}
} else if task.Type == models.TypeCloudBrainTwo {
if task.Status == string(models.ModelArtsRunning) {
} else if task.Status == string(models.ModelArtsTrainJobCompleted) {
}
}
}
} else {
log.Info("query running model safety test task 0.")
}
} else {
log.Info("query running model safety test task err." + err.Error())
}
}
func createForNPU(ctx *context.Context, jobName string) {
func createForNPU(ctx *context.Context, jobName string) {
}
}
@@ -349,19 +354,6 @@ func createForGPU(ctx *context.Context, jobName string) {
return
return
}
}
if _, err := os.Stat(codePath + "/" + BootFile); err != nil {
if os.IsNotExist(err) {
// file does not exist
log.Error(BootFile+" does not exist, %v", err, ctx.Data["MsgID"])
modelSafetyNewDataPrepare(ctx)
ctx.RenderWithErr(BootFile+" does not exist", tplCloudBrainModelSafetyNew, nil)
} else {
log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
modelSafetyNewDataPrepare(ctx)
ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNew, nil)
}
return
}
err = uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
err = uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
if err != nil {
if err != nil {
log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])