aiforge/routers/repo/cloudbrain.go

package repo

import (
	"code.gitea.io/gitea/modules/git"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/cloudbrain"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
)

const (
	tplCloudBrainIndex base.TplName = "repo/cloudbrain/index"
	tplCloudBrainNew   base.TplName = "repo/cloudbrain/new"
	tplCloudBrainShow  base.TplName = "repo/cloudbrain/show"
	tplCloudBrainShowModels  base.TplName = "repo/cloudbrain/show_models"
)

var (
	gpuInfos   *models.GpuInfos
	categories *models.Categories
)

// MustEnableDataset check if repository enable internal cb
func MustEnableCloudbrain(ctx *context.Context) {
	if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
		ctx.NotFound("MustEnableCloudbrain", nil)
		return
	}
}
func CloudBrainIndex(ctx *context.Context) {
	MustEnableCloudbrain(ctx)
	repo := ctx.Repo.Repository
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}

	ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: setting.UI.IssuePagingNum,
		},
		RepoID: repo.ID,
		Type:   models.TypeCloudBrainOne,
	})
	if err != nil {
		ctx.ServerError("Cloudbrain", err)
		return
	}

	timestamp := time.Now().Unix()
	for i, task := range ciTasks {
		if task.Status == string(models.JobRunning) && (timestamp-int64(task.CreatedUnix) > 30) {
			ciTasks[i].CanDebug = true
		} else {
			ciTasks[i].CanDebug = false
		}
	}

	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
	pager.SetDefaultParams(ctx)
	ctx.Data["Page"] = pager

	ctx.Data["PageIsCloudBrain"] = true
	ctx.Data["Tasks"] = ciTasks
	ctx.HTML(200, tplCloudBrainIndex)
}

func cutString(str string, lens int) string {
	if len(str) < lens {
		return str
	}
	return str[:lens]
}

func CloudBrainNew(ctx *context.Context) {
	ctx.Data["PageIsCloudBrain"] = true

	t := time.Now()
	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
	ctx.Data["job_name"] = jobName

	result, err := cloudbrain.GetImages()
	if err != nil {
		ctx.Data["error"] = err.Error()
		log.Error("cloudbrain.GetImages failed:", err.Error())
	}

	for i, payload := range result.Payload.ImageInfo {
		if strings.HasPrefix(result.Payload.ImageInfo[i].Place, "192.168") {
			result.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
		} else {
			result.Payload.ImageInfo[i].PlaceView = payload.Place
		}
	}

	ctx.Data["images"] = result.Payload.ImageInfo

	resultPublic, err := cloudbrain.GetPublicImages()
	if err != nil {
		ctx.Data["error"] = err.Error()
		log.Error("cloudbrain.GetPublicImages failed:", err.Error())
	}

	for i, payload := range resultPublic.Payload.ImageInfo {
		if strings.HasPrefix(resultPublic.Payload.ImageInfo[i].Place, "192.168") {
			resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
		} else {
			resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place
		}
	}

	ctx.Data["public_images"] = resultPublic.Payload.ImageInfo

	attachs, err := models.GetAllUserAttachments(ctx.User.ID)
	if err != nil {
		ctx.ServerError("GetAllUserAttachments failed:", err)
		return
	}

	ctx.Data["attachments"] = attachs
	ctx.Data["command"] = cloudbrain.Command
	ctx.Data["code_path"] = cloudbrain.CodeMountPath
	ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
	ctx.Data["model_path"] = cloudbrain.ModelMountPath
	ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
	ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled

	if categories == nil {
		json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
	}
	ctx.Data["benchmark_categories"] = categories.Category

	if gpuInfos == nil {
		json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
	}
	ctx.Data["gpu_types"] = gpuInfos.GpuInfo
	ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
	ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
	ctx.HTML(200, tplCloudBrainNew)
}

func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
	ctx.Data["PageIsCloudBrain"] = true
	jobName := form.JobName
	image := form.Image
	command := form.Command
	uuid := form.Attachment
	jobType := form.JobType
	gpuQueue := setting.JobType
	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath

	if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) {
		log.Error("jobtype error:", jobType)
		ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form)
		return
	}
	repo := ctx.Repo.Repository
	downloadCode(repo, codePath)

	modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
	err := os.MkdirAll(modelPath, os.ModePerm)
	if err != nil {
		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
		return
	}

	benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
	if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
		gpuQueue = form.GpuType
		var gpuType string
		for _, gpuInfo := range gpuInfos.GpuInfo {
			if gpuInfo.Queue == gpuQueue {
				gpuType = gpuInfo.Value
			}
		}
		downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType)
	}

	snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
	if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
		downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "")
	}

	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue)
	if err != nil {
		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
		return
	}
	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
}

func CloudBrainShow(ctx *context.Context) {
	ctx.Data["PageIsCloudBrain"] = true

	var jobID = ctx.Params(":jobid")
	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.Data["error"] = err.Error()
	}

	result, err := cloudbrain.GetJob(jobID)
	if err != nil {
		ctx.Data["error"] = err.Error()
	}

	if result != nil {
		jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
		ctx.Data["result"] = jobRes
		taskRoles := jobRes.TaskRoles
		taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
		ctx.Data["taskRes"] = taskRes
		task.Status = taskRes.TaskStatuses[0].State
		task.ContainerID = taskRes.TaskStatuses[0].ContainerID
		task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
		err = models.UpdateJob(task)
		if err != nil {
			ctx.Data["error"] = err.Error()
		}
	}

	ctx.Data["task"] = task
	ctx.Data["jobID"] = jobID
	ctx.HTML(200, tplCloudBrainShow)
}

func CloudBrainDebug(ctx *context.Context) {
	var jobID = ctx.Params(":jobid")
	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.ServerError("GetCloudbrainByJobID failed", err)
		return
	}

	debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
	ctx.Redirect(debugUrl)
}

func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
	var jobID = ctx.Params(":jobid")
	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   "GetCloudbrainByJobID failed",
		})
		return
	}

	err = cloudbrain.CommitImage(jobID, models.CommitImageParams{
		Ip:               task.ContainerIp,
		TaskContainerId:  task.ContainerID,
		ImageDescription: form.Description,
		ImageTag:         form.Tag,
	})
	if err != nil {
		log.Error("CommitImage(%s) failed:", task.JobName, err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   "CommitImage failed",
		})
		return
	}

	ctx.JSON(200, map[string]string{
		"result_code": "0",
		"error_msg":   "",
	})
}

func CloudBrainStop(ctx *context.Context) {
	var jobID = ctx.Params(":jobid")
	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.ServerError("GetCloudbrainByJobID failed", err)
		return
	}

	if task.Status == string(models.JobStopped) {
		log.Error("the job(%s) has been stopped", task.JobName)
		ctx.ServerError("the job has been stopped", errors.New("the job has been stopped"))
		return
	}

	err = cloudbrain.StopJob(jobID)
	if err != nil {
		log.Error("StopJob(%s) failed:%v", task.JobName, err.Error())
		ctx.ServerError("StopJob failed", err)
		return
	}

	task.Status = string(models.JobStopped)
	err = models.UpdateJob(task)
	if err != nil {
		ctx.ServerError("UpdateJob failed", err)
		return
	}

	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
}

func CloudBrainDel(ctx *context.Context) {
	var jobID = ctx.Params(":jobid")
	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.ServerError("GetCloudbrainByJobID failed", err)
		return
	}

	if task.Status != string(models.JobStopped) {
		log.Error("the job(%s) has not been stopped", task.JobName)
		ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
		return
	}

	err = models.DeleteJob(task)
	if err != nil {
		ctx.ServerError("DeleteJob failed", err)
		return
	}

	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
}

func CloudBrainGetModels(ctx *context.Context) {
	ctx.Data["PageIsCloudBrain"] = true

	var jobID = ctx.Params(":jobid")
	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		log.Error("no such job!")
		ctx.RenderWithErr("no such job!", tplCloudBrainIndex, nil)
		return
	}



	ctx.Data["task"] = task
	ctx.Data["jobID"] = jobID
	ctx.HTML(200, tplCloudBrainShowModels)
}

func GetRate(ctx *context.Context) {
	var jobID = ctx.Params(":jobid")
	job, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.ServerError("GetCloudbrainByJobID failed", err)
		return
	}

	if job.JobType == string(models.JobTypeBenchmark) {
		ctx.Redirect(setting.BenchmarkServerHost)
	} else if job.JobType == string(models.JobTypeSnn4imagenet) {
		ctx.Redirect(setting.Snn4imagenetServerHost)
	} else {
		log.Error("JobType error:", job.JobType)
	}
}

func downloadCode(repo *models.Repository, codePath string) error {
	if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
		log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
		return err
	}

	return nil
}

func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error {
	err := os.MkdirAll(codePath, os.ModePerm)
	if err != nil {
		log.Error("mkdir codePath failed", err.Error())
		return err
	}

	command := "git clone " + gitPath + " " + codePath
	cmd := exec.Command("/bin/bash", "-c", command)
	output, err := cmd.Output()
	log.Info(string(output))
	if err != nil {
		log.Error("exec.Command(%s) failed:%v", command, err)
		return err
	}

	fileName := codePath + cloudbrain.TaskInfoName
	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
	if err != nil {
		log.Error("OpenFile failed", err.Error())
		return err
	}

	defer f.Close()

	data, err := json.Marshal(models.TaskInfo{
		Username:          repo.Owner.Name,
		TaskName:          taskName,
		CodeName:          repo.Name,
		BenchmarkCategory: strings.Split(benchmarkCategory, ","),
		CodeLink:          strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
		GpuType:           gpuType,
	})
	if err != nil {
		log.Error("json.Marshal failed", err.Error())
		return err
	}

	_, err = f.Write(data)
	if err != nil {
		log.Error("WriteString failed", err.Error())
		return err
	}

	return nil
}