You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

cloudbrain.go 21 kB

4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732
  1. package repo
  2. import (
  3. "bufio"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "net/http"
  9. "os"
  10. "os/exec"
  11. "regexp"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "time"
  16. "code.gitea.io/gitea/modules/modelarts"
  17. "code.gitea.io/gitea/modules/git"
  18. "code.gitea.io/gitea/modules/storage"
  19. "code.gitea.io/gitea/models"
  20. "code.gitea.io/gitea/modules/auth"
  21. "code.gitea.io/gitea/modules/base"
  22. "code.gitea.io/gitea/modules/cloudbrain"
  23. "code.gitea.io/gitea/modules/context"
  24. "code.gitea.io/gitea/modules/log"
  25. "code.gitea.io/gitea/modules/setting"
  26. )
  27. const (
  28. tplCloudBrainIndex base.TplName = "repo/cloudbrain/index"
  29. tplCloudBrainNew base.TplName = "repo/cloudbrain/new"
  30. tplCloudBrainShow base.TplName = "repo/cloudbrain/show"
  31. tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"
  32. )
  33. var (
  34. gpuInfos *models.GpuInfos
  35. categories *models.Categories
  36. )
  37. var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
  38. // MustEnableDataset check if repository enable internal cb
  39. func MustEnableCloudbrain(ctx *context.Context) {
  40. if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
  41. ctx.NotFound("MustEnableCloudbrain", nil)
  42. return
  43. }
  44. }
  45. func CloudBrainIndex(ctx *context.Context) {
  46. MustEnableCloudbrain(ctx)
  47. repo := ctx.Repo.Repository
  48. page := ctx.QueryInt("page")
  49. if page <= 0 {
  50. page = 1
  51. }
  52. ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
  53. ListOptions: models.ListOptions{
  54. Page: page,
  55. PageSize: setting.UI.IssuePagingNum,
  56. },
  57. RepoID: repo.ID,
  58. Type: models.TypeCloudBrainOne,
  59. })
  60. if err != nil {
  61. ctx.ServerError("Cloudbrain", err)
  62. return
  63. }
  64. timestamp := time.Now().Unix()
  65. for i, task := range ciTasks {
  66. if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) {
  67. ciTasks[i].CanDebug = true
  68. } else {
  69. ciTasks[i].CanDebug = false
  70. }
  71. ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task)
  72. }
  73. pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
  74. pager.SetDefaultParams(ctx)
  75. ctx.Data["Page"] = pager
  76. ctx.Data["PageIsCloudBrain"] = true
  77. ctx.Data["Tasks"] = ciTasks
  78. ctx.HTML(200, tplCloudBrainIndex)
  79. }
  80. func cutString(str string, lens int) string {
  81. if len(str) < lens {
  82. return str
  83. }
  84. return str[:lens]
  85. }
  86. func jobNamePrefixValid(s string) string {
  87. lowStr := strings.ToLower(s)
  88. re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
  89. removeSpecial := re.ReplaceAllString(lowStr, "")
  90. re = regexp.MustCompile(`^[_\\-]+`)
  91. return re.ReplaceAllString(removeSpecial, "")
  92. }
  93. func cloudBrainNewDataPrepare(ctx *context.Context) error {
  94. ctx.Data["PageIsCloudBrain"] = true
  95. t := time.Now()
  96. var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
  97. ctx.Data["job_name"] = jobName
  98. result, err := cloudbrain.GetImages()
  99. if err != nil {
  100. ctx.Data["error"] = err.Error()
  101. log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"])
  102. }
  103. for i, payload := range result.Payload.ImageInfo {
  104. if strings.HasPrefix(result.Payload.ImageInfo[i].Place, "192.168") {
  105. result.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
  106. } else {
  107. result.Payload.ImageInfo[i].PlaceView = payload.Place
  108. }
  109. }
  110. ctx.Data["images"] = result.Payload.ImageInfo
  111. resultPublic, err := cloudbrain.GetPublicImages()
  112. if err != nil {
  113. ctx.Data["error"] = err.Error()
  114. log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"])
  115. }
  116. for i, payload := range resultPublic.Payload.ImageInfo {
  117. if strings.HasPrefix(resultPublic.Payload.ImageInfo[i].Place, "192.168") {
  118. resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
  119. } else {
  120. resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place
  121. }
  122. }
  123. ctx.Data["public_images"] = resultPublic.Payload.ImageInfo
  124. attachs, err := models.GetAllUserAttachments(ctx.User.ID)
  125. if err != nil {
  126. log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"])
  127. return err
  128. }
  129. ctx.Data["attachments"] = attachs
  130. ctx.Data["command"] = cloudbrain.Command
  131. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  132. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  133. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  134. ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
  135. ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
  136. if categories == nil {
  137. json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
  138. }
  139. ctx.Data["benchmark_categories"] = categories.Category
  140. if gpuInfos == nil {
  141. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  142. }
  143. ctx.Data["gpu_types"] = gpuInfos.GpuInfo
  144. if cloudbrain.ResourceSpecs == nil {
  145. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  146. }
  147. ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec
  148. ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
  149. ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
  150. ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
  151. ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled
  152. return nil
  153. }
  154. func CloudBrainNew(ctx *context.Context) {
  155. err := cloudBrainNewDataPrepare(ctx)
  156. if err != nil {
  157. ctx.ServerError("get new cloudbrain info failed", err)
  158. return
  159. }
  160. ctx.HTML(200, tplCloudBrainNew)
  161. }
  162. func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  163. ctx.Data["PageIsCloudBrain"] = true
  164. jobName := form.JobName
  165. image := form.Image
  166. command := form.Command
  167. uuid := form.Attachment
  168. jobType := form.JobType
  169. gpuQueue := form.GpuType
  170. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  171. resourceSpecId := form.ResourceSpecId
  172. if !jobNamePattern.MatchString(jobName) {
  173. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainNew, &form)
  174. return
  175. }
  176. if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
  177. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  178. cloudBrainNewDataPrepare(ctx)
  179. ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form)
  180. return
  181. }
  182. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID)
  183. if err != nil {
  184. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  185. cloudBrainNewDataPrepare(ctx)
  186. ctx.RenderWithErr("system error", tplCloudBrainNew, &form)
  187. return
  188. } else {
  189. if count >= 1 {
  190. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  191. cloudBrainNewDataPrepare(ctx)
  192. ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainNew, &form)
  193. return
  194. }
  195. }
  196. _, err = models.GetCloudbrainByName(jobName)
  197. if err == nil {
  198. log.Error("the job name did already exist", ctx.Data["MsgID"])
  199. cloudBrainNewDataPrepare(ctx)
  200. ctx.RenderWithErr("the job name did already exist", tplCloudBrainNew, &form)
  201. return
  202. } else {
  203. if !models.IsErrJobNotExist(err) {
  204. log.Error("system error, %v", err, ctx.Data["MsgID"])
  205. cloudBrainNewDataPrepare(ctx)
  206. ctx.RenderWithErr("system error", tplCloudBrainNew, &form)
  207. return
  208. }
  209. }
  210. repo := ctx.Repo.Repository
  211. downloadCode(repo, codePath)
  212. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
  213. err = os.MkdirAll(modelPath, os.ModePerm)
  214. if err != nil {
  215. cloudBrainNewDataPrepare(ctx)
  216. ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
  217. return
  218. }
  219. benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
  220. if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
  221. var gpuType string
  222. for _, gpuInfo := range gpuInfos.GpuInfo {
  223. if gpuInfo.Queue == gpuQueue {
  224. gpuType = gpuInfo.Value
  225. }
  226. }
  227. downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType)
  228. }
  229. snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
  230. if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
  231. downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "")
  232. }
  233. brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
  234. if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
  235. downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "")
  236. }
  237. err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId)
  238. if err != nil {
  239. cloudBrainNewDataPrepare(ctx)
  240. ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
  241. return
  242. }
  243. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  244. }
  245. func CloudBrainShow(ctx *context.Context) {
  246. ctx.Data["PageIsCloudBrain"] = true
  247. var jobID = ctx.Params(":jobid")
  248. task, err := models.GetCloudbrainByJobID(jobID)
  249. if err != nil {
  250. ctx.Data["error"] = err.Error()
  251. }
  252. result, err := cloudbrain.GetJob(jobID)
  253. if err != nil {
  254. ctx.Data["error"] = err.Error()
  255. }
  256. if result != nil {
  257. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  258. jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
  259. taskRoles := jobRes.TaskRoles
  260. if jobRes.JobStatus.State != string(models.JobFailed) {
  261. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  262. ctx.Data["taskRes"] = taskRes
  263. task.Status = taskRes.TaskStatuses[0].State
  264. task.ContainerID = taskRes.TaskStatuses[0].ContainerID
  265. task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  266. err = models.UpdateJob(task)
  267. if err != nil {
  268. ctx.Data["error"] = err.Error()
  269. }
  270. } else {
  271. task.Status = jobRes.JobStatus.State
  272. taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{
  273. {
  274. State: jobRes.JobStatus.State,
  275. },
  276. }}
  277. ctx.Data["taskRes"] = taskRes
  278. jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05")
  279. jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05")
  280. }
  281. ctx.Data["result"] = jobRes
  282. }
  283. ctx.Data["task"] = task
  284. ctx.Data["jobID"] = jobID
  285. ctx.HTML(200, tplCloudBrainShow)
  286. }
  287. func CloudBrainDebug(ctx *context.Context) {
  288. var jobID = ctx.Params(":jobid")
  289. if !ctx.IsSigned {
  290. log.Error("the user has not signed in")
  291. ctx.Error(http.StatusForbidden, "","the user has not signed in")
  292. return
  293. }
  294. task, err := models.GetCloudbrainByJobID(jobID)
  295. if err != nil {
  296. ctx.ServerError("GetCloudbrainByJobID failed", err)
  297. return
  298. }
  299. debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
  300. ctx.Redirect(debugUrl)
  301. }
  302. func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  303. var jobID = ctx.Params(":jobid")
  304. if !ctx.IsSigned {
  305. log.Error("the user has not signed in")
  306. ctx.Error(http.StatusForbidden, "","the user has not signed in")
  307. return
  308. }
  309. task, err := models.GetCloudbrainByJobID(jobID)
  310. if err != nil {
  311. ctx.JSON(200, map[string]string{
  312. "result_code": "-1",
  313. "error_msg": "GetCloudbrainByJobID failed",
  314. })
  315. return
  316. }
  317. err = cloudbrain.CommitImage(jobID, models.CommitImageParams{
  318. Ip: task.ContainerIp,
  319. TaskContainerId: task.ContainerID,
  320. ImageDescription: form.Description,
  321. ImageTag: form.Tag,
  322. })
  323. if err != nil {
  324. log.Error("CommitImage(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"])
  325. ctx.JSON(200, map[string]string{
  326. "result_code": "-1",
  327. "error_msg": "CommitImage failed",
  328. })
  329. return
  330. }
  331. ctx.JSON(200, map[string]string{
  332. "result_code": "0",
  333. "error_msg": "",
  334. })
  335. }
  336. func CloudBrainStop(ctx *context.Context) {
  337. var jobID = ctx.Params(":jobid")
  338. task, err := models.GetCloudbrainByJobID(jobID)
  339. if err != nil {
  340. ctx.ServerError("GetCloudbrainByJobID failed", err)
  341. return
  342. }
  343. if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) {
  344. log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
  345. ctx.ServerError("the job has been stopped", errors.New("the job has been stopped"))
  346. return
  347. }
  348. err = cloudbrain.StopJob(jobID)
  349. if err != nil {
  350. log.Error("StopJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"])
  351. ctx.ServerError("StopJob failed", err)
  352. return
  353. }
  354. task.Status = string(models.JobStopped)
  355. err = models.UpdateJob(task)
  356. if err != nil {
  357. ctx.ServerError("UpdateJob failed", err)
  358. return
  359. }
  360. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  361. }
  362. func StopJobsByUserID(userID int64) {
  363. cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
  364. if err != nil {
  365. log.Warn("Failed to get cloudBrain info", err)
  366. return
  367. }
  368. StopJobs(cloudBrains)
  369. }
  370. func StopJobsByRepoID(repoID int64) {
  371. cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
  372. if err != nil {
  373. log.Warn("Failed to get cloudBrain info", err)
  374. return
  375. }
  376. StopJobs(cloudBrains)
  377. }
  378. /**
  379. */
  380. func StopJobs(cloudBrains []*models.Cloudbrain) {
  381. for _, taskInfo := range cloudBrains {
  382. if taskInfo.Type == models.TypeCloudBrainOne {
  383. err := retry(3, time.Second*30, func() error {
  384. return cloudbrain.StopJob(taskInfo.JobID)
  385. })
  386. logErrorAndUpdateJobStatus(err, taskInfo)
  387. } else {
  388. param := models.NotebookAction{
  389. Action: models.ActionStop,
  390. }
  391. err := retry(3, time.Second*30, func() error {
  392. _, err := modelarts.StopJob(taskInfo.JobID, param)
  393. return err
  394. })
  395. logErrorAndUpdateJobStatus(err, taskInfo)
  396. }
  397. }
  398. }
  399. func retry(attempts int, sleep time.Duration, f func() error) (err error) {
  400. for i := 0; i < attempts; i++ {
  401. if i > 0 {
  402. log.Warn("retrying after error:", err)
  403. time.Sleep(sleep)
  404. }
  405. err = f()
  406. if err == nil {
  407. return nil
  408. }
  409. }
  410. return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
  411. }
  412. func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
  413. if err != nil {
  414. log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
  415. } else {
  416. taskInfo.Status = string(models.JobStopped)
  417. err = models.UpdateJob(taskInfo)
  418. if err != nil {
  419. log.Warn("UpdateJob failed", err)
  420. }
  421. }
  422. }
  423. func CloudBrainDel(ctx *context.Context) {
  424. var jobID = ctx.Params(":jobid")
  425. task, err := models.GetCloudbrainByJobID(jobID)
  426. if err != nil {
  427. ctx.ServerError("GetCloudbrainByJobID failed", err)
  428. return
  429. }
  430. if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) {
  431. log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
  432. ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
  433. return
  434. }
  435. err = models.DeleteJob(task)
  436. if err != nil {
  437. ctx.ServerError("DeleteJob failed", err)
  438. return
  439. }
  440. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  441. }
  442. func CloudBrainShowModels(ctx *context.Context) {
  443. ctx.Data["PageIsCloudBrain"] = true
  444. jobID := ctx.Params(":jobid")
  445. parentDir := ctx.Query("parentDir")
  446. dirArray := strings.Split(parentDir, "/")
  447. task, err := models.GetCloudbrainByJobID(jobID)
  448. if err != nil {
  449. log.Error("no such job!", ctx.Data["msgID"])
  450. ctx.ServerError("no such job:", err)
  451. return
  452. }
  453. //get dirs
  454. dirs, err := getModelDirs(task.JobName, parentDir)
  455. if err != nil {
  456. log.Error("getModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  457. ctx.ServerError("getModelDirs failed:", err)
  458. return
  459. }
  460. var fileInfos []storage.FileInfo
  461. err = json.Unmarshal([]byte(dirs), &fileInfos)
  462. if err != nil {
  463. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  464. ctx.ServerError("json.Unmarshal failed:", err)
  465. return
  466. }
  467. for i, fileInfo := range fileInfos {
  468. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  469. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  470. }
  471. sort.Slice(fileInfos, func(i, j int) bool {
  472. return fileInfos[i].ModTime > fileInfos[j].ModTime
  473. })
  474. ctx.Data["Path"] = dirArray
  475. ctx.Data["Dirs"] = fileInfos
  476. ctx.Data["task"] = task
  477. ctx.Data["JobID"] = jobID
  478. ctx.HTML(200, tplCloudBrainShowModels)
  479. }
  480. func GetPublicImages(ctx *context.Context) {
  481. getImages(ctx, cloudbrain.Public)
  482. }
  483. func GetCustomImages(ctx *context.Context) {
  484. getImages(ctx, cloudbrain.Custom)
  485. }
  486. func getImages(ctx *context.Context, imageType string) {
  487. log.Info("Get images begin")
  488. page := ctx.QueryInt("page")
  489. size := ctx.QueryInt("size")
  490. name := ctx.Query("name")
  491. getImagesResult, err := cloudbrain.GetImagesPageable(page, size, imageType, name)
  492. if err != nil {
  493. log.Error("Can not get images:%v", err)
  494. ctx.JSON(http.StatusOK, models.GetImagesPayload{
  495. Count: 0,
  496. TotalPages: 0,
  497. ImageInfo: []*models.ImageInfo{},
  498. })
  499. } else {
  500. ctx.JSON(http.StatusOK, getImagesResult.Payload)
  501. }
  502. log.Info("Get images end")
  503. }
  504. func getModelDirs(jobName string, parentDir string) (string, error) {
  505. var req string
  506. modelActualPath := setting.JobPath + jobName + "/model/"
  507. if parentDir == "" {
  508. req = "baseDir=" + modelActualPath
  509. } else {
  510. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  511. }
  512. return getDirs(req)
  513. }
  514. func CloudBrainDownloadModel(ctx *context.Context) {
  515. parentDir := ctx.Query("parentDir")
  516. fileName := ctx.Query("fileName")
  517. jobName := ctx.Query("jobName")
  518. filePath := "jobs/" + jobName + "/model/" + parentDir
  519. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  520. if err != nil {
  521. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  522. ctx.ServerError("PresignedGetURL", err)
  523. return
  524. }
  525. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  526. }
  527. func GetRate(ctx *context.Context) {
  528. var jobID = ctx.Params(":jobid")
  529. job, err := models.GetCloudbrainByJobID(jobID)
  530. if err != nil {
  531. ctx.ServerError("GetCloudbrainByJobID failed", err)
  532. return
  533. }
  534. if job.JobType == string(models.JobTypeBenchmark) {
  535. ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  536. } else if job.JobType == string(models.JobTypeSnn4imagenet) {
  537. ctx.Redirect(setting.Snn4imagenetServerHost)
  538. } else if job.JobType == string(models.JobTypeBrainScore) {
  539. ctx.Redirect(setting.BrainScoreServerHost)
  540. } else {
  541. log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
  542. }
  543. }
  544. func downloadCode(repo *models.Repository, codePath string) error {
  545. if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
  546. log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
  547. return err
  548. }
  549. configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
  550. if err != nil {
  551. log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
  552. return err
  553. }
  554. defer configFile.Close()
  555. pos := int64(0)
  556. reader := bufio.NewReader(configFile)
  557. for {
  558. line, err := reader.ReadString('\n')
  559. if err != nil {
  560. if err == io.EOF {
  561. log.Error("not find the remote-url")
  562. return nil
  563. } else {
  564. log.Error("read error: %v", err)
  565. return err
  566. }
  567. }
  568. if strings.Contains(line, "url") && strings.Contains(line, ".git") {
  569. originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
  570. if len(line) > len(originUrl) {
  571. originUrl += strings.Repeat(" ", len(line)-len(originUrl))
  572. }
  573. bytes := []byte(originUrl)
  574. _, err := configFile.WriteAt(bytes, pos)
  575. if err != nil {
  576. log.Error("WriteAt failed:%v", err)
  577. return err
  578. }
  579. break
  580. }
  581. pos += int64(len(line))
  582. }
  583. return nil
  584. }
  585. func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error {
  586. err := os.MkdirAll(codePath, os.ModePerm)
  587. if err != nil {
  588. log.Error("mkdir codePath failed", err.Error())
  589. return err
  590. }
  591. command := "git clone " + gitPath + " " + codePath
  592. cmd := exec.Command("/bin/bash", "-c", command)
  593. output, err := cmd.Output()
  594. log.Info(string(output))
  595. if err != nil {
  596. log.Error("exec.Command(%s) failed:%v", command, err)
  597. return err
  598. }
  599. fileName := codePath + cloudbrain.TaskInfoName
  600. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  601. if err != nil {
  602. log.Error("OpenFile failed", err.Error())
  603. return err
  604. }
  605. defer f.Close()
  606. data, err := json.Marshal(models.TaskInfo{
  607. Username: repo.Owner.Name,
  608. TaskName: taskName,
  609. CodeName: repo.Name,
  610. BenchmarkCategory: strings.Split(benchmarkCategory, ","),
  611. CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
  612. GpuType: gpuType,
  613. })
  614. if err != nil {
  615. log.Error("json.Marshal failed", err.Error())
  616. return err
  617. }
  618. _, err = f.Write(data)
  619. if err != nil {
  620. log.Error("WriteString failed", err.Error())
  621. return err
  622. }
  623. return nil
  624. }