You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

cloudbrain.go 20 kB

4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717
  1. package repo
  2. import (
  3. "bufio"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "net/http"
  9. "os"
  10. "os/exec"
  11. "regexp"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "time"
  16. "code.gitea.io/gitea/modules/modelarts"
  17. "code.gitea.io/gitea/modules/git"
  18. "code.gitea.io/gitea/modules/storage"
  19. "code.gitea.io/gitea/models"
  20. "code.gitea.io/gitea/modules/auth"
  21. "code.gitea.io/gitea/modules/base"
  22. "code.gitea.io/gitea/modules/cloudbrain"
  23. "code.gitea.io/gitea/modules/context"
  24. "code.gitea.io/gitea/modules/log"
  25. "code.gitea.io/gitea/modules/setting"
  26. )
  27. const (
  28. tplCloudBrainIndex base.TplName = "repo/cloudbrain/index"
  29. tplCloudBrainNew base.TplName = "repo/cloudbrain/new"
  30. tplCloudBrainShow base.TplName = "repo/cloudbrain/show"
  31. tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"
  32. )
  33. var (
  34. gpuInfos *models.GpuInfos
  35. categories *models.Categories
  36. )
  37. var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
  38. // MustEnableDataset check if repository enable internal cb
  39. func MustEnableCloudbrain(ctx *context.Context) {
  40. if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
  41. ctx.NotFound("MustEnableCloudbrain", nil)
  42. return
  43. }
  44. }
  45. func CloudBrainIndex(ctx *context.Context) {
  46. MustEnableCloudbrain(ctx)
  47. repo := ctx.Repo.Repository
  48. page := ctx.QueryInt("page")
  49. if page <= 0 {
  50. page = 1
  51. }
  52. ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
  53. ListOptions: models.ListOptions{
  54. Page: page,
  55. PageSize: setting.UI.IssuePagingNum,
  56. },
  57. RepoID: repo.ID,
  58. Type: models.TypeCloudBrainOne,
  59. })
  60. if err != nil {
  61. ctx.ServerError("Cloudbrain", err)
  62. return
  63. }
  64. timestamp := time.Now().Unix()
  65. for i, task := range ciTasks {
  66. if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) {
  67. ciTasks[i].CanDebug = true
  68. } else {
  69. ciTasks[i].CanDebug = false
  70. }
  71. ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task)
  72. }
  73. pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
  74. pager.SetDefaultParams(ctx)
  75. ctx.Data["Page"] = pager
  76. ctx.Data["PageIsCloudBrain"] = true
  77. ctx.Data["Tasks"] = ciTasks
  78. ctx.HTML(200, tplCloudBrainIndex)
  79. }
  80. func cutString(str string, lens int) string {
  81. if len(str) < lens {
  82. return str
  83. }
  84. return str[:lens]
  85. }
  86. func jobNamePrefixValid(s string) string {
  87. lowStr := strings.ToLower(s)
  88. re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
  89. removeSpecial := re.ReplaceAllString(lowStr, "")
  90. re = regexp.MustCompile(`^[_\\-]+`)
  91. return re.ReplaceAllString(removeSpecial, "")
  92. }
  93. func cloudBrainNewDataPrepare(ctx *context.Context) error {
  94. ctx.Data["PageIsCloudBrain"] = true
  95. t := time.Now()
  96. var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
  97. ctx.Data["job_name"] = jobName
  98. result, err := cloudbrain.GetImages()
  99. if err != nil {
  100. ctx.Data["error"] = err.Error()
  101. log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"])
  102. }
  103. for i, payload := range result.Payload.ImageInfo {
  104. if strings.HasPrefix(result.Payload.ImageInfo[i].Place, "192.168") {
  105. result.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
  106. } else {
  107. result.Payload.ImageInfo[i].PlaceView = payload.Place
  108. }
  109. }
  110. ctx.Data["images"] = result.Payload.ImageInfo
  111. resultPublic, err := cloudbrain.GetPublicImages()
  112. if err != nil {
  113. ctx.Data["error"] = err.Error()
  114. log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"])
  115. }
  116. for i, payload := range resultPublic.Payload.ImageInfo {
  117. if strings.HasPrefix(resultPublic.Payload.ImageInfo[i].Place, "192.168") {
  118. resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
  119. } else {
  120. resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place
  121. }
  122. }
  123. ctx.Data["public_images"] = resultPublic.Payload.ImageInfo
  124. attachs, err := models.GetAllUserAttachments(ctx.User.ID)
  125. if err != nil {
  126. log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"])
  127. return err
  128. }
  129. ctx.Data["attachments"] = attachs
  130. ctx.Data["command"] = cloudbrain.Command
  131. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  132. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  133. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  134. ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
  135. ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
  136. if categories == nil {
  137. json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
  138. }
  139. ctx.Data["benchmark_categories"] = categories.Category
  140. if gpuInfos == nil {
  141. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  142. }
  143. ctx.Data["gpu_types"] = gpuInfos.GpuInfo
  144. if cloudbrain.ResourceSpecs == nil {
  145. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  146. }
  147. ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec
  148. ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
  149. ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
  150. ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
  151. ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled
  152. return nil
  153. }
  154. func CloudBrainNew(ctx *context.Context) {
  155. err := cloudBrainNewDataPrepare(ctx)
  156. if err != nil {
  157. ctx.ServerError("get new cloudbrain info failed", err)
  158. return
  159. }
  160. ctx.HTML(200, tplCloudBrainNew)
  161. }
  162. func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  163. ctx.Data["PageIsCloudBrain"] = true
  164. jobName := form.JobName
  165. image := form.Image
  166. command := form.Command
  167. uuid := form.Attachment
  168. jobType := form.JobType
  169. gpuQueue := form.GpuType
  170. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  171. resourceSpecId := form.ResourceSpecId
  172. if !jobNamePattern.MatchString(jobName) {
  173. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainNew, &form)
  174. return
  175. }
  176. if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
  177. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  178. cloudBrainNewDataPrepare(ctx)
  179. ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form)
  180. return
  181. }
  182. _, err := models.GetCloudbrainByName(jobName)
  183. if err == nil {
  184. log.Error("the job name did already exist", ctx.Data["MsgID"])
  185. cloudBrainNewDataPrepare(ctx)
  186. ctx.RenderWithErr("the job name did already exist", tplCloudBrainNew, &form)
  187. return
  188. } else {
  189. if !models.IsErrJobNotExist(err) {
  190. log.Error("system error, %v", err, ctx.Data["MsgID"])
  191. cloudBrainNewDataPrepare(ctx)
  192. ctx.RenderWithErr("system error", tplCloudBrainNew, &form)
  193. return
  194. }
  195. }
  196. repo := ctx.Repo.Repository
  197. downloadCode(repo, codePath)
  198. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
  199. err = os.MkdirAll(modelPath, os.ModePerm)
  200. if err != nil {
  201. cloudBrainNewDataPrepare(ctx)
  202. ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
  203. return
  204. }
  205. benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
  206. if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
  207. var gpuType string
  208. for _, gpuInfo := range gpuInfos.GpuInfo {
  209. if gpuInfo.Queue == gpuQueue {
  210. gpuType = gpuInfo.Value
  211. }
  212. }
  213. downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType)
  214. }
  215. snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
  216. if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
  217. downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "")
  218. }
  219. brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
  220. if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
  221. downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "")
  222. }
  223. err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId)
  224. if err != nil {
  225. cloudBrainNewDataPrepare(ctx)
  226. ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
  227. return
  228. }
  229. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  230. }
  231. func CloudBrainShow(ctx *context.Context) {
  232. ctx.Data["PageIsCloudBrain"] = true
  233. var jobID = ctx.Params(":jobid")
  234. task, err := models.GetCloudbrainByJobID(jobID)
  235. if err != nil {
  236. ctx.Data["error"] = err.Error()
  237. }
  238. result, err := cloudbrain.GetJob(jobID)
  239. if err != nil {
  240. ctx.Data["error"] = err.Error()
  241. }
  242. if result != nil {
  243. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  244. jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
  245. taskRoles := jobRes.TaskRoles
  246. if jobRes.JobStatus.State != string(models.JobFailed) {
  247. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  248. ctx.Data["taskRes"] = taskRes
  249. task.Status = taskRes.TaskStatuses[0].State
  250. task.ContainerID = taskRes.TaskStatuses[0].ContainerID
  251. task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  252. err = models.UpdateJob(task)
  253. if err != nil {
  254. ctx.Data["error"] = err.Error()
  255. }
  256. } else {
  257. task.Status = jobRes.JobStatus.State
  258. taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{
  259. {
  260. State: jobRes.JobStatus.State,
  261. },
  262. }}
  263. ctx.Data["taskRes"] = taskRes
  264. jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05")
  265. jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05")
  266. }
  267. ctx.Data["result"] = jobRes
  268. }
  269. ctx.Data["task"] = task
  270. ctx.Data["jobID"] = jobID
  271. ctx.HTML(200, tplCloudBrainShow)
  272. }
  273. func CloudBrainDebug(ctx *context.Context) {
  274. var jobID = ctx.Params(":jobid")
  275. if !ctx.IsSigned {
  276. log.Error("the user has not signed in")
  277. ctx.Error(http.StatusForbidden, "","the user has not signed in")
  278. return
  279. }
  280. task, err := models.GetCloudbrainByJobID(jobID)
  281. if err != nil {
  282. ctx.ServerError("GetCloudbrainByJobID failed", err)
  283. return
  284. }
  285. debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
  286. ctx.Redirect(debugUrl)
  287. }
  288. func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  289. var jobID = ctx.Params(":jobid")
  290. if !ctx.IsSigned {
  291. log.Error("the user has not signed in")
  292. ctx.Error(http.StatusForbidden, "","the user has not signed in")
  293. return
  294. }
  295. task, err := models.GetCloudbrainByJobID(jobID)
  296. if err != nil {
  297. ctx.JSON(200, map[string]string{
  298. "result_code": "-1",
  299. "error_msg": "GetCloudbrainByJobID failed",
  300. })
  301. return
  302. }
  303. err = cloudbrain.CommitImage(jobID, models.CommitImageParams{
  304. Ip: task.ContainerIp,
  305. TaskContainerId: task.ContainerID,
  306. ImageDescription: form.Description,
  307. ImageTag: form.Tag,
  308. })
  309. if err != nil {
  310. log.Error("CommitImage(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"])
  311. ctx.JSON(200, map[string]string{
  312. "result_code": "-1",
  313. "error_msg": "CommitImage failed",
  314. })
  315. return
  316. }
  317. ctx.JSON(200, map[string]string{
  318. "result_code": "0",
  319. "error_msg": "",
  320. })
  321. }
  322. func CloudBrainStop(ctx *context.Context) {
  323. var jobID = ctx.Params(":jobid")
  324. task, err := models.GetCloudbrainByJobID(jobID)
  325. if err != nil {
  326. ctx.ServerError("GetCloudbrainByJobID failed", err)
  327. return
  328. }
  329. if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) {
  330. log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
  331. ctx.ServerError("the job has been stopped", errors.New("the job has been stopped"))
  332. return
  333. }
  334. err = cloudbrain.StopJob(jobID)
  335. if err != nil {
  336. log.Error("StopJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"])
  337. ctx.ServerError("StopJob failed", err)
  338. return
  339. }
  340. task.Status = string(models.JobStopped)
  341. err = models.UpdateJob(task)
  342. if err != nil {
  343. ctx.ServerError("UpdateJob failed", err)
  344. return
  345. }
  346. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  347. }
  348. func StopJobsByUserID(userID int64) {
  349. cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
  350. if err != nil {
  351. log.Warn("Failed to get cloudBrain info", err)
  352. return
  353. }
  354. StopJobs(cloudBrains)
  355. }
  356. func StopJobsByRepoID(repoID int64) {
  357. cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
  358. if err != nil {
  359. log.Warn("Failed to get cloudBrain info", err)
  360. return
  361. }
  362. StopJobs(cloudBrains)
  363. }
  364. /**
  365. */
  366. func StopJobs(cloudBrains []*models.Cloudbrain) {
  367. for _, taskInfo := range cloudBrains {
  368. if taskInfo.Type == models.TypeCloudBrainOne {
  369. err := retry(3, time.Second*30, func() error {
  370. return cloudbrain.StopJob(taskInfo.JobID)
  371. })
  372. logErrorAndUpdateJobStatus(err, taskInfo)
  373. } else {
  374. param := models.NotebookAction{
  375. Action: models.ActionStop,
  376. }
  377. err := retry(3, time.Second*30, func() error {
  378. _, err := modelarts.StopJob(taskInfo.JobID, param)
  379. return err
  380. })
  381. logErrorAndUpdateJobStatus(err, taskInfo)
  382. }
  383. }
  384. }
  385. func retry(attempts int, sleep time.Duration, f func() error) (err error) {
  386. for i := 0; i < attempts; i++ {
  387. if i > 0 {
  388. log.Warn("retrying after error:", err)
  389. time.Sleep(sleep)
  390. }
  391. err = f()
  392. if err == nil {
  393. return nil
  394. }
  395. }
  396. return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
  397. }
  398. func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
  399. if err != nil {
  400. log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
  401. } else {
  402. taskInfo.Status = string(models.JobStopped)
  403. err = models.UpdateJob(taskInfo)
  404. if err != nil {
  405. log.Warn("UpdateJob failed", err)
  406. }
  407. }
  408. }
  409. func CloudBrainDel(ctx *context.Context) {
  410. var jobID = ctx.Params(":jobid")
  411. task, err := models.GetCloudbrainByJobID(jobID)
  412. if err != nil {
  413. ctx.ServerError("GetCloudbrainByJobID failed", err)
  414. return
  415. }
  416. if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) {
  417. log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
  418. ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
  419. return
  420. }
  421. err = models.DeleteJob(task)
  422. if err != nil {
  423. ctx.ServerError("DeleteJob failed", err)
  424. return
  425. }
  426. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  427. }
  428. func CloudBrainShowModels(ctx *context.Context) {
  429. ctx.Data["PageIsCloudBrain"] = true
  430. jobID := ctx.Params(":jobid")
  431. parentDir := ctx.Query("parentDir")
  432. dirArray := strings.Split(parentDir, "/")
  433. task, err := models.GetCloudbrainByJobID(jobID)
  434. if err != nil {
  435. log.Error("no such job!", ctx.Data["msgID"])
  436. ctx.ServerError("no such job:", err)
  437. return
  438. }
  439. //get dirs
  440. dirs, err := getModelDirs(task.JobName, parentDir)
  441. if err != nil {
  442. log.Error("getModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  443. ctx.ServerError("getModelDirs failed:", err)
  444. return
  445. }
  446. var fileInfos []storage.FileInfo
  447. err = json.Unmarshal([]byte(dirs), &fileInfos)
  448. if err != nil {
  449. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  450. ctx.ServerError("json.Unmarshal failed:", err)
  451. return
  452. }
  453. for i, fileInfo := range fileInfos {
  454. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  455. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  456. }
  457. sort.Slice(fileInfos, func(i, j int) bool {
  458. return fileInfos[i].ModTime > fileInfos[j].ModTime
  459. })
  460. ctx.Data["Path"] = dirArray
  461. ctx.Data["Dirs"] = fileInfos
  462. ctx.Data["task"] = task
  463. ctx.Data["JobID"] = jobID
  464. ctx.HTML(200, tplCloudBrainShowModels)
  465. }
  466. func GetPublicImages(ctx *context.Context) {
  467. getImages(ctx, cloudbrain.Public)
  468. }
  469. func GetCustomImages(ctx *context.Context) {
  470. getImages(ctx, cloudbrain.Custom)
  471. }
  472. func getImages(ctx *context.Context, imageType string) {
  473. log.Info("Get images begin")
  474. page := ctx.QueryInt("page")
  475. size := ctx.QueryInt("size")
  476. name := ctx.Query("name")
  477. getImagesResult, err := cloudbrain.GetImagesPageable(page, size, imageType, name)
  478. if err != nil {
  479. log.Error("Can not get images:%v", err)
  480. ctx.JSON(http.StatusOK, models.GetImagesPayload{
  481. Count: 0,
  482. TotalPages: 0,
  483. ImageInfo: []*models.ImageInfo{},
  484. })
  485. } else {
  486. ctx.JSON(http.StatusOK, getImagesResult.Payload)
  487. }
  488. log.Info("Get images end")
  489. }
  490. func getModelDirs(jobName string, parentDir string) (string, error) {
  491. var req string
  492. modelActualPath := setting.JobPath + jobName + "/model/"
  493. if parentDir == "" {
  494. req = "baseDir=" + modelActualPath
  495. } else {
  496. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  497. }
  498. return getDirs(req)
  499. }
  500. func CloudBrainDownloadModel(ctx *context.Context) {
  501. parentDir := ctx.Query("parentDir")
  502. fileName := ctx.Query("fileName")
  503. jobName := ctx.Query("jobName")
  504. filePath := "jobs/" + jobName + "/model/" + parentDir
  505. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  506. if err != nil {
  507. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  508. ctx.ServerError("PresignedGetURL", err)
  509. return
  510. }
  511. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  512. }
  513. func GetRate(ctx *context.Context) {
  514. var jobID = ctx.Params(":jobid")
  515. job, err := models.GetCloudbrainByJobID(jobID)
  516. if err != nil {
  517. ctx.ServerError("GetCloudbrainByJobID failed", err)
  518. return
  519. }
  520. if job.JobType == string(models.JobTypeBenchmark) {
  521. ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  522. } else if job.JobType == string(models.JobTypeSnn4imagenet) {
  523. ctx.Redirect(setting.Snn4imagenetServerHost)
  524. } else if job.JobType == string(models.JobTypeBrainScore) {
  525. ctx.Redirect(setting.BrainScoreServerHost)
  526. } else {
  527. log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
  528. }
  529. }
  530. func downloadCode(repo *models.Repository, codePath string) error {
  531. if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
  532. log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
  533. return err
  534. }
  535. configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
  536. if err != nil {
  537. log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
  538. return err
  539. }
  540. defer configFile.Close()
  541. pos := int64(0)
  542. reader := bufio.NewReader(configFile)
  543. for {
  544. line, err := reader.ReadString('\n')
  545. if err != nil {
  546. if err == io.EOF {
  547. log.Error("not find the remote-url")
  548. return nil
  549. } else {
  550. log.Error("read error: %v", err)
  551. return err
  552. }
  553. }
  554. if strings.Contains(line, "url") && strings.Contains(line, ".git") {
  555. originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
  556. if len(line) > len(originUrl) {
  557. originUrl += strings.Repeat(" ", len(line)-len(originUrl))
  558. }
  559. bytes := []byte(originUrl)
  560. _, err := configFile.WriteAt(bytes, pos)
  561. if err != nil {
  562. log.Error("WriteAt failed:%v", err)
  563. return err
  564. }
  565. break
  566. }
  567. pos += int64(len(line))
  568. }
  569. return nil
  570. }
  571. func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error {
  572. err := os.MkdirAll(codePath, os.ModePerm)
  573. if err != nil {
  574. log.Error("mkdir codePath failed", err.Error())
  575. return err
  576. }
  577. command := "git clone " + gitPath + " " + codePath
  578. cmd := exec.Command("/bin/bash", "-c", command)
  579. output, err := cmd.Output()
  580. log.Info(string(output))
  581. if err != nil {
  582. log.Error("exec.Command(%s) failed:%v", command, err)
  583. return err
  584. }
  585. fileName := codePath + cloudbrain.TaskInfoName
  586. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  587. if err != nil {
  588. log.Error("OpenFile failed", err.Error())
  589. return err
  590. }
  591. defer f.Close()
  592. data, err := json.Marshal(models.TaskInfo{
  593. Username: repo.Owner.Name,
  594. TaskName: taskName,
  595. CodeName: repo.Name,
  596. BenchmarkCategory: strings.Split(benchmarkCategory, ","),
  597. CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
  598. GpuType: gpuType,
  599. })
  600. if err != nil {
  601. log.Error("json.Marshal failed", err.Error())
  602. return err
  603. }
  604. _, err = f.Write(data)
  605. if err != nil {
  606. log.Error("WriteString failed", err.Error())
  607. return err
  608. }
  609. return nil
  610. }