Browse Source

Merge branch 'train-job' into liuzx_trainjob

tags/v1.21.12.1
Gitea 4 years ago
parent
commit
1530a5d1a8
30 changed files with 3658 additions and 31 deletions
  1. +2
    -2
      Makefile
  2. +1
    -0
      custom/conf/app.ini.sample
  3. +1
    -1
      models/attachment.go
  4. +259
    -1
      models/cloudbrain.go
  5. +4
    -1
      models/file_chunk.go
  6. +10
    -9
      models/repo.go
  7. +29
    -0
      modules/auth/modelarts.go
  8. +2
    -1
      modules/cloudbrain/cloudbrain.go
  9. +180
    -3
      modules/modelarts/modelarts.go
  10. +485
    -4
      modules/modelarts/resty.go
  11. +4
    -0
      modules/setting/setting.go
  12. +46
    -0
      options/locale/locale_zh-CN.ini
  13. +28
    -0
      public/self/test.js
  14. +9
    -1
      routers/api/v1/api.go
  15. +80
    -1
      routers/api/v1/repo/modelarts.go
  16. +1
    -1
      routers/repo/attachment.go
  17. +788
    -2
      routers/repo/modelarts.go
  18. +25
    -0
      routers/routes/routes.go
  19. +6
    -4
      templates/repo/header.tmpl
  20. +5
    -0
      templates/repo/modelarts/index.tmpl
  21. +43
    -0
      templates/repo/modelarts/navbar.tmpl
  22. +221
    -0
      templates/repo/modelarts/notebook/index.tmpl
  23. +82
    -0
      templates/repo/modelarts/notebook/new.tmpl
  24. +122
    -0
      templates/repo/modelarts/notebook/show.tmpl
  25. +239
    -0
      templates/repo/modelarts/trainjob/edit_para.tmpl
  26. +225
    -0
      templates/repo/modelarts/trainjob/index.tmpl
  27. +407
    -0
      templates/repo/modelarts/trainjob/new.tmpl
  28. +154
    -0
      templates/repo/modelarts/trainjob/para_manage.tmpl
  29. +200
    -0
      templates/repo/modelarts/trainjob/show.tmpl
  30. +0
    -0
      web_src/less/themes/theme-arc-green.less

+ 2
- 2
Makefile View File

@@ -498,7 +498,7 @@ check: test

.PHONY: install $(TAGS_PREREQ)
install: $(wildcard *.go)
$(GO) install -v -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)'
$(GO) install -v -tags '$(TAGS)' -ldflags ' $(LDFLAGS)'

.PHONY: build
build: frontend backend
@@ -514,7 +514,7 @@ generate: $(TAGS_PREREQ)
CC= GOOS= GOARCH= $(GO) generate -mod=vendor -tags '$(TAGS)' $(GO_PACKAGES)

$(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
$(GO) build -mod=vendor $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
$(GO) build -mod=vendor $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags ' $(LDFLAGS)' -o $@

.PHONY: release
release: frontend generate release-windows release-linux release-darwin release-copy release-compress release-sources release-check


+ 1
- 0
custom/conf/app.ini.sample View File

@@ -1096,6 +1096,7 @@ LOCATION = cn-south-222
BASE_PATH = attachment/

[modelarts]
ORGANIZATION = modelarts
ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn
PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa
PROJECT_NAME = cn-south-222_test


+ 1
- 1
models/attachment.go View File

@@ -429,7 +429,7 @@ func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
func getModelArtsUserAttachments(e Engine, userID int64) ([]*AttachmentUsername, error) {
attachments := make([]*AttachmentUsername, 0, 10)
if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id "+
"= `user`.id").Where("attachment.type = ? and (uploader_id= ? or is_private = ?)", TypeCloudBrainTwo, userID, false).Find(&attachments); err != nil {
"= `user`.id").Where("attachment.type = ? and (uploader_id= ? or is_private = ?)", TypeCloudBrainNotebook, userID, false).Find(&attachments); err != nil {
return nil, err
}
return attachments, nil


+ 259
- 1
models/cloudbrain.go View File

@@ -5,6 +5,7 @@ import (
"fmt"
"strings"
"time"

"xorm.io/builder"
"xorm.io/xorm"

@@ -63,6 +64,10 @@ type Cloudbrain struct {
CanDel bool `xorm:"-"`
Type int `xorm:"INDEX DEFAULT 0"`

VersionID int64 `xorm:"INDEX DEFAULT 0"`
VersionName string
Uuid string

User *User `xorm:"-"`
Repo *Repository `xorm:"-"`
}
@@ -530,7 +535,260 @@ type NotebookDelResult struct {
InstanceID string `json:"instance_id"`
}

func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
type CreateTrainJobParams struct {
JobName string `json:"job_name"`
Description string `json:"job_desc"`
Config Config `json:"config"`
WorkspaceID string `json:"workspace_id"`
}

type Config struct {
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
CreateVersion bool `json:"create_version"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
}

type CreateConfigParams struct {
ConfigName string `json:"config_name"`
Description string `json:"config_desc"`
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
//CreateVersion bool `json:"create_version"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
}

type Parameter struct {
Label string `json:"label"`
Value string `json:"value"`
}

type Parameters struct {
Parameter []Parameter `json:"parameter"`
}

type DataSource struct {
DatasetID string `json:"dataset_id"`
DatasetVersion string `json:"dataset_version"`
Type string `json:"type"`
DataUrl string `json:"data_url"`
}

type Volumes struct {
Nfs Nfs `json:"nfs"`
HostPath HostPath `json:"host_path"`
}

type Nfs struct {
ID string `json:"id"`
SourcePath string `json:"src_path"`
DestPath string `json:"dest_path"`
ReadOnly bool `json:"read_only"`
}

type HostPath struct {
SourcePath string `json:"src_path"`
DestPath string `json:"dest_path"`
ReadOnly bool `json:"read_only"`
}

type Flavor struct {
Code string `json:"code"`
}

type CreateTrainJobResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
JobName string `json:"job_name"`
JobID int64 `json:"job_id"`
Status int `json:"status"`
CreateTime int64 `json:"create_time"`
VersionID int64 `json:"version_id"`
ResourceID string `json:"resource_id"`
VersionName string `json:"version_name"`
}

type CreateTrainJobConfigResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
}

type GetResourceSpecsResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
SpecTotalCount int `json:"spec_total_count"`
Specs []Specs `json:"specs"`
}

type Specs struct {
Core string `json:"core"`
Cpu string `json:"cpu"`
IsNoResource bool `json:"no_resource"`
GpuType string `json:"gpu_type"`
SpecID int64 `json:"spec_id"`
GpuNum int `json:"gpu_num"`
SpecCode string `json:"spec_code"`
Storage string `json:"storage"`
MaxNum int `json:"max_num"`
UnitNum int `json:"unit_num"`
InterfaceType int `json:"interface_type"`
}

type GetConfigListResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
ConfigTotalCount int `json:"config_total_count"`
ParaConfigs []ParaConfig `json:"configs"`
}

type ParaConfig struct {
ConfigName string `json:"config_name"`
ConfigDesc string `json:"config_desc"`
CreateTime int64 `json:"create_time"`
EngineType int `json:"engine_type"`
EngineName string `json:"engine_name"`
EngineId int64 `json:"engine_id"`
EngineVersion string `json:"engine_version"`
UserImageUrl string `json:"user_image_url"`
UserCommand string `json:"user_command"`
Result GetConfigResult
}

type GetConfigResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
ConfigName string `json:"config_name"`
Description string `json:"config_desc"`
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
//CreateVersion bool `json:"create_version"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
}

type ErrorResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_message"`
IsSuccess bool `json:"is_success"`
}

type GetTrainJobResult struct {
IsSuccess bool `json:"is_success"`
JobName string `json:"job_name"`
JobID int64 `json:"job_id"`
Description string `json:"job_desc"`
IntStatus int `json:"status"`
Status string
LongCreateTime int64 `json:"create_time"`
CreateTime string
Duration int64 `json:"duration"` //训练作业的运行时间,单位为毫秒
VersionID int64 `json:"version_id"`
ResourceID string `json:"resource_id"`
VersionName string `json:"version_name"`
PreVersionID int64 `json:"pre_version_id"`
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
EngineName string `json:"engine_name"`
EngineVersion string `json:"engine_version"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
PoolName string `json:"pool_name"`
NasMountPath string `json:"nas_mount_path"`
NasShareAddr string `json:"nas_share_addr"`
DatasetName string
}

type GetTrainJobLogResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
Content string `json:"content"`
Lines int `json:"lines"`
StartLine string `json:"start_line"`
EndLine string `json:"end_line"`
}

type GetTrainJobLogFileNamesResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
LogFileList []string `json:"log_file_list"`
}

type TrainJobResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
}

type LogFile struct {
Name string
}

func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) {
sess := x.NewSession()
defer sess.Close()



+ 4
- 1
models/file_chunk.go View File

@@ -14,7 +14,10 @@ const (
)

const (
TypeCloudBrainOne = 0
TypeCloudBrainOne = 0
TypeCloudBrainNotebook = 1
TypeCloudBrainTrainJob = 2

TypeCloudBrainTwo = 1
)



+ 10
- 9
models/repo.go View File

@@ -6,13 +6,14 @@
package models

import (
"code.gitea.io/gitea/modules/blockchain"
"context"
"crypto/md5"
"errors"
"fmt"
"html/template"

"code.gitea.io/gitea/modules/blockchain"

// Needed for jpeg support
_ "image/jpeg"
"image/png"
@@ -171,11 +172,11 @@ type Repository struct {
NumOpenIssues int `xorm:"-"`
NumPulls int
NumClosedPulls int
NumOpenPulls int `xorm:"-"`
NumMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumOpenMilestones int `xorm:"-"`
NumCommit int64 `xorm:"NOT NULL DEFAULT 0"`
NumOpenPulls int `xorm:"-"`
NumMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumOpenMilestones int `xorm:"-"`
NumCommit int64 `xorm:"NOT NULL DEFAULT 0"`

IsPrivate bool `xorm:"INDEX"`
IsEmpty bool `xorm:"INDEX"`
@@ -215,8 +216,8 @@ type Repository struct {
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`

Hot int64 `xorm:"-"`
Active int64 `xorm:"-"`
Hot int64 `xorm:"-"`
Active int64 `xorm:"-"`
}

// SanitizedOriginalURL returns a sanitized OriginalURL
@@ -2464,7 +2465,7 @@ func (repo *Repository) IncreaseCloneCnt() {
}

func UpdateRepositoryCommitNum(repo *Repository) error {
if _,err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil {
if _, err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil {
return err
}



+ 29
- 0
modules/auth/modelarts.go View File

@@ -14,3 +14,32 @@ type CreateModelArtsForm struct {
func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

type CreateModelArtsNotebookForm struct {
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment"`
Description string `form:"description"`
}

func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

type CreateModelArtsTrainJobForm struct {
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment" binding:"Required"`
BootFile string `form:"boot_file" binding:"Required"`
WorkServerNumber int `form:"work_server_number" binding:"Required"`
EngineID int `form:"engine_id" binding:"Required"`
PoolID string `form:"pool_id" binding:"Required"`
Flavor string `form:"flavor" binding:"Required"`
Params string `form:"run_para_list" binding:"Required"`
Description string `form:"description"`
IsSaveParam string `form:"is_save_para"`
ParameterTemplateName string `form:"parameter_template_name"`
PrameterDescription string `form:"parameter_description"`
}

func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

+ 2
- 1
modules/cloudbrain/cloudbrain.go View File

@@ -123,7 +123,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath,
JobName: jobName,
SubTaskName: SubTaskName,
JobType: jobType,
Type: models.TypeCloudBrainOne,
Type: models.TypeCloudBrainOne,
Uuid: uuid,
})

if err != nil {


+ 180
- 3
modules/modelarts/modelarts.go View File

@@ -1,22 +1,53 @@
package modelarts

import (
"encoding/json"
"path"
"strconv"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"encoding/json"
"path"
)

const (
//notebook
storageTypeOBS = "obs"
autoStopDuration = 4 * 60 * 60

DataSetMountPath = "/home/ma-user/work"
NotebookEnv = "Python3"
NotebookType = "Ascend"
FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB (modelarts.kat1.xlarge)"

//train-job
ResourcePools = "{\"resource_pool\":[{\"id\":\"pool1328035d\", \"value\":\"专属资源池\"}]}"
Engines = "{\"engine\":[{\"id\":1, \"value\":\"Ascend-Powered-Engine\"}]}"
EngineVersions = "{\"version\":[{\"id\":118,\"value\":\"MindSpore-1.0.0-c75-python3.7-euleros2.8-aarch64\"}," +
"{\"id\":119,\"value\":\"MindSpore-1.1.1-c76-python3.7-euleros2.8-aarch64\"}," +
"{\"id\":120,\"value\":\"MindSpore-1.1.1-c76-tr5-python3.7-euleros2.8-aarch64\"}," +
"{\"id\":117,\"value\":\"TF-1.15-c75-python3.7-euleros2.8-aarch64\"}" +
"]}"
// FlavorInfos = "{\"flavor\":[{\"code\":\"modelarts.bm.910.arm.public.2\",\"value\":\"Ascend : 2 * Ascend 910 CPU:48 核 512GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.8\",\"value\":\"Ascend : 8 * Ascend 910 CPU:192 核 2048GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU:96 核 1024GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU:24 核 256GiB\"}" +
// "]}"
CodePath = "/code/"
OutputPath = "/output/"
LogPath = "/log/"
JobPath = "/job/"
OrderDesc = "desc" //向下查询
OrderAsc = "asc" //向上查询
Lines = 20
TrainUrl = "train_url"
DataUrl = "data_url"
PerPage = 10

SortByCreateTime = "create_time"
ConfigTypeCustom = "custom"
)

var (
@@ -24,6 +55,50 @@ var (
FlavorInfos *models.FlavorInfos
)

type GenerateTrainJobReq struct {
JobName string
Uuid string
Description string
CodeObsPath string
BootFile string
DataUrl string
TrainUrl string
FlavorCode string
LogUrl string
PoolID string
WorkServerNumber int
EngineID int64
Parameters []models.Parameter
}

type VersionInfo struct {
Version []struct {
ID int `json:"id"`
Value string `json:"value"`
} `json:"version"`
}

type Flavor struct {
Info []struct {
Code string `json:"code"`
Value string `json:"value"`
} `json:"flavor"`
}

type Engine struct {
Info []struct {
ID int `json:"id"`
Value string `json:"value"`
} `json:"engine"`
}

type ResourcePool struct {
Info []struct {
ID string `json:"id"`
Value string `json:"value"`
} `json:"resource_pool"`
}

func GenerateTask(ctx *context.Context, jobName, uuid, description string) error {
var dataActualPath string
if uuid != "" {
@@ -78,13 +153,15 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description string) error
}

err = models.CreateCloudbrain(&models.Cloudbrain{

Status: string(models.JobWaiting),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: jobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTwo,
Type: models.TypeCloudBrainNotebook,
Uuid: uuid,
})

if err != nil {
@@ -93,3 +170,103 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description string) error

return nil
}

func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) error {
jobResult, err := createTrainJob(models.CreateTrainJobParams{
JobName: req.JobName,
Description: req.Description,
Config: models.Config{
WorkServerNum: req.WorkServerNumber,
AppUrl: req.CodeObsPath,
BootFileUrl: req.BootFile,
DataUrl: req.DataUrl,
EngineID: req.EngineID,
TrainUrl: req.TrainUrl,
LogUrl: req.LogUrl,
PoolID: req.PoolID,
CreateVersion: true,
Flavor: models.Flavor{
Code: req.FlavorCode,
},
Parameter: req.Parameters,
},
})
if err != nil {
log.Error("CreateJob failed: %v", err.Error())
return err
}

err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: strconv.FormatInt(jobResult.JobID, 10),
JobName: req.JobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTrainJob,
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
})

if err != nil {
log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error())
return err
}

return nil
}

func TransTrainJobStatus(status int) string {
switch status {
case 0:
return "UNKNOWN"
case 1:
return "INIT"
case 2:
return "IMAGE_CREATING"
case 3:
return "IMAGE_FAILED"
case 4:
return "SUBMIT_TRYING"
case 5:
return "SUBMIT_FAILED"
case 6:
return "DELETE_FAILED"
case 7:
return "WAITING"
case 8:
return "RUNNING"
case 9:
return "KILLING"
case 10:
return "COMPLETED"
case 11:
return "FAILED"
case 12:
return "KILLED"
case 13:
return "CANCELED"
case 14:
return "LOST"
case 15:
return "SCALING"
case 16:
return "SUBMIT_MODEL_FAILED"
case 17:
return "DEPLOY_SERVICE_FAILED"
case 18:
return "CHECK_INIT"
case 19:
return "CHECK_RUNNING"
case 20:
return "CHECK_RUNNING_COMPLETED"
case 21:
return "CHECK_FAILED"

default:
return strconv.Itoa(status)
}

return ""
}

+ 485
- 4
modules/modelarts/resty.go View File

@@ -1,13 +1,14 @@
package modelarts

import (
"code.gitea.io/gitea/modules/log"
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"strconv"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/go-resty/resty/v2"
)
@@ -23,6 +24,9 @@ const (

urlGetToken = "/v3/auth/tokens"
urlNotebook = "/demanager/instances"
urlTrainJob = "/training-jobs"
urlResourceSpecs = "/job/resource-specs"
urlTrainJobConfig = "/training-job-configs"
errorCodeExceedLimit = "ModelArts.0118"
)

@@ -104,7 +108,7 @@ sendjob:
Post(HOST + "/v1/" + setting.ProjectID + urlNotebook)

if err != nil {
return nil, fmt.Errorf("resty create job: %s", err)
return nil, fmt.Errorf("resty create notebook: %s", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
@@ -121,11 +125,11 @@ sendjob:
}

if len(response.ErrorCode) != 0 {
log.Error("CreateJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
log.Error("createNotebook failed(%s): %s", response.ErrorCode, response.ErrorMsg)
if response.ErrorCode == errorCodeExceedLimit {
response.ErrorMsg = "所选规格使用数量已超过最大配额限制。"
}
return &result, fmt.Errorf("CreateJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
return &result, fmt.Errorf("createNotebook failed(%s): %s", response.ErrorCode, response.ErrorMsg)
}

return &result, nil
@@ -210,6 +214,45 @@ sendjob:
return &result, nil
}

func DelNotebook(jobID string) (*models.NotebookDelResult, error) {
checkSetting()
client := getRestyClient()
var result models.NotebookDelResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetResult(&result).
Delete(HOST + "/v1/" + setting.ProjectID + urlNotebook + "/" + jobID)

if err != nil {
return &result, fmt.Errorf("resty DelJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

var response models.NotebookResult
err = json.Unmarshal(res.Body(), &response)
if err != nil {
log.Error("json.Unmarshal failed: %s", err.Error())
return &result, fmt.Errorf("son.Unmarshal failed: %s", err.Error())
}

if len(response.ErrorCode) != 0 {
log.Error("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
return &result, fmt.Errorf("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
}

return &result, nil
}

func DelJob(jobID string) (*models.NotebookDelResult, error) {
checkSetting()
client := getRestyClient()
@@ -287,3 +330,441 @@ sendjob:

return &result, nil
}

func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.CreateTrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.CreateTrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(createJobParams).
SetResult(&result).
Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob)

if err != nil {
return nil, fmt.Errorf("resty create train-job: %s", err)
}

req, _ := json.Marshal(createJobParams)
log.Info("%s", req)

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("createTrainJob failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("createTrainJob failed(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetResourceSpecs() (*models.GetResourceSpecsResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetResourceSpecsResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlResourceSpecs)

if err != nil {
return nil, fmt.Errorf("resty GetResourceSpecs: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetResourceSpecs failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("GetResourceSpecs failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetResourceSpecs failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("GetResourceSpecs failed(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func CreateTrainJobConfig(req models.CreateConfigParams) (*models.CreateTrainJobConfigResult, error) {
checkSetting()
client := getRestyClient()
var result models.CreateTrainJobConfigResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(req).
SetResult(&result).
Post(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig)

if err != nil {
return nil, fmt.Errorf("resty CreateTrainJobConfig: %s", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

//temp, _ := json.Marshal(req)
//log.Info("%s", temp)

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("CreateTrainJobConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("CreateTrainJobConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("CreateTrainJobConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("CreateTrainJobConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetConfigList(perPage, page int, sortBy, order, searchContent, configType string) (*models.GetConfigListResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetConfigListResult

retry := 0

sendjob:
res, err := client.R().
SetQueryParams(map[string]string{
"per_page": strconv.Itoa(perPage),
"page": strconv.Itoa(page),
"sortBy": sortBy,
"order": order,
"search_content": searchContent,
"config_type": configType,
}).
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig)

if err != nil {
return nil, fmt.Errorf("resty GetConfigList: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetConfigList failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("获取参数配置列表失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetConfigList failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("获取参数配置列表失败(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetParaConfig(configName, configType string) (models.GetConfigResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetConfigResult

retry := 0

sendjob:
res, err := client.R().
SetQueryParams(map[string]string{
"config_type": configType,
}).
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig + "/" + configName)

if err != nil {
return result, fmt.Errorf("resty GetParaConfig: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetParaConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return result, fmt.Errorf("获取参数配置详情失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetParaConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return result, fmt.Errorf("获取参数配置详情失败(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return result, nil
}

func GetTrainJob(jobID, versionID string) (*models.GetTrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetTrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID)

if err != nil {
return nil, fmt.Errorf("resty GetTrainJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("获取作业详情失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetTrainJob(%s) failed", jobID)
return &result, fmt.Errorf("获取作业详情失败")
}

return &result, nil
}

func GetTrainJobLog(jobID, versionID, baseLine, logFile, order string, lines int) (*models.GetTrainJobLogResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetTrainJobLogResult

retry := 0

sendjob:
res, err := client.R().
SetQueryParams(map[string]string{
"base_line": baseLine,
"lines": strconv.Itoa(lines),
"log_file": logFile,
"order": order,
}).
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/aom-log")

if err != nil {
return nil, fmt.Errorf("resty GetTrainJobLog: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJobLog failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("获取作业日志失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetTrainJobLog(%s) failed", jobID)
return &result, fmt.Errorf("获取作业日志失败:%s", result.ErrorMsg)
}

return &result, nil
}

func GetTrainJobLogFileNames(jobID, versionID string) (*models.GetTrainJobLogFileNamesResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetTrainJobLogFileNamesResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/log/file-names")

if err != nil {
return nil, fmt.Errorf("resty GetTrainJobLogFileNames: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJobLogFileNames failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("GetTrainJobLogFileNames failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetTrainJobLogFileNames(%s) failed", jobID)
return &result, fmt.Errorf("获取作业日志文件失败:%s", result.ErrorMsg)
}

return &result, nil
}

func DelTrainJob(jobID string) (*models.TrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.TrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Delete(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID)

if err != nil {
return &result, fmt.Errorf("resty DelTrainJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("DelTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("删除训练作业失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("DelTrainJob(%s) failed", jobID)
return &result, fmt.Errorf("删除训练作业失败:%s", result.ErrorMsg)
}

return &result, nil
}

func StopTrainJob(jobID, versionID string) (*models.TrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.TrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/stop")

if err != nil {
return &result, fmt.Errorf("resty StopTrainJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("StopTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("停止训练作业失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("StopTrainJob(%s) failed", jobID)
return &result, fmt.Errorf("停止训练作业失败:%s", result.ErrorMsg)
}

return &result, nil
}

+ 4
- 0
modules/setting/setting.go View File

@@ -468,6 +468,7 @@ var (
Bucket string
Location string
BasePath string
CodePathPrefix string
UserBasePath string

//modelarts config
@@ -478,6 +479,7 @@ var (
ModelArtsUsername string
ModelArtsPassword string
ModelArtsDomain string
AllowedOrg string
ProfileID string
PoolInfos string
Flavor string
@@ -1198,6 +1200,7 @@ func NewContext() {
Bucket = sec.Key("BUCKET").MustString("testopendata")
Location = sec.Key("LOCATION").MustString("cn-south-222")
BasePath = sec.Key("BASE_PATH").MustString("attachment/")
CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/")
UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/")
PROXYURL = sec.Key("PROXY_URL").MustString("")

@@ -1209,6 +1212,7 @@ func NewContext() {
ModelArtsUsername = sec.Key("USERNAME").MustString("")
ModelArtsPassword = sec.Key("PASSWORD").MustString("")
ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222")
AllowedOrg = sec.Key("ORGANIZATION").MustString("")
ProfileID = sec.Key("PROFILE_ID").MustString("")
PoolInfos = sec.Key("POOL_INFOS").MustString("")
Flavor = sec.Key("FLAVOR").MustString("")


+ 46
- 0
options/locale/locale_zh-CN.ini View File

@@ -777,6 +777,52 @@ cloudbrain_task=任务名称
cloudbrain_operate=操作
cloudbrain_status_createtime=状态/创建时间

modelarts.notebook=调试作业
modelarts.train_job=训练作业
modelarts.train_job.new=新建作业
modelarts.train_job.basic_info=基本信息
modelarts.train_job.job_status=作业状态
modelarts.train_job.job_name=作业名称
modelarts.train_job.version=作业版本
modelarts.train_job.start_time=开始时间
modelarts.train_job.dura_time=持续时间
modelarts.train_job.description=作业描述
modelarts.train_job.parameter_setting=参数设置
modelarts.train_job.parameter_setting_info=参数信息
modelarts.train_job.fast_parameter_setting=一键式参数配置
modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置,可单击
modelarts.train_job.fast_parameter_setting_config_link=这里
modelarts.train_job.frames=常用框架
modelarts.train_job.algorithm_origin=算法来源
modelarts.train_job.AI_driver=AI引擎
modelarts.train_job.start_file=启动文件
modelarts.train_job.boot_file_helper=启动文件是您程序执行的入口文件,必须是以.py结尾的文件。
modelarts.train_job.dataset=数据集
modelarts.train_job.run_parameter=运行参数
modelarts.train_job.add_run_parameter=增加运行参数
modelarts.train_job.parameter_name=参数名
modelarts.train_job.parameter_value=参数值
modelarts.train_job.resource_setting=资源设置
modelarts.train_job.resource_setting_info=资源信息
modelarts.train_job.resource_pool=资源池
modelarts.train_job.resource_type=资源类型
modelarts.train_job.standard=规格
modelarts.train_job.NAS_address=NAS地址
modelarts.train_job.NAS_mount_path=NAS挂载路径
modelarts.train_job.query_whether_save_parameter=保存作业参数
modelarts.train_job.save_helper=保存当前作业的配置参数,后续您可以使用已保存的配置参数快速创建训练作业。
modelarts.train_job.common_frame=常用框架
modelarts.train_job.amount_of_compute_node=计算节点个数
modelarts.train_job.job_parameter_name=作业参数名称
modelarts.train_job.parameter_description=作业参数描述
modelarts.log=日志
modelarts.version_manage=版本管理
modelarts.back=返回
modelarts.train_job_para_admin=作业参数管理
modelarts.train_job_para.edit=编辑
modelarts.train_job_para.connfirm=确定


template.items=模板选项
template.git_content=Git数据(默认分支)
template.git_hooks=Git 钩子


+ 28
- 0
public/self/test.js View File

@@ -0,0 +1,28 @@

function displayDir(uuid){
console.log('uuid 1=' + uuid);
var html="<tr>\
<th></th>\
<th id=\"dataset_head\"></th>\
<th>数据集名称</th>\
<th>数据集类型</th>\
<th>数据集描述</th>\
<th>数据集创建者</th>\
</tr>";
for (var i=0;i<1;i++){
var row = "<tr>\
<td><input type=\"checkbox\"/></td>\
<td id=\"dataset_id\">"+uuid+"</td>\
<td>" + uuid +"</td>\
<td>测试</td>\
<td>测试</td>\
<td>测试</td>\
</tr>";
html=html+row;
}
document.getElementById('dataset-files-table').innerHTML=html;
console.log('uuid 2=' + uuid);
}

+ 9
- 1
routers/api/v1/api.go View File

@@ -853,7 +853,15 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/:jobid", repo.GetCloudbrainTask)
}, reqRepoReader(models.UnitTypeCloudBrain))
m.Group("/modelarts", func() {
m.Get("/:jobid", repo.GetModelArtsTask)
m.Group("/notebook", func() {
m.Get("/:jobid", repo.GetModelArtsNotebook)
})
m.Group("/train-job", func() {
m.Group("/:jobid", func() {
m.Get("", repo.GetModelArtsTrainJob)
m.Get("/log", repo.TrainJobGetLog)
})
})
}, reqRepoReader(models.UnitTypeCloudBrain))
}, repoAssignment())
})


+ 80
- 1
routers/api/v1/repo/modelarts.go View File

@@ -11,9 +11,10 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/modelarts"
"net/http"
"strconv"
)

func GetModelArtsTask(ctx *context.APIContext) {
func GetModelArtsNotebook(ctx *context.APIContext) {
var (
err error
)
@@ -43,3 +44,81 @@ func GetModelArtsTask(ctx *context.APIContext) {
})

}

func GetModelArtsTrainJob(ctx *context.APIContext) {
var (
err error
)

jobID := ctx.Params(":jobid")
repoID := ctx.Repo.Repository.ID
job, err := models.GetRepoCloudBrainByJobID(repoID, jobID)
if err != nil {
ctx.NotFound(err)
return
}
result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(job.VersionID, 10))
if err != nil {
ctx.NotFound(err)
return
}

job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"JobID": jobID,
"JobStatus": job.Status,
})

}

func TrainJobGetLog(ctx *context.APIContext) {
var (
err error
)

log.Info("test")

var jobID = ctx.Params(":jobid")
var logFileName = ctx.Query("file_name")
var baseLine = ctx.Query("base_line")
var order = ctx.Query("order")

if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
log.Error("order(%s) check failed", order)
ctx.JSON(http.StatusBadRequest, map[string]interface{}{
"err_msg": "order check failed",
})
return
}

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"err_msg": "GetCloudbrainByJobID failed",
})
return
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"err_msg": "GetTrainJobLog failed",
})
return
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"JobID": jobID,
"StartLine": result.StartLine,
"EndLine": result.EndLine,
"Content": result.Content,
"Lines": result.Lines,
})
}

+ 1
- 1
routers/repo/attachment.go View File

@@ -1000,7 +1000,7 @@ func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) {
}

func checkTypeCloudBrain(typeCloudBrain int) error {
if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainTwo {
if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainNotebook {
log.Error("type error:", typeCloudBrain)
return errors.New("type error")
}


+ 788
- 2
routers/repo/modelarts.go View File

@@ -1,14 +1,23 @@
package repo

import (
"code.gitea.io/gitea/modules/modelarts"
"encoding/json"
"errors"
"github.com/unknwon/com"
"fmt"
"io"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"

"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/obs"
"code.gitea.io/gitea/modules/storage"
"github.com/unknwon/com"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/base"
@@ -18,9 +27,18 @@ import (
)

const (
// tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index"
tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index"
tplModelArtsNotebookNew base.TplName = "repo/modelarts/notebook/new"
tplModelArtsNotebookShow base.TplName = "repo/modelarts/notebook/show"

tplModelArtsIndex base.TplName = "repo/modelarts/index"
tplModelArtsNew base.TplName = "repo/modelarts/new"
tplModelArtsShow base.TplName = "repo/modelarts/show"

tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index"
tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new"
tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show"
)

// MustEnableDataset check if repository enable internal cb
@@ -30,6 +48,7 @@ func MustEnableModelArts(ctx *context.Context) {
return
}
}

func ModelArtsIndex(ctx *context.Context) {
MustEnableModelArts(ctx)
repo := ctx.Repo.Repository
@@ -249,3 +268,770 @@ func ModelArtsDel(ctx *context.Context) {

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts")
}

func NotebookIndex(ctx *context.Context) {
MustEnableModelArts(ctx)
repo := ctx.Repo.Repository
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}

ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
Type: models.TypeCloudBrainNotebook,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
return
}

for i, task := range ciTasks {
if task.Status == string(models.JobRunning) {
ciTasks[i].CanDebug = true
} else {
ciTasks[i].CanDebug = false
}
}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager

ctx.Data["PageIsNotebook"] = true
ctx.Data["Tasks"] = ciTasks
ctx.HTML(200, tplModelArtsNotebookIndex)
}

func NotebookNew(ctx *context.Context) {
ctx.Data["PageIsNotebook"] = true

t := time.Now()
var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
ctx.Data["job_name"] = jobName

attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
if err != nil {
ctx.ServerError("GetAllUserAttachments failed:", err)
return
}

ctx.Data["attachments"] = attachs
ctx.Data["dataset_path"] = modelarts.DataSetMountPath
ctx.Data["env"] = modelarts.NotebookEnv
ctx.Data["notebook_type"] = modelarts.NotebookType
if modelarts.FlavorInfos == nil {
json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos)
}
ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo
ctx.HTML(200, tplModelArtsNew)
}

func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) {
ctx.Data["PageIsNotebook"] = true
jobName := form.JobName
uuid := form.Attachment
description := form.Description

err := modelarts.GenerateTask(ctx, jobName, uuid, description)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
}

func NotebookShow(ctx *context.Context) {
ctx.Data["PageIsNotebook"] = true

var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.Data["error"] = err.Error()
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

result, err := modelarts.GetJob(jobID)
if err != nil {
ctx.Data["error"] = err.Error()
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

if result != nil {
task.Status = result.Status
err = models.UpdateJob(task)
if err != nil {
ctx.Data["error"] = err.Error()
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

createTime, _ := com.StrTo(result.CreationTimestamp).Int64()
result.CreateTime = time.Unix(int64(createTime/1000), 0).Format("2006-01-02 15:04:05")
endTime, _ := com.StrTo(result.LatestUpdateTimestamp).Int64()
result.LatestUpdateTime = time.Unix(int64(endTime/1000), 0).Format("2006-01-02 15:04:05")
result.QueuingInfo.BeginTime = time.Unix(int64(result.QueuingInfo.BeginTimestamp/1000), 0).Format("2006-01-02 15:04:05")
result.QueuingInfo.EndTime = time.Unix(int64(result.QueuingInfo.EndTimestamp/1000), 0).Format("2006-01-02 15:04:05")
}

ctx.Data["task"] = task
ctx.Data["jobID"] = jobID
ctx.Data["result"] = result
ctx.HTML(200, tplModelArtsNotebookShow)
}

func NotebookDebug(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
_, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.ServerError("GetCloudbrainByJobID failed", err)
return
}

result, err := modelarts.GetJob(jobID)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

res, err := modelarts.GetJobToken(jobID)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

urls := strings.Split(result.Spec.Annotations.Url, "/")
urlPrefix := result.Spec.Annotations.TargetDomain
for i, url := range urls {
if i > 2 {
urlPrefix += "/" + url
}
}

debugUrl := urlPrefix + "?token=" + res.Token
ctx.Redirect(debugUrl)
}

func NotebookStop(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
log.Info(jobID)
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.ServerError("GetCloudbrainByJobID failed", err)
return
}

if task.Status != string(models.JobRunning) {
log.Error("the job(%s) is not running", task.JobName)
ctx.ServerError("the job is not running", errors.New("the job is not running"))
return
}

param := models.NotebookAction{
Action: models.ActionStop,
}
res, err := modelarts.StopJob(jobID, param)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.JobName, err.Error())
ctx.ServerError("StopJob failed", err)
return
}

task.Status = res.CurrentStatus
err = models.UpdateJob(task)
if err != nil {
ctx.ServerError("UpdateJob failed", err)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
}

func NotebookDel(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.ServerError("GetCloudbrainByJobID failed", err)
return
}

if task.Status != string(models.JobStopped) {
log.Error("the job(%s) has not been stopped", task.JobName)
ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
return
}

_, err = modelarts.DelNotebook(jobID)
if err != nil {
log.Error("DelJob(%s) failed:%v", task.JobName, err.Error())
ctx.ServerError("DelJob failed", err)
return
}

err = models.DeleteJob(task)
if err != nil {
ctx.ServerError("DeleteJob failed", err)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
}

func TrainJobIndex(ctx *context.Context) {
MustEnableModelArts(ctx)

can, err := canUserCreateTrainJob(ctx.User.ID)
if err != nil {
ctx.ServerError("canUserCreateTrainJob", err)
return
}

ctx.Data["CanCreate"] = can

repo := ctx.Repo.Repository
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}

tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
Type: models.TypeCloudBrainTrainJob,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
return
}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager

ctx.Data["PageIsTrainJob"] = true
ctx.Data["Tasks"] = tasks
ctx.HTML(200, tplModelArtsTrainJobIndex)
}

func TrainJobNew(ctx *context.Context) {
ctx.Data["PageIsTrainJob"] = true

can, err := canUserCreateTrainJob(ctx.User.ID)
if err != nil {
ctx.ServerError("canUserCreateTrainJob", err)
return
}

if !can {
log.Error("the user can not create train-job")
ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job"))
return
}

t := time.Now()
var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
ctx.Data["job_name"] = jobName

attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
if err != nil {
ctx.ServerError("GetAllUserAttachments failed:", err)
return
}
ctx.Data["attachments"] = attachs

var resourcePools modelarts.ResourcePool
if err = json.Unmarshal([]byte(modelarts.ResourcePools), &resourcePools); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["resource_pools"] = resourcePools.Info

var engines modelarts.Engine
if err = json.Unmarshal([]byte(modelarts.Engines), &engines); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["engines"] = engines.Info

var versionInfos modelarts.VersionInfo
if err = json.Unmarshal([]byte(modelarts.EngineVersions), &versionInfos); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["engine_versions"] = versionInfos.Version

var flavorInfos modelarts.Flavor
if err = json.Unmarshal([]byte(setting.FlavorInfos), &flavorInfos); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["flavor_infos"] = flavorInfos.Info

outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
ctx.Data["train_url"] = outputObsPath

configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
if err != nil {
ctx.ServerError("getConfigList failed:", err)
return
}

ctx.Data["config_list"] = configList.ParaConfigs

ctx.HTML(200, tplModelArtsTrainJobNew)
}

func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
ctx.Data["PageIsTrainJob"] = true
jobName := form.JobName
uuid := form.Attachment
description := form.Description
workServerNumber := form.WorkServerNumber
engineID := form.EngineID
bootFile := form.BootFile
flavorCode := form.Flavor
params := form.Params
poolID := form.PoolID
isSaveParam := form.IsSaveParam
repo := ctx.Repo.Repository
codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath
dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"

can, err := canUserCreateTrainJob(ctx.User.ID)
if err != nil {
ctx.ServerError("canUserCreateTrainJob", err)
return
}

if !can {
log.Error("the user can not create train-job")
ctx.RenderWithErr("the user can not create train-job", tplModelArtsTrainJobNew, &form)
return
}

//param check
if err := paramCheckCreateTrainJob(form); err != nil {
log.Error("paramCheckCreateTrainJob failed:(%v)", err)
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
return
}

if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{}); err != nil {
log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form)
return
}

//todo: upload code (send to file_server todo this work?)
if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil {
log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobNew, &form)
return
}

if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath); err != nil {
log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobNew, &form)
return
}

if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobNew, &form)
return
}

//todo: del local code?

var parameters models.Parameters
param := make([]models.Parameter, 0)
param = append(param, models.Parameter{
Label: modelarts.TrainUrl,
Value: outputObsPath,
}, models.Parameter{
Label: modelarts.DataUrl,
Value: dataPath,
})
if len(params) != 0 {
err := json.Unmarshal([]byte(params), &parameters)
if err != nil {
log.Error("Failed to Unmarshal params: %s (%v)", params, err)
ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobNew, &form)
return
}

for _, parameter := range parameters.Parameter {
if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl {
param = append(param, models.Parameter{
Label: parameter.Label,
Value: parameter.Value,
})
}
}
}

//save param config
if isSaveParam == "on" {
if form.ParameterTemplateName == "" {
log.Error("ParameterTemplateName is empty")
ctx.RenderWithErr("保存作业参数时,作业参数名称不能为空", tplModelArtsTrainJobNew, &form)
return
}

_, err := modelarts.CreateTrainJobConfig(models.CreateConfigParams{
ConfigName: form.ParameterTemplateName,
Description: form.PrameterDescription,
DataUrl: dataPath,
AppUrl: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
TrainUrl: outputObsPath,
Flavor: models.Flavor{
Code: flavorCode,
},
WorkServerNum: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Parameter: param,
})

if err != nil {
log.Error("Failed to CreateTrainJobConfig: %v", err)
ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobNew, &form)
return
}
}

req := &modelarts.GenerateTrainJobReq{
JobName: jobName,
DataUrl: dataPath,
Description: description,
CodeObsPath: codeObsPath,
BootFile: codeObsPath + bootFile,
TrainUrl: outputObsPath,
FlavorCode: flavorCode,
WorkServerNumber: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Uuid: uuid,
Parameters: param,
}

err = modelarts.GenerateTrainJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

// readDir reads the directory named by dirname and returns
// a list of directory entries sorted by filename.
func readDir(dirname string) ([]os.FileInfo, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
}

list, err := f.Readdir(100)
f.Close()
if err != nil {
//todo: can not upload empty folder
if err == io.EOF {
return nil, nil
}
return nil, err
}

//sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })
return list, nil
}

func uploadCodeToObs(codePath, jobName, parentDir string) error {
files, err := readDir(codePath)
if err != nil {
log.Error("readDir(%s) failed: %s", codePath, err.Error())
return err
}

for _, file := range files {
if file.IsDir() {
input := &obs.PutObjectInput{}
input.Bucket = setting.Bucket
input.Key = parentDir + file.Name() + "/"
_, err = storage.ObsCli.PutObject(input)
if err != nil {
log.Error("PutObject(%s) failed: %s", input.Key, err.Error())
return err
}

if err = uploadCodeToObs(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
log.Error("uploadCodeToObs(%s) failed: %s", file.Name(), err.Error())
return err
}
} else {
input := &obs.PutFileInput{}
input.Bucket = setting.Bucket
input.Key = setting.CodePathPrefix + jobName + "/code/" + parentDir + file.Name()
input.SourceFile = codePath + file.Name()
_, err = storage.ObsCli.PutFile(input)
if err != nil {
log.Error("PutFile(%s) failed: %s", input.SourceFile, err.Error())
return err
}
}
}

return nil
}

func obsMkdir(dir string) error {
input := &obs.PutObjectInput{}
input.Bucket = setting.Bucket
input.Key = dir
_, err := storage.ObsCli.PutObject(input)
if err != nil {
log.Error("PutObject(%s) failed: %s", input.Key, err.Error())
return err
}

return nil
}

func paramCheckCreateTrainJob(form auth.CreateModelArtsTrainJobForm) error {
if !strings.HasSuffix(form.BootFile, ".py") {
log.Error("the boot file(%s) must be a python file", form.BootFile)
return errors.New("启动文件必须是python文件")
}

if form.WorkServerNumber > 25 || form.WorkServerNumber < 1 {
log.Error("the WorkServerNumber(%d) must be in (1,25)", form.WorkServerNumber)
return errors.New("计算节点数必须在1-25之间")
}

return nil
}

func TrainJobShow(ctx *context.Context) {
ctx.Data["PageIsTrainJob"] = true

var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

attach, err := models.GetAttachmentByUUID(task.Uuid)
if err != nil {
log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
log.Error("GetJob(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

if result != nil {
result.CreateTime = time.Unix(int64(result.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05")
result.Status = modelarts.TransTrainJobStatus(result.IntStatus)
result.DatasetName = attach.Name
}

resultLogFile, resultLog, err := trainJobGetLog(jobID)
if err != nil {
log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

ctx.Data["log_file_name"] = resultLogFile.LogFileList[0]
ctx.Data["log"] = resultLog
ctx.Data["task"] = task
ctx.Data["jobID"] = jobID
ctx.Data["result"] = result
ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
}

func TrainJobGetLog(ctx *context.Context) {
ctx.Data["PageIsTrainJob"] = true

var jobID = ctx.Params(":jobid")
var logFileName = ctx.Query("file_name")
var baseLine = ctx.Query("base_line")
var order = ctx.Query("order")

if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
log.Error("order(%s) check failed", order)
ctx.HTML(http.StatusBadRequest, tplModelArtsTrainJobShow)
return
}

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

ctx.Data["log"] = result
//ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
}

func trainJobGetLog(jobID string) (*models.GetTrainJobLogFileNamesResult, *models.GetTrainJobLogResult, error) {
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
return nil, nil, err
}

resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error())
return nil, nil, err
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), "", resultLogFile.LogFileList[0], modelarts.OrderDesc, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
return nil, nil, err
}

return resultLogFile, result, err
}

func TrainJobDel(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

_, err = modelarts.DelTrainJob(jobID)
if err != nil {
log.Error("DelTrainJob(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

err = models.DeleteJob(task)
if err != nil {
ctx.ServerError("DeleteJob failed", err)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

func TrainJobStop(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

_, err = modelarts.StopTrainJob(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
log.Error("StopTrainJob(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

func canUserCreateTrainJob(uid int64) (bool, error) {
org, err := models.GetOrgByName(setting.AllowedOrg)
if err != nil {
log.Error("get allowed org failed: ", setting.AllowedOrg)
return false, err
}

return org.IsOrgMember(uid)
}

func TrainJobGetConfigList(ctx *context.Context) {
ctx.Data["PageIsTrainJob"] = true

var jobID = ctx.Params(":jobid")
var logFileName = ctx.Query("file_name")
var baseLine = ctx.Query("base_line")
var order = ctx.Query("order")

if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
log.Error("order(%s) check failed", order)
ctx.HTML(http.StatusBadRequest, tplModelArtsTrainJobShow)
return
}

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

ctx.Data["log"] = result
//ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
}

func getConfigList(perPage, page int, sortBy, order, searchContent, configType string) (*models.GetConfigListResult, error) {
var result models.GetConfigListResult

list, err := modelarts.GetConfigList(perPage, page, sortBy, order, searchContent, configType)
if err != nil {
log.Error("GetConfigList failed:", err)
return &result, err
}

for _, config := range list.ParaConfigs {
paraConfig, err := modelarts.GetParaConfig(config.ConfigName, configType)
if err != nil {
log.Error("GetParaConfig failed:", err)
return &result, err
}

config.Result = paraConfig
}

return list, nil
}

+ 25
- 0
routers/routes/routes.go View File

@@ -970,6 +970,31 @@ func RegisterRoutes(m *macaron.Macaron) {
})
m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate)

m.Group("/notebook", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookShow)
m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug)
m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel)
})
m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate)
})

m.Group("/train-job", func() {
m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow)
m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel)
m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog)
})
m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate)
m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList)
})
}, context.RepoRef())

m.Group("/blockchain", func() {


+ 6
- 4
templates/repo/header.tmpl View File

@@ -176,13 +176,13 @@
<label for="CloudBrain">{{$.i18n.Tr "repo.cloudbrain_platform_selection"}}</label>
<div class="field">
<div class="ui radio checkbox">
<input type="radio" name="CloudBrain" checked tabindex="0" class="hidden" value="0">
<input type="radio" name="CloudBrainSelect" checked tabindex="0" class="hidden" value="0">
<label>{{$.i18n.Tr "repo.cloudbrain1"}}</label>
</div>
</div>
<div class="field">
<div class="ui radio checkbox">
<input type="radio" name="CloudBrain" tabindex="0" class="hidden" value="1">
<input type="radio" name="CloudBrainSelect" tabindex="0" class="hidden" value="1">
<label>{{$.i18n.Tr "repo.cloudbrain2"}}</label>
</div>
</div>
@@ -209,14 +209,16 @@
$('.ui.radio.checkbox').checkbox();

var repolink = $(".cloudbrain_link").text()
console.log(repolink)
$(".ui.positive.right.icon.button").click(function(){
// 声明一个变量来接收以及获取单选框选择的情况
var checked_radio = $("input[type='radio']:checked").val()
var checked_radio = $("input[name='CloudBrainSelect']:checked").val()
console.log(checked_radio)

if(checked_radio=='0'){
window.location.href = repolink+'/cloudbrain'
}else if(checked_radio=='1'){
window.location.href = repolink+'/modelarts'
window.location.href = repolink+'/modelarts/notebook'
}else{
return;
}


+ 5
- 0
templates/repo/modelarts/index.tmpl View File

@@ -214,6 +214,11 @@
{{if .Permission.CanWrite $.UnitTypeCloudBrain}}
<a class="ui green button" href="{{.RepoLink}}/modelarts/create">{{.i18n.Tr "repo.cloudbrain.new"}}</a> {{end}}
</div>

<div class="column right aligned">
{{if .Permission.CanWrite $.UnitTypeCloudBrain}}
<a class="ui blue button" href="{{.RepoLink}}/modelarts/train-job/create">{{.i18n.Tr "repo.cloudbrain.new"}}</a> {{end}}
</div>
</div>

<p>使用鹏城云脑计算资源进行调试,云脑1提供CPU / GPU资源,云脑2提供Ascend NPU资源;调试使用的数据集也需要上传到对应的环境。</p>


+ 43
- 0
templates/repo/modelarts/navbar.tmpl View File

@@ -0,0 +1,43 @@
<style>
.dis {
margin-bottom: 20px;
}
.disabled {
cursor: pointer;
pointer-events: none;
}

.ui.segment.bottom.attached {
border: none;
}
.ui.secondary.vertical.pointing.menu{
border-right-width: 0px !important;
}

.vertical.menu .item {
border-right-color: white !important;
}

.vertical.menu .activate.item {
font-weight: 700;
}
</style>
<div class="three wide column">
<div class="ui grid">
<div class="sixteen wide column ui secondary sticky pointing tabular vertical menu">
<a class="{{if .PageIsNotebook}}active{{end}} item" href="{{.RepoLink}}/modelarts/notebook">
{{svg "octicon-repo" 16}} {{.i18n.Tr "repo.modelarts.notebook"}}
</a>
<a class="{{if .PageIsTrainJob}}active{{end}} item" href="{{.RepoLink}}/modelarts/train-job">
{{svg "octicon-inbox" 16}} {{.i18n.Tr "repo.modelarts.train_job"}}
</a>
<!--
<a class="{{if .PageIsParaManage}}active{{end}} item" href="{{.RepoLink}}/modelarts/train-job/para-manage">
{{svg "octicon-inbox" 16}} {{.i18n.Tr "repo.modelarts.train_job_para_admin"}}
</a>
-->
</div>
</div>
</div>

+ 221
- 0
templates/repo/modelarts/notebook/index.tmpl View File

@@ -0,0 +1,221 @@
{{template "base/head" .}}
<style>
#deletemodel {
width: 100%;
height: 100%;
}
.alert {
display: none;
position: fixed;
width: 100%;
z-index: 1001;
padding: 15px;
border: 1px solid transparent;
border-radius: 4px;
text-align: center;
font-weight: bold;
}

.alert-success {
color: #3c763d;
background-color: #dff0d8;
border-color: #d6e9c6;
}

.alert-danger {
color: #a94442;
background-color: #f2dede;
border-color: #ebccd1;
}
</style>
<div class="modelarts">
<div class="repository release dataset-list view container">
<div class="alert"></div>
{{template "repo/header" .}}
<div class="ui container">
<div class="ui grid">
{{template "repo/modelarts/navbar" .}}
<!-- 右侧 -->
<div class="ui thirteen wide column">
<div class="ui three column stackable grid">
<div class="column">
<h2>{{.i18n.Tr "repo.modelarts.notebook"}}</h2>
</div>
<div class="column">
</div>
<div class="column right aligned">
{{if .Permission.CanWrite $.UnitTypeCloudBrain}}
<a class="ui green button" href="{{.RepoLink}}/modelarts/notebook/create">{{.i18n.Tr "repo.cloudbrain.new"}}</a> {{end}}
</div>
</div>
<div class="ui divider"></div>
<div class="ui grid">
<div class="row">
<div class="ui sixteen wide column">
<div class="ui sixteen wide column">
<div class="ui two column stackable grid">
<div class="column">
</div>
<div class="column right aligned">
<div class="ui right dropdown type jump item">
<span class="text">
{{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
</span>
</div>
</div>
</div>
</div>

<!-- 任务展示 -->
<div class="dataset list">
{{range .Tasks}}
<div class="ui grid stackable item">
<div class="row">
<!-- 任务名 -->
<div class="four wide center column">
<a class="title" href="{{$.Link}}/{{.JobID}}">
<!-- <span class="fitted">{{svg "octicon-tasklist" 16}}</span> -->
<span class="fitted">{{.JobName}}</span>
</a>
</div>

<!--任务状态 -->
<div class="two wide center column job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</div>

<!-- 任务创建时间 -->
<div class="two wide center column">
<span class="ui text center">{{svg "octicon-flame" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}</span>
</div>

<!-- 查看 -->
<div class="two wide center column">
<span class="ui text clipboard">
<a class="title" href="{{$.Link}}/{{.JobID}}">
<span class="fitted">查看</span>
</a>
</span>
</div>

<!-- 删除任务 -->
<div class="two wide center column">
<div class="ui text center clipboard">
<form id="delForm-{{.JobID}}" action="{{if ne .Status "STOPPED"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
{{$.CsrfTokenHtml}}
<a class="fitted" onclick="assertDelete(this)" style="{{if ne .Status "STOPPED"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">删除</a>
</form>
</div>
</div>

<!-- 调试 -->
<div class="two wide center column">
<div class="ui text center clipboard">
<a class="title" onclick="stop(this)" href="{{if not .CanDebug}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/debug{{end}}" style="{{if not .CanDebug}}color:#CCCCCC{{end}}">
<span class="fitted">调试</span>
</a>
</div>
</div>

<!-- 停止 -->
<div class="two wide center column">
<div class="ui text center clipboard">
<form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post">
{{$.CsrfTokenHtml}}
<a class="fitted" onclick="document.getElementById('stopForm-{{.JobID}}').submit();" style="{{if ne .Status "RUNNING"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">停止</a>
</form>
</div>
</div>
</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>

</div>
</div>
</div>

</div>
</div>
</div>
</div>

<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> 删除任务
</div>

<div class="content">
<p>你确认删除该任务么?此任务一旦删除不可恢复。</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> 取消操作
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> 确定操作
</div>
</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
// 调试新开窗口
function stop(obj) {
if (obj.style.color != "rgb(204, 204, 204)") {
obj.target = '_blank'
} else {
return
}
}

// 删除时用户确认
function assertDelete(obj) {
if (obj.style.color == "rgb(204, 204, 204)") {
return
} else {
var delId = obj.parentNode.id
flag = 1;
$('.ui.basic.modal')
.modal({
onDeny: function() {
flag = false
},
onApprove: function() {
document.getElementById(delId).submit()
flag = true
},
onHidden: function() {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')
}
}

// 加载任务状态
$(document).ready(function() {
$(".job-status").each((index, job) => {
const jobID = job.dataset.jobid;
const repoPath = job.dataset.repopath;
if (job.textContent.trim() == 'STOPPED') {
return
}

$.get(`/api/v1/repos/${repoPath}/modelarts/notebook/${jobID}`, (data) => {
const jobID = data.JobID
const status = data.JobStatus
$('#' + jobID).text(status)
// console.log(data)
}).fail(function(err) {
console.log(err);
});
});
});
</script>

+ 82
- 0
templates/repo/modelarts/notebook/new.tmpl View File

@@ -0,0 +1,82 @@
{{template "base/head" .}}
<div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div>
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<h3 class="ui top attached header">
{{.i18n.Tr "repo.cloudbrain.new"}}
</h3>
<div class="ui attached segment">
<!-- <br> -->
<div class="inline required field">
<label>任务名称</label>
<input name="job_name" id="cloudbrain_job_name" placeholder="任务名称" value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
</div>

<div class="inline required field">
<label>数据集</label>
<select id="cloudbrain_dataset" class="ui search dropdown" placeholder="选择数据集" style='width:385px' name="attachment">
{{range .attachments}}
<option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>

{{end}}
</select>
</div>

<div class="inline required field">
<label>工作环境</label>
<input name="de" id="cloudbrain_de" value="{{.env}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>类型</label>
<input name="job_type" id="cloudbrain_job_type" value="{{.notebook_type}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>规格</label>
<input name="flavor" id="cloudbrain_flavor" value="{{.flavor}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>数据集存放路径</label>
<input name="dataset_path" id="cloudbrain_dataset_path" value="{{.dataset_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline field">
<label>描述</label>
<input name="description" id="cloudbrain_description" tabindex="3" autofocus maxlength="255">
</div>
<div class="inline field">
<label></label>
<button class="ui green button" onclick="showmask()">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('select.dropdown')
.dropdown();

$('.ui.green.button').click(function(){
$('.ui.page.dimmer').dimmer('show')
})
$(function() {
$("#cloudbrain_job_type").change(function() {
if ($(this).val() == 'BENCHMARK') {
$(".cloudbrain_benchmark").show();
} else {
$(".cloudbrain_benchmark").hide();
}
})
})
</script>

+ 122
- 0
templates/repo/modelarts/notebook/show.tmpl View File

@@ -0,0 +1,122 @@
{{template "base/head" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}

<h4 class="ui header" id="vertical-segment">
<a href="javascript:window.history.back();"><i class="arrow left icon"></i>返回</a>
</h4>
<div>
<div class="ui yellow segment">
{{with .task}}
<p>任务名称: {{.JobName}}</p>
{{end}}
</div>
<div class="ui green segment">
<p>任务结果:</p>
{{with .result}}
<table class="ui celled striped table">
<tbody>
<tr>
<td class="four wide"> 状态 </td>
<td> {{.Status}} </td>
</tr>
<tr>
<td> 开始时间 </td>
<td>{{.CreateTime}}</td>
</tr>
<tr>
<td> 最后更新时间 </td>
<td>{{.LatestUpdateTime}}</td>
</tr>
</tbody>
</table>
{{end}}
</div>
<div class="ui blue segment">
{{with .result}}
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> 配置信息 </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> 开发环境类型 </td>
<td>{{.Profile.DeType}}</td>
</tr>
<tr>
<td> 硬件类型 </td>
<td>{{.Profile.FlavorType}}</td>
</tr>
</tbody>
</table>

<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> 机器规格详情 </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> 机器规格 </td>
<td> {{.Flavor}} </td>
</tr>
<tr>
<td> 规格名称 </td>
<td>{{.FlavorDetails.Name}}</td>
</tr>
<tr>
<td> 规格销售状态 </td>
<td>{{.FlavorDetails.Status}}</td>
</tr>
<tr>
<td> 排队个数 </td>
<td>{{.FlavorDetails.QueuingNum}}</td>
</tr>
<tr>
<td> 排到队的剩余时间(秒) </td>
<td>{{.FlavorDetails.QueueLeftTime}}</td>
</tr>
<tr>
<td> 自动停止时间(秒) </td>
<td>{{.FlavorDetails.Duration}}</td>
</tr>
</tbody>
</table>

<table class="ui celled striped table" {{if eq .QueuingInfo.RemainTime 0}}hidden{{end}}>
<thead>
<tr> <th colspan="2"> 排队信息 </th> </tr>
</thead>
<tbody>
<tr>
<td> 实例状态 </td>
<td>{{.QueuingInfo.Status}}</td>
</tr>
<tr>
<td> 实例排队的开始时间 </td>
<td>{{.QueuingInfo.BeginTime}}</td>
</tr>
<tr>
<td> 排到队的剩余时间(秒) </td>
<td>{{.QueuingInfo.RemainTime}}</td>
</tr>
<tr>
<td> 实例排队的预计停止时间 </td>
<td>{{.QueuingInfo.EndTime}}</td>
</tr>
<tr>
<td> 实例在队列中的排位 </td>
<td>{{.QueuingInfo.Rank}}</td>
</tr>
</tbody>
</table>
{{end}}
</div>
</div>

</div>
</div>
</div>
{{template "base/footer" .}}

+ 239
- 0
templates/repo/modelarts/trainjob/edit_para.tmpl View File

@@ -0,0 +1,239 @@
{{template "base/head" .}}
<div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div>

<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
</h4>
<div class="ui attached segment">
<!-- equal width -->
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input name="job_name" id="trainjob_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.job_name}}" tabindex="3" autofocus required maxlength="255" readonly="">
</div>
<div class="field">
<label for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea id="description" name="description" rows="2"></textarea>
</div>
<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.algorithm_origin"}}</label>
<div class="ui top attached tabular menu">
<a class="item active" data-tab="frame">{{svg "octicon-repo" 16}}{{.i18n.Tr "repo.modelarts.train_job.frames"}}</a>
</div>
<div class="ui bottom attached tab active segment" data-tab="frame">
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label>
<div class="two fields">
<div class="field">
<select class="ui search dropdown" id="trainjob_engines" style='width:385px'>
{{range .engines}}
<option value="{{.Value}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="field">
<select class="ui search dropdown" id="trainjob_engine_versions" style='width:385px' name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>
</div>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
<input name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255">
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</div>
</div>
</div>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.dataset"}}</label>
<select class="ui search dropdown" id="trainjob_datasets" style='width:385px' name="attachment">
{{range .attachments}}
<option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
{{end}}
</select>
</div>
<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para"><i class="plus circle icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field">
{{range .para}}
<div class="two fields">
<div class="field">
<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}>
</div>
<div class="field">
<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>
</div>
<span>
<i class="trash icon">
</i>
</span>
</div>
{{end}}
</div>
</div>

<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.resource_setting"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
<select class="ui search dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
{{range .resource_pools}}
<option value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>

<div class="required grouped fields">
<label for="resource_type">{{.i18n.Tr "repo.modelarts.train_job.resource_type"}}</label>
{{range .benchmark_categories}}
<div class="field">
<div class="ui grid">
<div class="four wide column">
<div class="ui radio checkbox">
<input type="radio" name="resource_type" checked="" tabindex="0" class="hidden">
</div>
</div>
<div class="four wide column">train-private-1</div>
<div class="four wide column">{{svg "octicon-verified" 16}} 运行中</div>
<div class="four wide column"> CPU:192 核 2048GiB</div>
</div>
</div>
{{end}}
</div>

<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui search dropdown" id="trainjob-flavor" style='width:385px' name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.Code}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>
<input name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255">
</div>
<div class="inline field">
<button class="ui green button">
{{.i18n.Tr "repo.modelarts.train_job_para.connfirm"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('select.dropdown')
.dropdown();

$('.menu .item')
.tab();

// 参数增加、删除、修改、保存
function Add_parameter(){
value = '<div class="two fields">' +
'<div class="field">' +
'<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' +
'</div> ' +
'<div class="field"> ' +
'<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
'</div>'+
'<span>' +
'<i class="trash icon">' +
'</i>' +
'</span>' +
'</div>'
$(".dynamic.field").append(value)
}

$('#add_run_para').click(function(){
Add_parameter()
});

$(".dynamic.field").on("click",".trash.icon", function() {
var index = $(this).parent().parent().index()
$(this).parent().parent().remove()
});

$('.question.circle.icon').hover(function(){
$(this).popup('show')
});

$('.ui.deny.button').click(function(){
$('.ui.parameter.modal')
.modal('hide');
})

function validate(){
$('.ui.form')
.form({
on: 'blur',
inline:true,
fields: {
boot_file: {
identifier : 'boot_file',
rules: [
{
type: 'regExp[/.+\.py$/g]',
prompt : '启动文件必须为.py结尾'
}
]
},
work_server_number: {
identifier : 'work_server_number',
rules: [
{
type : 'integer[1..25]',
prompt : '计算节点需要在1-25之间,请您键入正确的值'
}
]
}
},
onSuccess: function(){
$('.ui.page.dimmer').dimmer('show')
},
onFailure: function(e){
return false;
}
})
}

function send_run_para(){
var run_parameters = []
var msg = {}
$(".dynamic.field .two.fields").each(function(){
var para_name = $(this).find('input[name=shipping_first-name]').val()
var para_value = $(this).find('input[name=shipping_last-name]').val()
run_parameters.push({"label": para_name, "value": para_value})
})
msg["parameter"] = run_parameters
msg = JSON.stringify(msg)
$('#store_run_para').val(msg)
}

$('.ui.green.button').click(function(e) {
send_run_para()
validate()
})

</script>

+ 225
- 0
templates/repo/modelarts/trainjob/index.tmpl View File

@@ -0,0 +1,225 @@
{{template "base/head" .}}
<style>
#deletemodel {
width: 100%;
height: 100%;
}
.alert {
display: none;
position: fixed;
width: 100%;
z-index: 1001;
padding: 15px;
border: 1px solid transparent;
border-radius: 4px;
text-align: center;
font-weight: bold;
}

.alert-success {
color: #3c763d;
background-color: #dff0d8;
border-color: #d6e9c6;
}

.alert-danger {
color: #a94442;
background-color: #f2dede;
border-color: #ebccd1;
}

/*

.disabled {
cursor: pointer;
pointer-events: none;
}

.ui.segment.bottom.attached {
border: none;
}
.ui.secondary.vertical.pointing.menu{
border-right-width: 0px !important;
}

.vertical.menu .item {
border-right-color: white !important;
}

.vertical.menu .activate.item {
font-weight: 700;
} */
</style>
<div class="modelarts">
<div class="repository release modelarts train_job view container">
<div class="alert"></div>
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<div class="ui grid">
{{template "repo/modelarts/navbar" .}}
<!-- 右侧 -->
<div class="ui thirteen wide column">
<div class="ui three column stackable grid">
<div class="column">
<h2>{{.i18n.Tr "repo.modelarts.train_job"}}</h2>
</div>
<div class="column">
</div>
<div class="column right aligned">
{{if .CanCreate}}
<a class="ui green button" href="{{.RepoLink}}/modelarts/train-job/create">{{.i18n.Tr "repo.modelarts.train_job.new"}}</a>
{{end}}
</div>
</div>
<div class="ui divider"></div>
<div class="ui grid">
<div class="row">
<div class="ui sixteen wide column">
<div class="ui sixteen wide column">
<div class="ui two column stackable grid">
<div class="column">
</div>
<div class="column right aligned">
<div class="ui right dropdown type jump item">
<span class="text">
{{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
</span>
</div>
</div>
</div>
</div>
<!-- 任务展示 -->
<div class="dataset list">
{{range .Tasks}}
<div class="ui grid stackable item">
<div class="row">
<!-- 任务名 -->
<div class="four wide center column">
<a class="title" href="{{$.Link}}/{{.JobID}}">
<!-- <span class="fitted">{{svg "octicon-tasklist" 16}}</span> -->
<span class="fitted">{{.JobName}}</span>
</a>
</div>
<!--任务状态 -->
<div class="five wide center column job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</div>

<!-- 运行时长 -->
<div class="three wide center column">
<span class="ui text center">{{svg "octicon-flame" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}</span>
</div>
<!-- 删除 -->
<div class="two wide center column">
<div class="ui text center clipboard">
<form id="delForm-{{.JobID}}" action="{{$.Link}}/{{.JobID}}/del" method="post">
{{$.CsrfTokenHtml}}
<a class="fitted" onclick="assertDelete(this)" style="font-size:16px; font-weight:bold">删除</a>
</form>
</div>
</div>
<!-- 停止 -->
<div class="two wide column">
<div class="ui text center clipboard">
<form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post">
{{$.CsrfTokenHtml}}
<a class="fitted" onclick="document.getElementById('stopForm-{{.JobID}}').submit();" style="{{if ne .Status "RUNNING"}}color:#CCCCCC{{end}};font-size:16px; font-weight:bold">停止</a>
</form>
</div>
</div>
</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> 删除任务
</div>

<div class="content">
<p>你确认删除该任务么?此任务一旦删除不可恢复。</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> 取消操作
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> 确定操作
</div>
</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$(".vertical.menu a").click(function(){
$(this).siblings().removeClass("activate")
$(this).addClass("activate")
})

// 删除时用户确认
function assertDelete(obj) {
if (obj.style.color == "rgb(204, 204, 204)") {
return
} else {
var delId = obj.parentNode.id
flag = 1;
$('.ui.basic.modal')
.modal({
onDeny: function() {
flag = false
},
onApprove: function() {
document.getElementById(delId).submit()
flag = true
},
onHidden: function() {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')
}
}

// 加载任务状态
$(document).ready(function() {
$(".job-status").each((index, job) => {
const jobID = job.dataset.jobid;
const repoPath = job.dataset.repopath;
if (job.textContent.trim() == 'STOPPED') {
return
}

$.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => {
const jobID = data.JobID
const status = data.JobStatus
$('#' + jobID).text(status)
// console.log(data)
}).fail(function(err) {
console.log(err);
});
});
});
</script>

+ 407
- 0
templates/repo/modelarts/trainjob/new.tmpl View File

@@ -0,0 +1,407 @@
{{template "base/head" .}}
<div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div>

<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
</h4>
<div class="ui attached segment">
<!-- equal width -->
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input name="job_name" id="trainjob_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
</div>
<!--<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.version"}}</label>
<span>第一版本</span>
</div>
-->
<div class="field">
<label for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea id="description" name="description" rows="2"></textarea>
</div>
<!-- <h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}</h4>
<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting"}}</label>
<span>
{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting_config"}}
<a class="item active parameter_config">{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting_config_link"}}</a>
</span>
</div> -->
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.algorithm_origin"}}</label>
<div class="ui top attached tabular menu">
<a class="item active" data-tab="frame">{{svg "octicon-repo" 16}}{{.i18n.Tr "repo.modelarts.train_job.frames"}}</a>
</div>
<div class="ui bottom attached tab active segment" data-tab="frame">
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label>
<div class="two fields">
<div class="field">
<select class="ui search dropdown" id="trainjob_engines" style='width:385px'>
{{range .engines}}
<option value="{{.Value}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="field">
<select class="ui search dropdown" id="trainjob_engine_versions" style='width:385px' name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>
</div>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
<input name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255">
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</div>
</div>
</div>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.dataset"}}</label>
<select class="ui search dropdown" id="trainjob_datasets" style='width:385px' name="attachment">
{{range .attachments}}
<option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
{{end}}
</select>
</div>
<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para"><i class="plus circle icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field">
</div>
</div>

<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.resource_setting"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
<select class="ui search dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
{{range .resource_pools}}
<option value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>

<div class="required grouped fields">
<label for="resource_type">{{.i18n.Tr "repo.modelarts.train_job.resource_type"}}</label>
<div class="field">
<div class="ui grid">
<div class="column">
<div class="ui radio checkbox">
<input type="radio" name="resource_type" checked="" tabindex="0">
</div>
</div>
<div class="three wide column">train-private-1</div>
<div class="three wide column">{{svg "octicon-verified" 16}} 运行中</div>
<div class="three wide column"> CPU:192 核 2048GiB</div>
</div>
</div>
</div>

<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui search dropdown" id="trainjob-flavor" style='width:385px' name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.Code}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>
<input name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255">
</div>
<!--
<div class="inline field">
<div class="ui save checkbox">
<input name="is_save_para" type="checkbox">
<label>{{.i18n.Tr "repo.modelarts.train_job.query_whether_save_parameter"}}
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.save_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</label>
</div>
</div>
<div class="disabled field" id="save_para">
<div class="field">
<label>{{.i18n.Tr "repo.modelarts.train_job.job_parameter_name"}}</label>
<input name="parameter_template_name" id="parameter_template_name" tabindex="3" autofocus maxlength="255">
</div>
<div class="field">
<label for="parameter_description">{{.i18n.Tr "repo.modelarts.train_job.parameter_description"}}</label>
<textarea id="parameter_description" name="parameter_description" rows="2"></textarea>
</div>
</div>
-->
<div class="inline field">
<button class="ui create_train_job green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
<!-- 模态框 -->
<div class="ui parameter modal" style="height: 70%;">
<div class="header">
{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting"}}
</div>
<div class="content" style="height:100%">
<div class="ui grid" style="height: 100%; margin-top: auto; margin-bottom: auto">
<!-- 左侧列表 -->
<div class="five wide column" style="height:100%">
<div class="ui vertical menu" style="height:100%;">
<div class="item">
<div class="ui input">
<input type="text" placeholder="搜索...">
</div>
</div>
<div class="item" style="height:85%; overflow:auto;">
<div class="menu">
{{range .config_list}}
<a class="item">{{.ConfigName}}</a>
</div>
</div>
</div>
</div>
<!-- 右侧详情 -->
<div class="eleven wide column content" style="height:100%">
<div class="ui green segment" style="height:100%; overflow:auto;">
<p>配置详情:</p>
<table class="ui celled striped table">
<tbody>
<tr>
<td class="four wide"> {{$.i18n.Tr "repo.modelarts.train_job.job_name"}} </td>
<td>{{.Result.ConfigName}}</td>
</tr>
<tr>
<td> {{$.i18n.Tr "repo.modelarts.train_job.description"}} </td>
<td></td>
</tr>
<tr>
<td> {{$.i18n.Tr "repo.modelarts.train_job.dataset"}} </td>
<td><input id="store_uuid" type="hidden" name="get_uuid"></td>
</tr>
<tr>
<td> {{$.i18n.Tr "repo.modelarts.train_job.start_file"}} </td>
<td></td>
</tr>
<tr>
<td> {{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}} </td>
<td></td>
</tr>
<tr>
<td> {{$.i18n.Tr "repo.modelarts.train_job.resource_pool"}} </td>
<td><input id="store_pool_id" type="hidden" name="get_pool_id"></td>
</tr>
<tr>
<td> {{$.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}} </td>
<td></td>
</tr>
</tbody>
{{end}}
</table>
</div>
</div>
</div>
</div>
<div class="actions">
<button class="ui parameter green button">
{{.i18n.Tr "repo.confirm_choice"}}
</button>
<div class="ui deny button">
{{.i18n.Tr "repo.cloudbrain.cancel"}}
</div>
</div>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('select.dropdown')
.dropdown();

$('.menu .item')
.tab();

// 参数增加、删除、修改、保存
function Add_parameter(i){
value = '<div class="two fields" id= "para'+ i +'">' +
'<div class="field">' +
'<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' +
'</div> ' +
'<div class="field"> ' +
'<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
'</div>'+
'<span>' +
'<i class="trash icon">' +
'</i>' +
'</span>' +
'</div>'
$(".dynamic.field").append(value)
}

$('#add_run_para').click(function(){
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
});

$(".dynamic.field").on("click",".trash.icon", function() {
var index = $(this).parent().parent().index()
$(this).parent().parent().remove()
var len = $(".dynamic.field .two.fields").length
$(".dynamic.field .two.fields").each(function(){
var cur_index = $(this).index()
$(this).attr('id', 'para' + cur_index)
})
});

$('.ui.parameter.green.button').click(function(){
var parameters = [];
$('table tr').each(function() {
$(this).find('td:eq(1)').each(function(){
parameters.push($(this).text());
})
$(this).find('input').each(function(){
parameters.push($(this).text())
})
});
console.log(parameters)
$('.ui.parameter.modal')
.modal('hide');
for(var i = 2; i < parameters.length; i++){
switch(i) {
// 数据集uuid待完成
// case (2):
// console.log(1)
// break;
// $("#trainjob_datasets").val(parameters[i]);
// console.log($("#trainjob_datasets").val())
case (3):
$("input[name='boot_file']").val(parameters[i]);
break;
case (4):
var para = parameters[i].split(" ")
for(var j = 0; j < para.length; j++){
var para_name = para[j].split('=')[0]
var para_value = para[j].split('=')[1]
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
var pid = 'para' + len
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_first-name]").val(para_name)
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_last-name]").val(para_value)
}
break;
// 数据集pool_id待完成
// case (5):
// $("select[name='pool_id']").val(parameters[i]);
// break;
case (6):
$("input[name='work_server_number']").val(parameters[i]);
break;
}
}
})

$('.ui.save.checkbox').click(function(){
$(this).checkbox({
onChange: function(){
if ($('.ui.save.checkbox').checkbox('is checked')){
$('#save_para').removeClass("disabled")
}else{
$('#save_para').addClass("disabled")
}
}
});
})

$('.question.circle.icon').hover(function(){
$(this).popup('show')
});

$(".item.active.parameter_config").click(function(){
$('.ui.parameter.modal')
.modal('setting', 'closable', false)
.modal('show');
})

$('.ui.deny.button').click(function(){
$('.ui.parameter.modal')
.modal('hide');
})

function validate(){
$('.ui.form')
.form({
on: 'blur',
inline:true,
fields: {
boot_file: {
identifier : 'boot_file',
rules: [
{
type: 'regExp[/.+\.py$/g]',
prompt : '启动文件必须为.py结尾'
}
]
},
work_server_number: {
identifier : 'work_server_number',
rules: [
{
type : 'integer[1..25]',
prompt : '计算节点需要在1-25之间,请您键入正确的值'
}
]
}
},
onSuccess: function(){
$('.ui.page.dimmer').dimmer('show')
},
onFailure: function(e){
return false;
}
})
}

function send_run_para(){
var run_parameters = []
var msg = {}
$(".dynamic.field .two.fields").each(function(){
var para_name = $(this).find('input[name=shipping_first-name]').val()
var para_value = $(this).find('input[name=shipping_last-name]').val()
run_parameters.push({"label": para_name, "value": para_value})
})
msg["parameter"] = run_parameters
msg = JSON.stringify(msg)
$('#store_run_para').val(msg)
}

$('.ui.create_train_job.green.button').click(function(e) {
send_run_para()
validate()
})
</script>

+ 154
- 0
templates/repo/modelarts/trainjob/para_manage.tmpl View File

@@ -0,0 +1,154 @@
{{template "base/head" .}}
<div class="modelarts">
<div class="repository release modelarts train_job view container">
{{template "repo/header" .}}
<div class="ui container">
<div class="ui grid">
{{template "repo/modelarts/navbar" .}}
<!-- 右侧 -->
<div class="ui thirteen wide column">
<div class="ui column stackable grid">
<div class="column">
<h2>{{.i18n.Tr "repo.modelarts.train_job_para_admin"}}</h2>
</div>
</div>
<div class="ui divider"></div>
<div class="ui grid">
<div class="row">
<div class="ui sixteen wide column">
<div class="ui sixteen wide column">
<div class="ui two column stackable grid">
<div class="column">
</div>
<div class="column right aligned">
<div class="ui right dropdown type jump item">
<span class="text">
{{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
</span>
</div>
</div>
</div>
</div>
<!-- 任务展示 -->
<div class="dataset list">
{{range .Tasks}}
<div class="ui grid stackable item">
<div class="row">
<!-- 任务名 -->
<div class="five wide column">
<a class="title" href="{{$.Link}}/{{.JobID}}">
<span class="fitted">{{svg "octicon-tasklist" 16}}</span>
<span class="fitted">{{.JobName}}</span>
</a>
</div>
<!-- 引擎类型-->
<div class="four wide column job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</div>

<!-- 创建时间 -->
<div class="three wide column">
<span class="ui text center">{{svg "octicon-clock" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}</span>
</div>

<!-- 编辑 -->
<div class="two wide column">
<a class="title" href="{{$.Link}}/{{.JobID}}/edit">
<span class="fitted">编辑</span>
</a>
</div>
<!-- 删除 -->
<div class="two wide column">
<div class="ui text center clipboard">
<form id="delForm-{{.JobID}}" action="{{$.Link}}/{{.JobID}}/del" method="post">
{{$.CsrfTokenHtml}}
<a class="fitted" onclick="assertDelete(this)" style="font-size:16px; font-weight:bold">删除</a>
</form>
</div>
</div>
</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> 删除任务
</div>

<div class="content">
<p>你确认删除该任务么?此任务一旦删除不可恢复。</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> 取消操作
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> 确定操作
</div>
</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
// 删除时用户确认
function assertDelete(obj) {
if (obj.style.color == "rgb(204, 204, 204)") {
return
} else {
var delId = obj.parentNode.id
flag = 1;
$('.ui.basic.modal')
.modal({
onDeny: function() {
flag = false
},
onApprove: function() {
document.getElementById(delId).submit()
flag = true
},
onHidden: function() {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')
}
}

// 加载任务状态
$(document).ready(function() {
$(".job-status").each((index, job) => {
const jobID = job.dataset.jobid;
const repoPath = job.dataset.repopath;
if (job.textContent.trim() == 'STOPPED') {
return
}

$.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => {
const jobID = data.JobID
const status = data.JobStatus
$('#' + jobID).text(status)
// console.log(data)
}).fail(function(err) {
console.log(err);
});
});
});
</script>

+ 200
- 0
templates/repo/modelarts/trainjob/show.tmpl View File

@@ -0,0 +1,200 @@
{{template "base/head" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}
<h4 class="ui top attached header">
<div class="ui two column grid">
<div class="column">
{{$.i18n.Tr "repo.modelarts.version_manage"}}
</div>
<div class="column right aligned">
<a href="javascript:window.history.back();">{{svg "octicon-reply" 16}}{{$.i18n.Tr "repo.modelarts.back"}}</a>
</div>
</div>
</h4>

<div class="ui attached segment">
<div class="ui style accordion">
<div class="title active">
<i class="dropdown icon"></i>
{{$.i18n.Tr "repo.modelarts.train_job.version"}}
</div>
<div class="content active">
<div class="ui container">
<div class="ui top attached tabular menu">
<a class="item active" data-tab="configs">配置信息</a>
<a class="item logs" data-tab="logs">{{$.i18n.Tr "repo.modelarts.log"}}</a>
<!-- <a class="item" data-tab="resources">资源占用情况</a> -->
</div>
<div class="ui bottom attached tab segment active" data-tab="configs">
<div>
<div class="ui yellow segment">
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> {{.i18n.Tr "repo.modelarts.train_job.basic_info"}} </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.job_name"}} </td>
<td>{{.result.JobName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.job_status"}} </td>
<td>{{.result.Status}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.version"}} </td>
<td>{{.result.VersionName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.start_time"}} </td>
<td>{{.result.CreateTime}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.dura_time"}} </td>
<td>{{.result.Duration}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.description"}} </td>
<td>{{.result.Description}}</td>
</tr>
</tbody>
</table>
</div>
<div class="ui green segment">
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> {{.i18n.Tr "repo.modelarts.train_job.parameter_setting_info"}} </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.AI_driver"}} </td>
<td>{{.result.EngineName}} | {{.result.EngineVersion}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.start_file"}}</td>
<td>{{.result.BootFileUrl}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.dataset"}} </td>
<td>{{.result.DatasetName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.run_parameter"}} </td>
<td>{{.result.Parameter}}</td>
</tr>
</tbody>
</table>
</div>
<div class="ui blue segment">
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> {{.i18n.Tr "repo.modelarts.train_job.resource_setting_info"}} </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.resource_pool"}} </td>
<td>{{.result.PoolName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</td>
<td>{{.result.WorkServerNum}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.NAS_mount_path"}} </td>
<td>{{.result.NasMountPath}}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="ui bottom attached tab segment" data-tab="logs">
<div class="ui message" style="display: none;">
<div class="header">
</div>
</div>
<div class="ui top attached segment" style="background: #f0f0f0;">
<div class="center aligned">
<label>{{$.i18n.Tr "repo.modelarts.log"}}:</label>
<span class="fitted file_name">{{.log_file_name}}</span>
<input type="hidden" name="file_name" value={{.log_file_name}}>
<input type="hidden" name="start_line" value={{.log.StartLine}}>
<input type="hidden" name="end_line" value={{.log.EndLine}}>
</div>
</div>
<div class="ui attached segment log" style="height: 300px !important; overflow: auto;">
<pre>{{.log.Content}}</pre>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('.menu .item').tab()
$('.ui.style.accordion').accordion();

var userName
var repoPath
var jobID
$(document).ready(function(){
var url = window.location.href;
var urlArr = url.split('/')
userName = urlArr.slice(-5)[0]
repoPath = urlArr.slice(-4)[0]
jobID = urlArr.slice(-1)[0]
})
$(".log").scroll(function () {
var scrollTop = $(this)[0].scrollTop; // 滚动距离
var scrollHeight = $(this)[0].scrollHeight; // 文档高度
var divHeight = $(this).height(); // 可视区高度
var file_name = $('input[name=file_name]').val()

if(parseInt(scrollTop) + divHeight + 29 == scrollHeight){
var end_line = $('input[name=end_line]').val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?file_name=${file_name}&base_line=${end_line}&order=desc`, (data) => {
if (data.lines == 0){
$('.header').text('您已翻阅至日志底部')
$('.message').css('display', 'block')
setTimeout(function(){
$('.message').css('display', 'none')
}, 1000)
}else{
$('input[name=end_line]').val(data.EndLine)
$('.log').append('<pre>' + data.Content)
}
}).fail(function(err) {
console.log(err);
});
}
if(scrollTop == 0){
var start_line = $('input[name=start_line]').val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?file_name=${file_name}&base_line=${start_line}&order=asc`, (data) => {
if (data.lines == 0){
$('.header').text('您已翻阅至日志顶部')
$('.message').css('display', 'block')
setTimeout(function(){
$('.message').css('display', 'none')
}, 1000)
}else{
$('input[name=start_line]').val(data.StartLine) //如果变动就改变所对应的值
$(".log").prepend('<pre>' + data.Content)
}
}).fail(function(err) {
console.log(err);
});
}
})
</script>

+ 0
- 0
web_src/less/themes/theme-arc-green.less View File


Loading…
Cancel
Save