diff --git a/Makefile b/Makefile index f35d5fa2a..8e7f46fea 100755 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/))) -WEBPACK_SOURCES := $(shell find web_src/js web_src/less -type f) +WEBPACK_SOURCES := $(shell find web_src/js web_src/less web_src/vuepages -type f) WEBPACK_CONFIGS := webpack.config.js WEBPACK_CONFIGS_TMP := webpack_temp.config.js WEBPACK_CONFIGS_PRO := webpack_pro.config.js diff --git a/models/action.go b/models/action.go index d0a763e08..4b6f1dbad 100755 --- a/models/action.go +++ b/models/action.go @@ -65,7 +65,6 @@ const ( ActionCreateImage //36 ActionImageRecommend //37 ActionChangeUserAvatar //38 - ) // Action represents user operation type and other information to diff --git a/models/badge.go b/models/badge.go new file mode 100644 index 000000000..fcfbdc27f --- /dev/null +++ b/models/badge.go @@ -0,0 +1,181 @@ +package models + +import ( + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "path/filepath" + "strings" + "xorm.io/builder" +) + +type Badge struct { + ID int64 `xorm:"pk autoincr"` + Name string + LightedIcon string `xorm:"varchar(2048)"` + GreyedIcon string `xorm:"varchar(2048)"` + Url string `xorm:"varchar(2048)"` + CategoryId int64 + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + DeletedAt timeutil.TimeStamp `xorm:"deleted"` +} + +func (m *Badge) ToUserShow() *Badge4UserShow { + return &Badge4UserShow{ + Name: m.Name, + LightedIcon: GetIconOuterLink(m.LightedIcon), + GreyedIcon: GetIconOuterLink(m.GreyedIcon), + Url: m.Url, + } +} + +type GetBadgeOpts struct { + BadgeType BadgeType + CategoryId int64 + ListOpts ListOptions +} + +type BadgeAndCategory struct { + Badge Badge `xorm:"extends"` + Category BadgeCategory `xorm:"extends"` +} + +func (*BadgeAndCategory) TableName() string { + return "badge" +} + +func (m *BadgeAndCategory) ToShow() *Badge4AdminShow { + return &Badge4AdminShow{ + ID: m.Badge.ID, + Name: m.Badge.Name, + LightedIcon: GetIconOuterLink(m.Badge.LightedIcon), + GreyedIcon: GetIconOuterLink(m.Badge.GreyedIcon), + Url: m.Badge.Url, + CategoryName: m.Category.Name, + CategoryId: m.Category.ID, + CreatedUnix: m.Badge.CreatedUnix, + UpdatedUnix: m.Badge.UpdatedUnix, + } +} + +type Badge4AdminShow struct { + ID int64 + Name string + LightedIcon string + GreyedIcon string + Url string + CategoryName string + CategoryId int64 + CreatedUnix timeutil.TimeStamp + UpdatedUnix timeutil.TimeStamp +} + +func (m Badge4AdminShow) ToDTO() Badge { + return Badge{ + Name: m.Name, + LightedIcon: m.LightedIcon, + GreyedIcon: m.GreyedIcon, + Url: m.Url, + CategoryId: m.CategoryId, + } +} + +type BadgeOperateReq struct { + ID int64 + Name string + LightedIcon string + GreyedIcon string + Url string + CategoryId int64 +} + +func (m BadgeOperateReq) ToDTO() Badge { + return Badge{ + Name: m.Name, + LightedIcon: m.LightedIcon, + GreyedIcon: m.GreyedIcon, + Url: m.Url, + CategoryId: m.CategoryId, + } +} + +type Badge4UserShow struct { + Name string + LightedIcon string + GreyedIcon string + Url string +} + +type BadgeShowWithStatus struct { + Badge *Badge4UserShow + IsLighted bool +} + +type UserAllBadgeInCategory struct { + CategoryName string + CategoryId int64 + LightedNum int + Badges []*BadgeShowWithStatus +} + +func GetBadgeList(opts GetBadgeOpts) (int64, []*BadgeAndCategory, error) { + if opts.ListOpts.Page <= 0 { + opts.ListOpts.Page = 1 + } + var cond = builder.NewCond() + if opts.BadgeType > 0 { + cond = cond.And(builder.Eq{"badge_category.type": opts.BadgeType}) + } + if opts.CategoryId > 0 { + cond = cond.And(builder.Eq{"badge_category.id": opts.CategoryId}) + } + n, err := x.Join("INNER", "badge_category", "badge_category.ID = badge.category_id").Where(cond).Count(&BadgeAndCategory{}) + if err != nil { + return 0, nil, err + } + r := make([]*BadgeAndCategory, 0) + if err = x.Join("INNER", "badge_category", "badge_category.ID = badge.category_id").Where(cond).OrderBy("badge.created_unix desc").Limit(opts.ListOpts.PageSize, (opts.ListOpts.Page-1)*opts.ListOpts.PageSize).Find(&r); err != nil { + return 0, nil, err + } + return n, r, nil +} + +func AddBadge(m Badge) (int64, error) { + return x.Insert(&m) +} + +func UpdateBadgeById(id int64, param Badge) (int64, error) { + return x.ID(id).Update(¶m) +} + +func DelBadge(id int64) (int64, error) { + return x.ID(id).Delete(&Badge{}) +} + +func GetBadgeById(id int64) (*Badge, error) { + m := &Badge{} + has, err := x.ID(id).Get(m) + if err != nil { + return nil, err + } else if !has { + return nil, &ErrRecordNotExist{} + } + return m, nil +} + +func GetBadgeByCategoryId(categoryId int64) ([]*Badge, error) { + r := make([]*Badge, 0) + err := x.Where("category_id = ?", categoryId).Find(&r) + return r, err +} + +func GetCustomIconByHash(hash string) string { + if len(hash) == 0 { + return "" + } + return filepath.Join(setting.IconUploadPath, hash) +} + +func GetIconOuterLink(hash string) string { + return strings.TrimRight(setting.AppSubURL, "/") + "/show/icon/" + hash +} diff --git a/models/badge_category.go b/models/badge_category.go new file mode 100644 index 000000000..069fb6b10 --- /dev/null +++ b/models/badge_category.go @@ -0,0 +1,94 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type BadgeType int + +const ( + CustomizeBadge = iota + 1 + SystemBadge +) + +type BadgeCategory struct { + ID int64 `xorm:"pk autoincr"` + Name string + Position int64 + Type BadgeType + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + DeletedAt timeutil.TimeStamp `xorm:"deleted"` +} + +func (m *BadgeCategory) ToShow() *BadgeCategory4Show { + return &BadgeCategory4Show{ + ID: m.ID, + Name: m.Name, + Position: m.Position, + Type: m.Type, + CreatedUnix: m.CreatedUnix, + } +} + +type BadgeCategory4Show struct { + ID int64 `xorm:"pk autoincr"` + Name string + Position int64 + Type BadgeType + CreatedUnix timeutil.TimeStamp `xorm:"created"` +} + +func (m BadgeCategory4Show) ToDTO() BadgeCategory { + return BadgeCategory{ + ID: m.ID, + Name: m.Name, + Position: m.Position, + Type: m.Type, + CreatedUnix: m.CreatedUnix, + } +} + +func GetBadgeCategoryListPaging(opts ListOptions) (int64, []*BadgeCategory, error) { + n, err := x.Count(&BadgeCategory{}) + if err != nil { + return 0, nil, err + } + if opts.Page <= 0 { + opts.Page = 1 + } + r := make([]*BadgeCategory, 0) + if err := x.OrderBy("position asc,created_unix desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&r); err != nil { + return 0, nil, err + } + return n, r, nil +} + +func GetBadgeCategoryList() ([]*BadgeCategory, error) { + r := make([]*BadgeCategory, 0) + if err := x.OrderBy("position asc,created_unix desc").Find(&r); err != nil { + return nil, err + } + return r, nil +} + +func AddBadgeCategory(m BadgeCategory) (int64, error) { + return x.Insert(&m) +} + +func UpdateBadgeCategoryById(id int64, param BadgeCategory) (int64, error) { + return x.ID(id).Update(¶m) +} + +func DelBadgeCategory(id int64) (int64, error) { + return x.ID(id).Delete(&BadgeCategory{}) +} + +func GetBadgeCategoryById(id int64) (*BadgeCategory, error) { + m := &BadgeCategory{} + has, err := x.ID(id).Get(m) + if err != nil { + return nil, err + } else if !has { + return nil, ErrRecordNotExist{} + } + return m, nil +} diff --git a/models/badge_user.go b/models/badge_user.go new file mode 100644 index 000000000..9b556bc0e --- /dev/null +++ b/models/badge_user.go @@ -0,0 +1,159 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" +) + +const ( + ActionAddBadgeUser = 1 + ActionDelBadgeUser = 2 +) + +type BadgeUser struct { + ID int64 `xorm:"pk autoincr"` + UserId int64 `xorm:"unique(user_badge)"` + BadgeId int64 `xorm:"unique(user_badge) index"` + CreatedUnix timeutil.TimeStamp `xorm:"created index"` +} + +type BadgeUserLog struct { + ID int64 `xorm:"pk autoincr"` + UserId int64 `xorm:"index"` + BadgeId int64 `xorm:"index"` + Action int + CreatedUnix timeutil.TimeStamp `xorm:"created index"` +} + +type BadgeUserDetail struct { + BadgeUser BadgeUser `xorm:"extends"` + User User `xorm:"extends"` +} + +func (*BadgeUserDetail) TableName() string { + return "badge_user" +} + +func (m *BadgeUserDetail) ToShow() *BadgeUser4SHow { + return &BadgeUser4SHow{ + ID: m.BadgeUser.ID, + UserId: m.BadgeUser.UserId, + Name: m.User.Name, + Avatar: m.User.RelAvatarLink(), + Email: m.User.Email, + CreatedUnix: m.BadgeUser.CreatedUnix, + } +} + +type BadgeUser4SHow struct { + ID int64 + UserId int64 + Name string + Avatar string + Email string + CreatedUnix timeutil.TimeStamp +} + +type AddBadgeUsersReq struct { + BadgeId int64 + Users string +} +type DelBadgeUserReq struct { + ID int64 +} + +type GetUserBadgesOpts struct { + CategoryId int64 + ListOptions +} + +func AddBadgeUser(m BadgeUser) (int64, error) { + sess := x.NewSession() + defer sess.Close() + sess.Begin() + n, err := sess.Insert(&m) + if err != nil || n == 0 { + return 0, err + } + _, err = sess.Insert(&BadgeUserLog{ + UserId: m.UserId, + BadgeId: m.BadgeId, + Action: ActionAddBadgeUser, + }) + if err != nil { + sess.Rollback() + return 0, err + } + return n, sess.Commit() +} + +func DelBadgeUser(id int64) (int64, error) { + m := BadgeUser{} + has, err := x.ID(id).Get(&m) + if err != nil { + return 0, err + } + if !has { + return 0, ErrRecordNotExist{} + } + sess := x.NewSession() + defer sess.Close() + sess.Begin() + n, err := x.ID(m.ID).Delete(&BadgeUser{}) + if err != nil || n == 0 { + return 0, err + } + _, err = sess.Insert(&BadgeUserLog{ + UserId: m.UserId, + BadgeId: m.BadgeId, + Action: ActionDelBadgeUser, + }) + if err != nil { + sess.Rollback() + return 0, err + } + return n, sess.Commit() +} + +func GetBadgeUsers(badgeId int64, opts ListOptions) (int64, []BadgeUserDetail, error) { + n, err := x.Join("LEFT", "public.user", "public.user.ID = badge_user.user_id").Where("badge_user.badge_id = ?", badgeId).Count(&BadgeUserDetail{}) + if err != nil { + return 0, nil, err + } + if opts.Page <= 0 { + opts.Page = 1 + } + m := make([]BadgeUserDetail, 0) + err = x.Join("LEFT", "public.user", "public.user.ID = badge_user.user_id").Where("badge_user.badge_id = ?", badgeId).OrderBy("badge_user.id desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&m) + if err != nil { + return 0, nil, err + } + return n, m, nil +} + +func GetUserBadgesPaging(userId int64, opts GetUserBadgesOpts) ([]*Badge, error) { + cond := builder.NewCond() + cond = cond.And(builder.Eq{"badge_user.user_id": userId}) + if opts.CategoryId > 0 { + cond = cond.And(builder.Eq{"badge.category_id": opts.CategoryId}) + } + + r := make([]*Badge, 0) + err := x.Join("INNER", "badge_user", "badge_user.badge_id = badge.id").Where(cond).OrderBy("badge_user.id desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&r) + return r, err +} +func CountUserBadges(userId int64) (int64, error) { + return x.Where("user_id = ?", userId).Count(&BadgeUser{}) +} + +func GetUserBadges(userId, categoryId int64) ([]*Badge, error) { + cond := builder.NewCond() + cond = cond.And(builder.Eq{"badge_user.user_id": userId}) + if categoryId > 0 { + cond = cond.And(builder.Eq{"badge.category_id": categoryId}) + } + + r := make([]*Badge, 0) + err := x.Join("INNER", "badge_user", "badge_user.badge_id = badge.id").Where(cond).OrderBy("badge_user.created_unix desc").Find(&r) + return r, err +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index d088a9127..3f58284fd 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -53,8 +53,11 @@ const ( JobFailed CloudbrainStatus = "FAILED" JobRunning CloudbrainStatus = "RUNNING" + ModelSafetyTesting CloudbrainStatus = "TESTING" + JobTypeDebug JobType = "DEBUG" JobTypeBenchmark JobType = "BENCHMARK" + JobTypeModelSafety JobType = "MODELSAFETY" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" @@ -172,26 +175,26 @@ type Cloudbrain struct { ImageID string //grampus image_id AiCenter string //grampus ai center: center_id+center_name - TrainUrl string //输出模型的obs路径 - BranchName string //分支名称 - Parameters string //传给modelarts的param参数 - BootFile string //启动文件 - DataUrl string //数据集的obs路径 - LogUrl string //日志输出的obs路径 - PreVersionId int64 //父版本的版本id - FlavorCode string //modelarts上的规格id - Description string `xorm:"varchar(256)"` //描述 - WorkServerNumber int //节点数 - FlavorName string //规格名称 - EngineName string //引擎名称 - TotalVersionCount int //任务的所有版本数量，包括删除的 - LabelName string //标签名称 - ModelName string //模型名称 - ModelVersion string //模型版本 - CkptName string //权重文件名称 - PreTrainModelUrl string //预训练模型地址 - ResultUrl string //推理结果的obs路径 - + TrainUrl string //输出模型的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id + Description string `xorm:"varchar(256)"` //描述 + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 + LabelName string //标签名称 + ModelName string //模型名称 + ModelVersion string //模型版本 + CkptName string //权重文件名称 + PreTrainModelUrl string //预训练模型地址 + ResultUrl string //推理结果的obs路径 + ResultJson string `xorm:"varchar(4000)"` User *User `xorm:"-"` Repo *Repository `xorm:"-"` BenchmarkType string `xorm:"-"` //算法评测，模型评测 @@ -1490,6 +1493,8 @@ type GrampusTasks struct { ReplicaNum int `json:"replicaNum"` Datasets []GrampusDataset `json:"datasets"` Models []GrampusDataset `json:"models"` + Code GrampusDataset `json:"code"` + BootFile string `json:"bootFile"` } type GrampusDataset struct { @@ -2003,6 +2008,13 @@ func GetStoppedJobWithNoStartTimeEndTime() ([]*Cloudbrain, error) { return cloudbrains, x.SQL("select * from cloudbrain where status in (?,?,?,?,?,?,?) and (start_time is null or end_time is null) limit 100", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded).Find(&cloudbrains) } +func GetModelSafetyTestTask() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0) + sess := x.Where("job_type = ?", string(JobTypeModelSafety)) + err := sess.Find(&cloudbrains) + return cloudbrains, err +} + func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) return int(count), err @@ -2017,7 +2029,12 @@ func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) { } func GetBenchmarkCountByUserID(userID int64) (int, error) { - count, err := x.In("status", JobWaiting, JobRunning).And("(job_type = ? or job_type = ? or job_type = ?) and user_id = ? and type = ?", string(JobTypeBenchmark), string(JobTypeBrainScore), string(JobTypeSnn4imagenet), userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + count, err := x.In("status", JobWaiting, JobRunning).And("(job_type = ? or job_type = ? or job_type = ?) and user_id = ? and type = ?", string(JobTypeBenchmark), string(JobTypeModelSafety), string(JobTypeBrainScore), string(JobTypeSnn4imagenet), userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + return int(count), err +} + +func GetModelSafetyCountByUserID(userID int64) (int, error) { + count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain)) return int(count), err } @@ -2279,9 +2296,9 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.Limit(opts.PageSize, start) } - sess.OrderBy("cloudbrain.created_unix DESC") + // sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/models/models.go b/models/models.go index 7665c3b0f..b2b7a6472 100755 --- a/models/models.go +++ b/models/models.go @@ -161,6 +161,10 @@ func init() { new(CloudbrainSpec), new(CloudbrainTemp), new(DatasetReference), + new(BadgeCategory), + new(Badge), + new(BadgeUser), + new(BadgeUserLog), ) tablesStatistic = append(tablesStatistic, diff --git a/models/repo.go b/models/repo.go index 6e5ee3ff6..2c4fda39b 100755 --- a/models/repo.go +++ b/models/repo.go @@ -679,7 +679,7 @@ func (repo *Repository) getAssignees(e Engine) (_ []*User, err error) { userIDs[i] = accesses[i].UserID } - if err = e.In("id", userIDs).Find(&users); err != nil { + if err = e.In("id", userIDs).OrderBy("name asc").Find(&users); err != nil { return nil, err } } diff --git a/models/user.go b/models/user.go index f40eb699f..b21858e37 100755 --- a/models/user.go +++ b/models/user.go @@ -2184,3 +2184,24 @@ func GetBlockChainUnSuccessUsers() ([]*User, error) { Find(&users) return users, err } + +//GetUserIdsByUserNames Get userIDs in batches through username paging, this method will ignore errors +func GetUserIdsByUserNames(names []string) []int64 { + pageSize := 200 + length := len(names) + r := make([]int64, 0, length) + for i := 0; i < length; i = i + pageSize { + if length-i < 200 { + pageSize = length - i + } + userNameTemp := names[i : i+pageSize] + t := make([]int64, 0, length) + err := x.Table("public.user").Cols("id").In("name", userNameTemp).Find(&t) + if err != nil { + continue + } + r = append(r, t...) + + } + return r +} diff --git a/modules/aisafety/resty.go b/modules/aisafety/resty.go new file mode 100644 index 000000000..be6468529 --- /dev/null +++ b/modules/aisafety/resty.go @@ -0,0 +1,249 @@ +package aisafety + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "sort" + "strings" + + "code.gitea.io/gitea/modules/log" + + "github.com/go-resty/resty/v2" +) + +var ( + restyClient *resty.Client + HOST string + KEY string +) + +type TaskReq struct { + UnionId string //评测任务ID，唯一标识，由后端生成UUID + EvalName string //评测任务名称 + EvalContent string //评测任务描述 + TLPath string // + Indicators []string //评测指标，由GetAlgorithmList接口返回的指标列表中的title属性值 + CDPath string + CDName string //对抗数据集名称 + BDPath string + BDName string //原数据集名称 +} + +type ReturnMsg struct { + Code string `json:"code"` + Msg string `json:"msg"` + Data ReturnData `json:"data"` + Times int64 `json:"times"` +} + +type ReturnData struct { + ID int `json:"id"` + No string `json:"no"` + StandardJson string `json:"standardJson"` + Code int `json:"code"` + Msg string `json:"msg"` + Status int `json:"status"` +} + +const ( + APPID = "1" + LogPageSize = 500 + LogPageTokenExpired = "5m" + pageSize = 15 + Success = "S000" +) + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + } + return restyClient +} + +func checkSetting() { + if len(HOST) != 0 && len(KEY) != 0 { + return + } + _ = loginCloudbrain() +} + +func loginCloudbrain() error { + HOST = "http://221.122.70.196:8081/atp-api" + KEY = "1" + return nil +} + +func createSign(params map[string]interface{}, signKey string) string { + var sceneList []string + for k := range params { + sceneList = append(sceneList, k) + } + sort.Strings(sceneList) + re := "" + for _, key := range sceneList { + if params[key] != nil { + re += key + "=" + fmt.Sprint(params[key]) + "&" + } + } + re += "key=" + signKey + log.Info("sign key:" + re) + h := md5.New() + h.Write([]byte(re)) + return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) +} + +func getParams(req TaskReq) (map[string]interface{}, string) { + params := make(map[string]interface{}) + reStr := "" + + params["unionId"] = req.UnionId + reStr += "unionId=" + req.UnionId + params["evalName"] = req.EvalName + reStr += "&evalName=" + req.EvalName + params["evalContent"] = req.EvalContent + reStr += "&evalContent=" + url.QueryEscape(req.EvalContent) + params["TLPath"] = req.TLPath + reStr += "&TLPath=" + url.QueryEscape(req.TLPath) + + params["CDName"] = req.CDName + reStr += "&CDName=" + url.QueryEscape(req.CDName) + + params["BDName"] = req.BDName + reStr += "&BDName=" + url.QueryEscape(req.BDName) + + if req.CDPath != "" { + params["CDPath"] = req.CDPath + reStr += "&CDPath=" + url.QueryEscape(req.CDPath) + } + if req.BDPath != "" { + params["BDPath"] = req.BDPath + reStr += "&BDPath=" + url.QueryEscape(req.BDPath) + } + indicators := "" + if len(req.Indicators) > 0 { + for _, tmp := range req.Indicators { + indicators += tmp + "|" + } + } + if len(indicators) > 0 { + indicators = indicators[0 : len(indicators)-1] + } + + params["Indicators"] = indicators + log.Info("indicators=" + indicators) + reStr += "&Indicators=" + url.QueryEscape(indicators) + + return params, reStr +} + +func CreateSafetyTask(req TaskReq, jsonstr string) (string, error) { + checkSetting() + client := getRestyClient() + + //reqPara, _ := json.Marshal(body) + //log.Warn("job req:", jsonstr) + + params, urlQuerys := getParams(req) + + bodyMap := make(map[string]interface{}) + //reJsonMap := make(map[string]interface{}) + bodyMap["resJson"] = jsonstr + //bodyMap["externalEvalParam"] = reJsonMap + + //reqPara, _ := json.Marshal(bodyMap) + //log.Warn("job req json:", string(reqPara)) + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetHeader("appId", APPID). + SetHeader("sign", createSign(params, KEY)). + //SetAuthToken(TOKEN). + SetBody(bodyMap). + Post(HOST + "/v1/external/eval-standard/create?" + urlQuerys) + log.Info("url=" + HOST + "/v1/external/eval-standard/create?" + urlQuerys) + + responseStr := string(res.Body()) + log.Info("CreateSafetyTask responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) + + if err != nil { + return "", fmt.Errorf("resty create job: %s", err) + } else { + log.Info("result string=" + " res code=" + fmt.Sprint(res.StatusCode())) + } + + reMap := make(map[string]interface{}) + + err = json.Unmarshal(res.Body(), &reMap) + if err == nil && reMap["code"] == "0" { + dataMap := (reMap["data"]).(map[string]interface{}) + return fmt.Sprint(dataMap["serialNo"]), nil + } + return "", nil +} + +func GetAlgorithmList() (map[string]interface{}, error) { + checkSetting() + client := getRestyClient() + params := make(map[string]interface{}) + + jsonResult := make(map[string]interface{}) + sign := createSign(params, KEY) + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetHeader("appId", APPID). + SetHeader("sign", sign). + Get(HOST + "/v1/external/eval-standard/algorithmList") + + log.Info("url=" + HOST + "/v1/external/eval-standard/algorithmList" + " sign=" + sign + " appId=" + APPID) + + jsonerr := json.Unmarshal(res.Body(), &jsonResult) + if jsonerr == nil { + log.Info("jsonResult code=" + fmt.Sprint(jsonResult["msg"])) + } + responseStr := string(res.Body()) + log.Info("GetAlgorithmList responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) + + if err != nil { + log.Info("error =" + err.Error()) + return nil, fmt.Errorf("resty GetJob: %v", err) + } else { + reMap := make(map[string]interface{}) + err = json.Unmarshal(res.Body(), &reMap) + if err == nil && reMap["code"] == "0" { + return reMap, nil + } else { + return nil, fmt.Errorf("get error,code not 0") + } + } + +} + +func GetTaskStatus(jobID string) (*ReturnMsg, error) { + checkSetting() + client := getRestyClient() + var reMap ReturnMsg + params := make(map[string]interface{}) + params["serialNo"] = jobID + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetHeader("appId", APPID). + SetHeader("sign", createSign(params, KEY)). + SetResult(&reMap). + Get(HOST + "/v1/external/eval-standard/query?serialNo=" + jobID) + + log.Info("url=" + HOST + "/v1/external/eval-standard/query?serialNo=" + jobID) + responseStr := string(res.Body()) + log.Info("GetTaskStatus responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) + + if err != nil { + log.Info("error =" + err.Error()) + return nil, fmt.Errorf("Get task status error: %v", err) + } else { + return &reMap, nil + } +} diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 4e527b6bf..e57bd8d7e 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -20,19 +20,19 @@ import ( const ( //Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` - CommandBenchmark = `cd /benchmark && bash run_bk.sh >/model/benchmark-log.txt` - CodeMountPath = "/code" - DataSetMountPath = "/dataset" - ModelMountPath = "/model" + CommandBenchmark = `cd /benchmark && bash run_bk.sh >/model/benchmark-log.txt` + CodeMountPath = "/code" + DataSetMountPath = "/dataset" + ModelMountPath = "/model" PretrainModelMountPath = "/pretrainmodel" - LogFile = "log.txt" - BenchMarkMountPath = "/benchmark" - BenchMarkResourceID = 1 - Snn4imagenetMountPath = "/snn4imagenet" - BrainScoreMountPath = "/brainscore" - TaskInfoName = "/taskInfo" - Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` - BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` + LogFile = "log.txt" + BenchMarkMountPath = "/benchmark" + BenchMarkResourceID = 1 + Snn4imagenetMountPath = "/snn4imagenet" + BrainScoreMountPath = "/brainscore" + TaskInfoName = "/taskInfo" + Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` SubTaskName = "task1" @@ -405,7 +405,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { } func IsBenchmarkJob(jobType string) bool { - return string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType + return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType } func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 38ac37852..958595960 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -123,6 +123,17 @@ func registerHandleUnDecompressAttachment() { }) } +func registerHandleModelSafetyTask() { + RegisterTaskFatal("handle_modelsafety_task", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 5m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.TimerHandleModelSafetyTestTask() + return nil + }) +} + func registerHandleBlockChainUnSuccessUsers() { RegisterTaskFatal("handle_blockchain_unsuccess_users", &BaseConfig{ Enabled: true, @@ -291,5 +302,7 @@ func initBasicTasks() { //registerRewardPeriodTask() registerCloudbrainPointDeductTask() + + registerHandleModelSafetyTask() registerHandleCloudbrainDurationStatistic() } diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index d72a7b10e..7bacb46d3 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -1,6 +1,7 @@ package grampus import ( + "code.gitea.io/gitea/modules/cloudbrain" "encoding/json" "strings" @@ -73,6 +74,7 @@ type GenerateTrainJobReq struct { PreTrainModelPath string PreTrainModelUrl string Spec *models.Specification + CodeName string } func getEndPoint() string { @@ -102,6 +104,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error centerID, centerName := getCentersParamter(ctx, req) var datasetGrampus, modelGrampus []models.GrampusDataset + var codeGrampus models.GrampusDataset if ProcessorTypeNPU == req.ProcessType { datasetGrampus = getDatasetGrampus(req.DatasetInfos) if len(req.ModelName) != 0 { @@ -114,6 +117,12 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error }, } } + codeGrampus = models.GrampusDataset{ + Name: req.CodeName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ObjectKey: req.CodeObsPath + cloudbrain.DefaultBranchName + ".zip", + } } jobResult, err := createJob(models.CreateGrampusJobRequest{ @@ -130,6 +139,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error ReplicaNum: 1, Datasets: datasetGrampus, Models: modelGrampus, + Code: codeGrampus, + BootFile: req.BootFile, }, }, }) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 6b3d1f128..06521993e 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,7 +1,6 @@ package modelarts import ( - "code.gitea.io/gitea/modules/modelarts_cd" "encoding/json" "errors" "fmt" @@ -9,6 +8,8 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/modelarts_cd" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -70,8 +71,8 @@ const ( var ( poolInfos *models.PoolInfos TrainFlavorInfos *Flavor - SpecialPools *models.SpecialPools - MultiNodeConfig *MultiNodes + SpecialPools *models.SpecialPools + MultiNodeConfig *MultiNodes ) type GenerateTrainJobReq struct { @@ -141,6 +142,7 @@ type GenerateInferenceJobReq struct { ResultUrl string Spec *models.Specification DatasetName string + JobType string } type VersionInfo struct { @@ -173,12 +175,12 @@ type ResourcePool struct { } `json:"resource_pool"` } -type MultiNodes struct{ +type MultiNodes struct { Info []OrgMultiNode `json:"multinode"` } -type OrgMultiNode struct{ +type OrgMultiNode struct { Org string `json:"org"` - Node []int `json:"node"` + Node []int `json:"node"` } // type Parameter struct { @@ -709,7 +711,7 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e Status: models.TempJobStatus, Type: models.TypeCloudBrainTwo, JobName: req.JobName, - JobType: string(models.JobTypeInference), + JobType: req.JobType, }) if err != nil { log.Error("InsertCloudbrainTemp failed: %v", err.Error()) @@ -732,7 +734,7 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e JobID: jobID, JobName: req.JobName, DisplayJobName: req.DisplayJobName, - JobType: string(models.JobTypeInference), + JobType: req.JobType, Type: models.TypeCloudBrainTwo, VersionID: jobResult.VersionID, VersionName: jobResult.VersionName, @@ -769,7 +771,15 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) return err } - notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateInferenceTask) + if req.JobType == string(models.JobTypeModelSafety) { + task, err := models.GetCloudbrainByJobID(jobID) + if err == nil { + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, fmt.Sprint(task.ID), req.DisplayJobName, models.ActionCreateBenchMarkTask) + } + } else { + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateInferenceTask) + } + return nil } @@ -798,8 +808,8 @@ func InitSpecialPool() { } } -func InitMultiNode(){ - if MultiNodeConfig ==nil && setting.ModelArtsMultiNode!=""{ +func InitMultiNode() { + if MultiNodeConfig == nil && setting.ModelArtsMultiNode != "" { json.Unmarshal([]byte(setting.ModelArtsMultiNode), &MultiNodeConfig) } diff --git a/modules/repository/create.go b/modules/repository/create.go index 0844c43c3..049fe3abf 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "strings" + "text/template" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" @@ -34,7 +35,7 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *m Name: opts.Name, Alias: opts.Alias, LowerName: strings.ToLower(opts.Name), - Description: opts.Description, + Description: template.HTMLEscapeString(opts.Description), OriginalURL: opts.OriginalURL, OriginalServiceType: opts.GitServiceType, IsPrivate: opts.IsPrivate, diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d6e4824ef..2f468c850 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -394,7 +394,7 @@ var ( DefaultGitTreesPerPage int DefaultMaxBlobSize int64 }{ - EnableSwagger: true, + EnableSwagger: false, SwaggerURL: "", MaxResponseItems: 50, DefaultPagingNum: 30, @@ -622,6 +622,13 @@ var ( DeductTaskRange time.Duration DeductTaskRangeForFirst time.Duration + //badge config + BadgeIconMaxFileSize int64 + BadgeIconMaxWidth int + BadgeIconMaxHeight int + BadgeIconDefaultSize uint + IconUploadPath string + //wechat auto reply config UserNameOfWechatReply string RepoNameOfWechatReply string @@ -707,6 +714,22 @@ var ( NPU_MINDSPORE_IMAGE_ID int NPU_TENSORFLOW_IMAGE_ID int }{} + + ModelSafetyTest = struct { + NPUBaseDataSetName string + NPUBaseDataSetUUID string + NPUCombatDataSetName string + NPUCombatDataSetUUID string + + GPUBaseDataSetName string + GPUBaseDataSetUUID string + GPUCombatDataSetName string + GPUCombatDataSetUUID string + }{} + + ModelApp = struct { + DesensitizationUrl string + }{} ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1515,6 +1538,14 @@ func NewContext() { CloudBrainPayInterval = sec.Key("CLOUDBRAIN_PAY_INTERVAL").MustDuration(60 * time.Minute) DeductTaskRange = sec.Key("DEDUCT_TASK_RANGE").MustDuration(30 * time.Minute) DeductTaskRangeForFirst = sec.Key("DEDUCT_TASK_RANGE_FOR_FIRST").MustDuration(3 * time.Hour) + + sec = Cfg.Section("icons") + BadgeIconMaxFileSize = sec.Key("BADGE_ICON_MAX_FILE_SIZE").MustInt64(1048576) + BadgeIconMaxWidth = sec.Key("BADGE_ICON_MAX_WIDTH").MustInt(4096) + BadgeIconMaxHeight = sec.Key("BADGE_ICON_MAX_HEIGHT").MustInt(3072) + BadgeIconDefaultSize = sec.Key("BADGE_ICON_DEFAULT_SIZE").MustUint(200) + IconUploadPath = sec.Key("ICON_UPLOAD_PATH").MustString(path.Join(AppDataPath, "icons")) + SetRadarMapConfig() sec = Cfg.Section("warn_mail") @@ -1527,6 +1558,20 @@ func NewContext() { getGrampusConfig() getModelartsCDConfig() getModelConvertConfig() + getModelSafetyConfig() + getModelAppConfig() +} + +func getModelSafetyConfig() { + sec := Cfg.Section("model_safety_test") + ModelSafetyTest.GPUBaseDataSetName = sec.Key("GPUBaseDataSetName").MustString("") + ModelSafetyTest.GPUBaseDataSetUUID = sec.Key("GPUBaseDataSetUUID").MustString("") + ModelSafetyTest.GPUCombatDataSetName = sec.Key("GPUCombatDataSetName").MustString("") + ModelSafetyTest.GPUCombatDataSetUUID = sec.Key("GPUCombatDataSetUUID").MustString("") + ModelSafetyTest.NPUBaseDataSetName = sec.Key("NPUBaseDataSetName").MustString("") + ModelSafetyTest.NPUBaseDataSetUUID = sec.Key("NPUBaseDataSetUUID").MustString("") + ModelSafetyTest.NPUCombatDataSetName = sec.Key("NPUCombatDataSetName").MustString("") + ModelSafetyTest.NPUCombatDataSetUUID = sec.Key("NPUCombatDataSetUUID").MustString("") } func getModelConvertConfig() { @@ -1548,6 +1593,12 @@ func getModelConvertConfig() { ModelConvert.NPU_TENSORFLOW_IMAGE_ID = sec.Key("NPU_TENSORFLOW_IMAGE_ID").MustInt(35) } +func getModelAppConfig() { + sec := Cfg.Section("model_app") + ModelApp.DesensitizationUrl = sec.Key("desensitization_url").MustString("") + +} + func getModelartsCDConfig() { sec := Cfg.Section("modelarts-cd") diff --git a/modules/timer/timer.go b/modules/timer/timer.go deleted file mode 100755 index a0220380c..000000000 --- a/modules/timer/timer.go +++ /dev/null @@ -1,30 +0,0 @@ -package timer - -import ( - "github.com/robfig/cron/v3" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/routers/repo" -) - -func LaunchCronJob() { - log.Trace("Run cron job") - - c := cron.New() - - spec := "*/10 * * * *" - c.AddFunc(spec, repo.HandleUnDecompressAttachment) - - specCheckBlockChainUserSuccess := "*/10 * * * *" - c.AddFunc(specCheckBlockChainUserSuccess, repo.HandleBlockChainUnSuccessUsers) - - specCheckRepoBlockChainSuccess := "*/1 * * * *" - c.AddFunc(specCheckRepoBlockChainSuccess, repo.HandleBlockChainUnSuccessRepos) - - specCheckUnTransformedPRs := "*/1 * * * *" - c.AddFunc(specCheckUnTransformedPRs, repo.HandleBlockChainMergedPulls) - - specCheckBlockChainCommitSuccess := "*/3 * * * *" - c.AddFunc(specCheckBlockChainCommitSuccess, repo.HandleBlockChainUnSuccessCommits) - c.Start() -} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 19d604062..324930544 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -525,6 +525,7 @@ datasets = Datasets activity = Public Activity followers = Followers starred = Starred Repositories +badge = Achievement Badge following = Following follow = Follow unfollow = Unfollow @@ -3210,11 +3211,11 @@ wrong_specification=You cannot use this specification, please choose another ite resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). -train_dataset_path_rule = The dataset location is stored in the environment variable data_url, the pre-trained model is storaged in the environment ckpt_url, and the output path is stored in the environment variable train_url. -infer_dataset_path_rule = The dataset location is stored in the environment variable data_url, and the output path is stored in the environment variable result_url. +train_dataset_path_rule = The dataset location is stored in the run parameter data_url, the pre-trained model is storaged in the run parameter ckpt_url, and the output path is stored in the run parameter train_url. +infer_dataset_path_rule = The dataset location is stored in the run parameter data_url, and the output path is stored in the run parameter result_url. view_sample = View sample -inference_output_path_rule = The inference output path is stored in the environment variable result_url. -model_file_path_rule=The model file location is stored in the environment variable ckpt_url +inference_output_path_rule = The inference output path is stored in the run parameter result_url. +model_file_path_rule=The model file location is stored in the run parameter ckpt_url model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] delete_task = Delete task @@ -3234,6 +3235,7 @@ point_hr = Point/hr DEBUG = DEBUG SNN4IMAGENET = BENCHMARK BRAINSCORE = BENCHMARK +MODELSAFETY = BENCHMARK TRAIN = TRAIN INFERENCE = INFERENCE BENCHMARK = BENCHMARK @@ -3247,8 +3249,8 @@ Stopped_success_update_status_fail=Succeed in stopping th job, but failed to upd load_code_failed=Fail to load code, please check if the right branch is selected. error.dataset_select = dataset select error：the count exceed the limit or has same name -new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online -new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online +new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online +new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online new_infer_gpu_tooltips = The dataset is stored in %s, the model file is stored in %s, please store the inference output in %s for subsequent downloads. [points] @@ -3260,3 +3262,20 @@ hours = Hours expected_time = , expected to be available for points_acquisition_instructions = Points Acquisition Instructions insufficient_points_balance = Insufficient points balance + +[modelsafety] +model_security_evaluation = Model Security Evaluation +base_data_set = Base Data Set +combat_data_set = Combat Data Set +evaluation_indicators = Evaluation Indicators +evaluation_result = Evaluation Result +no_data = No Data +untargetted = Untargetted +targetted = targetted +new_model_security_evaluation_tips = Model Security Evaluation just used for image classification + +[model_app] +get_file_fail= Can not get the image content, please try again later. +content_type_unsupported=Allowed image type is jpg, jpeg or png. +process_image_fail=Fail to process image, please try again later. + diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 9fa7f5595..fd5596a2e 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -530,6 +530,7 @@ datasets=数据集 activity=公开活动 followers=关注者 starred=已点赞 +badge=成就徽章 following=关注中 follow=关注 unfollow=取消关注 @@ -3228,11 +3229,11 @@ card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 -train_dataset_path_rule = 数据集位置存储在环境变量data_url中，预训练模型存放在环境变量ckpt_url中，训练输出路径存储在环境变量train_url中。 -infer_dataset_path_rule = 数据集位置存储在环境变量data_url中，推理输出路径存储在环境变量result_url中。 +train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。 +infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。 view_sample = 查看样例 -inference_output_path_rule = 推理输出路径存储在环境变量result_url中。 -model_file_path_rule = 模型文件位置存储在环境变量ckpt_url中。 +inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。 +model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。 model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] delete_task = 删除任务 @@ -3252,6 +3253,7 @@ point_hr = 积分/时 DEBUG = 调试任务 SNN4IMAGENET = 评测任务 BRAINSCORE = 评测任务 +MODELSAFETY = 评测任务 TRAIN = 训练任务 INFERENCE = 推理任务 BENCHMARK = 评测任务 @@ -3266,9 +3268,9 @@ load_code_failed=代码加载失败，请确认选择了正确的分支。 error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 -new_train_gpu_tooltips =训练脚本存储在%s中，数据集存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 -new_train_npu_tooltips =训练脚本存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 -new_infer_gpu_tooltips = 数据集存储在%s中，模型文件存储在%s中，推理输出请存储在%s中以供后续下载。 +new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 +new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 +new_infer_gpu_tooltips = 数据集存储在 %s 中，模型文件存储在 %s 中，推理输出请存储在 %s 中以供后续下载。 [points] points = 积分 @@ -3280,4 +3282,19 @@ expected_time = ，预计可用 points_acquisition_instructions = 积分获取说明 insufficient_points_balance = 积分余额不足 +[modelsafety] +model_security_evaluation = 模型安全评测 +base_data_set = 原始数据集 +combat_data_set = 对抗数据集 +evaluation_indicators = 评测指标 +evaluation_result = 评测结果 +no_data = 无数据 +untargetted = 非定向 +targetted = 定向 +new_model_security_evaluation_tips = 模型安全评测只适用于图像分类 + +[model_app] +get_file_fail= 获取上传文件失败，请稍后再试。 +content_type_unsupported=请上传jpg、jpeg或png图片。 +process_image_fail=图片处理失败，请稍后再试。 diff --git a/public/home/home.js b/public/home/home.js index c1849b3e3..853c3ef23 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -81,8 +81,11 @@ var isZh = true; if(lang != null && lang.nodeValue =="en-US" ){ isZh=false; } - document.onreadystatechange = function () { + if(document.readyState != "complete"){ + return; + } + console.log("Start to open WebSocket." + document.readyState); queryRecommendData(); var output = document.getElementById("newmessage"); @@ -102,10 +105,9 @@ document.onreadystatechange = function () { var html = ""; if (data != null){ if(messageQueue.length > maxSize){ - delete messageQueue[0]; - }else{ - messageQueue.push(data); + messageQueue.splice(0,1); } + messageQueue.push(data); var currentTime = new Date().getTime(); for(var i = 0; i < messageQueue.length; i++){ var record = messageQueue[i]; diff --git a/public/img/origin.png b/public/img/origin.png new file mode 100644 index 000000000..26eccf7ce Binary files /dev/null and b/public/img/origin.png differ diff --git a/public/img/tuomin.png b/public/img/tuomin.png new file mode 100644 index 000000000..116662b2f Binary files /dev/null and b/public/img/tuomin.png differ diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 96db935fe..d03c00ae6 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -52,7 +52,7 @@ func CloudBrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/admin/resources.go b/routers/admin/resources.go index 8a8c55f86..1d3870a14 100644 --- a/routers/admin/resources.go +++ b/routers/admin/resources.go @@ -221,6 +221,7 @@ func GetResourceSceneList(ctx *context.Context) { func AddResourceScene(ctx *context.Context, req models.ResourceSceneReq) { req.CreatorId = ctx.User.ID + req.ExclusiveOrg = strings.ReplaceAll(req.ExclusiveOrg, " ", "") err := resource.AddResourceScene(req) if err != nil { log.Error("AddResourceScene error. %v", err) @@ -238,6 +239,7 @@ func UpdateResourceScene(ctx *context.Context, req models.ResourceSceneReq) { var err error switch action { case "edit": + req.ExclusiveOrg = strings.ReplaceAll(req.ExclusiveOrg, " ", "") err = resource.UpdateResourceScene(req) case "delete": err = resource.DeleteResourceScene(id) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 439190121..ba46ab58c 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "sort" + "strconv" "strings" "time" @@ -67,49 +68,62 @@ func GetCloudbrainTask(ctx *context.APIContext) { log.Error("GetCloudbrainByID failed:", err) return } - jobResult, err := cloudbrain.GetJob(job.JobID) - if err != nil { - ctx.NotFound(err) - log.Error("GetJob failed:", err) - return - } - result, _ := models.ConvertToJobResultPayload(jobResult.Payload) - if err != nil { - ctx.NotFound(err) - log.Error("ConvertToJobResultPayload failed:", err) - return - } - oldStatus := job.Status - job.Status = result.JobStatus.State - taskRoles := result.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { - job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - job.ContainerID = taskRes.TaskStatuses[0].ContainerID - job.Status = taskRes.TaskStatuses[0].State - } - - if result.JobStatus.State != string(models.JobWaiting) { - models.ParseAndSetDurationFromCloudBrainOne(result, job) - if oldStatus != job.Status { - notification.NotifyChangeCloudbrainStatus(job, oldStatus) + if job.JobType == string(models.JobTypeModelSafety) { + routerRepo.GetAiSafetyTaskByJob(job) + job, err = models.GetCloudbrainByID(ID) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "ID": ID, + "JobName": job.JobName, + "JobStatus": job.Status, + "SubState": "", + "CreatedTime": job.CreatedUnix.Format("2006-01-02 15:04:05"), + "CompletedTime": job.UpdatedUnix.Format("2006-01-02 15:04:05"), + "JobDuration": job.TrainJobDuration, + }) + } else { + jobResult, err := cloudbrain.GetJob(job.JobID) + if err != nil { + ctx.NotFound(err) + log.Error("GetJob failed:", err) + return } - err = models.UpdateJob(job) + result, _ := models.ConvertToJobResultPayload(jobResult.Payload) if err != nil { - log.Error("UpdateJob failed:", err) + ctx.NotFound(err) + log.Error("ConvertToJobResultPayload failed:", err) + return + } + oldStatus := job.Status + job.Status = result.JobStatus.State + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + job.ContainerID = taskRes.TaskStatuses[0].ContainerID + job.Status = taskRes.TaskStatuses[0].State } - } - ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": result.Config.JobName, - "JobStatus": result.JobStatus.State, - "SubState": result.JobStatus.SubState, - "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"), - "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"), - "JobDuration": job.TrainJobDuration, - }) + if result.JobStatus.State != string(models.JobWaiting) { + models.ParseAndSetDurationFromCloudBrainOne(result, job) + if oldStatus != job.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "ID": ID, + "JobName": result.Config.JobName, + "JobStatus": result.JobStatus.State, + "SubState": result.JobStatus.SubState, + "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"), + "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"), + "JobDuration": job.TrainJobDuration, + }) + } } func GetCloudBrainInferenceJob(ctx *context.APIContext) { @@ -370,6 +384,96 @@ func CloudbrainForModelConvertGetLog(ctx *context.Context) { ctx.JSON(http.StatusOK, result) } +func ModelSafetyGetLog(ctx *context.APIContext) { + ID := ctx.Params(":id") + job, err := models.GetCloudbrainByID(ID) + if err != nil { + log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"]) + ctx.ServerError(err.Error(), err) + return + } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainTwo { + //TrainJobForModelConvertGetLog(ctx) + var baseLine = ctx.Query("base_line") + var order = ctx.Query("order") + var lines = ctx.Query("lines") + lines_int, err := strconv.Atoi(lines) + if err != nil { + log.Error("change lines(%d) string to int failed", lines_int) + } + + if order != modelarts.OrderDesc && order != modelarts.OrderAsc { + log.Error("order(%s) check failed", order) + ctx.JSON(http.StatusBadRequest, map[string]interface{}{ + "err_msg": "order check failed", + }) + return + } + resultLogFile, err := modelarts.GetTrainJobLogFileNames(job.JobID, strconv.FormatInt(job.VersionID, 10)) + if err != nil { + log.Error("GetTrainJobLogFileNames(%s) failed:%v", job.JobID, err.Error()) + } + result, err := modelarts.GetTrainJobLog(job.JobID, strconv.FormatInt(job.VersionID, 10), baseLine, resultLogFile.LogFileList[0], order, lines_int) + if err != nil { + log.Error("GetTrainJobLog(%s) failed:%v", job.JobID, err.Error()) + } + if err != nil { + log.Error("trainJobGetLog(%s) failed:%v", job.JobID, err.Error()) + // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": job.JobID, + "LogFileName": "", + "StartLine": "0", + "EndLine": "0", + "Content": "", + "Lines": 0, + "CanLogDownload": false, + }) + return + } + ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": job.JobID, + "LogFileName": resultLogFile.LogFileList[0], + "StartLine": result.StartLine, + "EndLine": result.EndLine, + "Content": result.Content, + "Lines": result.Lines, + "CanLogDownload": isCanDownloadLog(ctx, job), + }) + } + } + //result := "" + //ctx.JSON(http.StatusOK, result) +} + +func isCanDownloadLog(ctx *context.APIContext, job *models.Cloudbrain) bool { + if !ctx.IsSigned { + return false + } + return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID +} + +func ModelSafetyDownloadLogFile(ctx *context.Context) { + ID := ctx.Params(":id") + job, err := models.GetCloudbrainByID(ID) + if err != nil { + log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"]) + ctx.ServerError(err.Error(), err) + return + } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainOne { + CloudbrainDownloadLogFile(ctx) + } else if job.Type == models.TypeCloudBrainTwo { + ctx.SetParams("jobid", job.JobID) + ctx.Req.Form.Set("version_name", job.VersionName) + routerRepo.TrainJobDownloadLogFile(ctx) + } + } +} + func CloudbrainDownloadLogFile(ctx *context.Context) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) @@ -378,13 +482,18 @@ func CloudbrainDownloadLogFile(ctx *context.Context) { ctx.ServerError(err.Error(), err) return } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainTwo { + ModelSafetyDownloadLogFile(ctx) + return + } + } logDir := "/model" - if job.JobType == string(models.JobTypeInference) { + if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { logDir = cloudbrain.ResultPath } - prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir - files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") + files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix+job.JobName+logDir, "") if err != nil { log.Error("query cloudbrain model failed: %v", err) return @@ -397,17 +506,22 @@ func CloudbrainDownloadLogFile(ctx *context.Context) { } } if fileName != "" { + prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName) if err != nil { log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("Get minio get SignedUrl failed", err) return } + log.Info("fileName=" + fileName) http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) + } else { + log.Info("fileName is null.") + } } -func CloudbrainGetLog(ctx *context.Context) { +func CloudbrainGetLog(ctx *context.APIContext) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) if err != nil { @@ -415,12 +529,31 @@ func CloudbrainGetLog(ctx *context.Context) { ctx.ServerError(err.Error(), err) return } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(job.JobID) + existStr := "" + if err == nil && result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + existStr = taskRes.TaskStatuses[0].ExitDiagnostics + } + ctx.Data["existStr"] = existStr + log.Info("existStr=" + existStr) + } else { + ModelSafetyGetLog(ctx) + return + } + + } + lines := ctx.QueryInt("lines") baseLine := ctx.Query("base_line") order := ctx.Query("order") var result map[string]interface{} resultPath := "/model" - if job.JobType == string(models.JobTypeInference) { + if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { resultPath = "/result" } if baseLine == "" && order == "desc" { @@ -447,12 +580,19 @@ func CloudbrainGetLog(ctx *context.Context) { return } } + content := "" + if result["Content"] != nil { + content = result["Content"].(string) + } + if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 { + content = content + ctx.Data["existStr"].(string) + } re := map[string]interface{}{ "JobID": ID, "LogFileName": result["FileName"], "StartLine": result["StartLine"], "EndLine": result["EndLine"], - "Content": result["Content"], + "Content": content, "Lines": result["Lines"], "CanLogDownload": result["FileName"] != "", } @@ -485,7 +625,7 @@ func getAllLineFromFile(path string) int { } func getLastLogFromModelDir(jobName string, lines int, resultPath string) map[string]interface{} { - prefix := "/" + setting.CBCodePathPrefix + jobName + resultPath + prefix := setting.CBCodePathPrefix + jobName + resultPath files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") if err != nil { log.Error("query cloudbrain model failed: %v", err) @@ -546,7 +686,7 @@ func getLastLogFromModelDir(jobName string, lines int, resultPath string) map[st } func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath string) map[string]interface{} { - prefix := "/" + setting.CBCodePathPrefix + jobName + resultPath + prefix := setting.CBCodePathPrefix + jobName + resultPath files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") if err != nil { log.Error("query cloudbrain model failed: %v", err) @@ -577,8 +717,12 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s r := bufio.NewReader(reader) for i := 0; i < endLine; i++ { line, error := r.ReadString('\n') - log.Info("line=" + line) if error == io.EOF { + if i >= startLine { + fileEndLine = i + re = re + line + count++ + } log.Info("read file completed.") break } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 1f4ea1ca3..0ae516750 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -121,9 +121,6 @@ func GetOverviewDuration(ctx *context.Context) { recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix now := time.Now() endTime := now - page := 1 - pagesize := 1000 - count := pagesize worker_server_num := 1 cardNum := 1 durationAllSum := int64(0) @@ -139,54 +136,46 @@ func GetOverviewDuration(ctx *context.Context) { c2NetDuration := int64(0) cDCenterDuration := int64(0) - for count == pagesize && count != 0 { - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: pagesize, - }, - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - models.LoadSpecs4CloudbrainInfo(cloudbrains) - - for _, cloudbrain := range cloudbrains { - if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { - worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber - } else { - worker_server_num = 1 - } - if cloudbrain.Cloudbrain.Spec == nil { - cardNum = 1 - } else { - cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum - } - duration := cloudbrain.Duration - durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration += duration - cloudBrainOneCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration += duration - cloudBrainTwoCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - c2NetDuration += duration - c2NetCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { - cDCenterDuration += duration - cDNetCardDuSum += durationSum - } + cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + models.LoadSpecs4CloudbrainInfo(cloudbrains) - durationAllSum += duration - cardDuSum += durationSum - count = len(cloudbrains) - page += 1 + for _, cloudbrain := range cloudbrains { + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber + } else { + worker_server_num = 1 } + if cloudbrain.Cloudbrain.Spec == nil { + cardNum = 1 + } else { + cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + duration := cloudbrain.Duration + durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration += duration + cloudBrainOneCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration + cloudBrainTwoCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration + c2NetCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDNetCardDuSum += durationSum + } + + durationAllSum += duration + cardDuSum += durationSum } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, @@ -642,7 +631,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), - string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} + string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -725,7 +714,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/api/v1/repo/fork.go b/routers/api/v1/repo/fork.go index a753f192d..9416035e7 100644 --- a/routers/api/v1/repo/fork.go +++ b/routers/api/v1/repo/fork.go @@ -7,6 +7,7 @@ package repo import ( "fmt" + "html/template" "net/http" "code.gitea.io/gitea/models" @@ -118,7 +119,7 @@ func CreateFork(ctx *context.APIContext, form api.CreateForkOption) { forker = org } - fork, err := repo_service.ForkRepository(ctx.User, forker, repo, repo.Name, repo.Description, repo.Alias) + fork, err := repo_service.ForkRepository(ctx.User, forker, repo, repo.Name, template.HTMLEscapeString(repo.Description), repo.Alias) if err != nil { ctx.Error(http.StatusInternalServerError, "ForkRepository", err) return diff --git a/routers/badge/badge.go b/routers/badge/badge.go new file mode 100644 index 000000000..6d8725b12 --- /dev/null +++ b/routers/badge/badge.go @@ -0,0 +1,136 @@ +package badge + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/badge" + "errors" + "github.com/unknwon/com" + "net/http" + "strings" +) + +func GetCustomizeBadgeList(ctx *context.Context) { + page := ctx.QueryInt("page") + category := ctx.QueryInt64("category") + pageSize := 50 + n, r, err := badge.GetBadgeList(models.GetBadgeOpts{CategoryId: category, BadgeType: models.CustomizeBadge, ListOpts: models.ListOptions{PageSize: pageSize, Page: page}}) + if err != nil { + log.Error("GetCustomizeBadgeList error.%v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + m := make(map[string]interface{}) + m["List"] = r + m["Total"] = n + m["PageSize"] = pageSize + ctx.JSON(http.StatusOK, response.SuccessWithData(m)) +} + +func OperateBadge(ctx *context.Context, req models.BadgeOperateReq) { + action := ctx.Params(":action") + + var err *response.BizError + switch action { + case "edit": + err = badge.EditBadge(req, ctx.User) + case "new": + err = badge.AddBadge(req, ctx.User) + case "del": + err = badge.DelBadge(req.ID, ctx.User) + default: + err = response.NewBizError(errors.New("action type error")) + } + + if err != nil { + log.Error("OperateBadge error ,%v", err) + ctx.JSON(http.StatusOK, response.ResponseError(err)) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} + +func GetBadgeUsers(ctx *context.Context) { + page := ctx.QueryInt("page") + badgeId := ctx.QueryInt64("badge") + pageSize := 50 + n, r, err := badge.GetBadgeUsers(badgeId, models.ListOptions{PageSize: pageSize, Page: page}) + if err != nil { + log.Error("GetBadgeUsers error.%v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + m := make(map[string]interface{}) + m["List"] = r + m["Total"] = n + m["PageSize"] = pageSize + ctx.JSON(http.StatusOK, response.SuccessWithData(m)) +} + +func AddOperateBadgeUsers(ctx *context.Context, req models.AddBadgeUsersReq) { + userStr := req.Users + if userStr == "" { + ctx.JSON(http.StatusOK, response.Success()) + return + } + userStr = strings.ReplaceAll(userStr, " ", "") + userStr = strings.ReplaceAll(userStr, "\r", "") + userNames := strings.Split(userStr, "\n") + n, err := badge.AddBadgeUsers(req.BadgeId, userNames) + if err != nil { + log.Error("AddOperateBadgeUsers error.%v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + m := make(map[string]interface{}) + m["Total"] = len(userNames) + m["Success"] = n + ctx.JSON(http.StatusOK, response.SuccessWithData(m)) +} + +func DelBadgeUsers(ctx *context.Context, req models.DelBadgeUserReq) { + id := req.ID + if id <= 0 { + ctx.JSON(http.StatusOK, response.Success()) + return + } + + err := badge.DelBadgeUser(id) + if err != nil { + log.Error("DelBadgeUsers error.%v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} + +func UploadIcon(ctx *context.Context, form badge.IconUploadForm) { + + uploader := badge.NewIconUploader(badge.IconUploadConfig{ + FileMaxSize: setting.BadgeIconMaxFileSize, + FileMaxWidth: setting.BadgeIconMaxWidth, + FileMaxHeight: setting.BadgeIconMaxHeight, + NeedSquare: true, + }) + iconName, err := uploader.Upload(form, ctx.User) + if err != nil { + log.Error("UploadIcon error.%v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + m := make(map[string]string, 0) + m["IconName"] = iconName + ctx.JSON(http.StatusOK, response.SuccessWithData(m)) +} + +func GetIcon(ctx *context.Context) { + hash := ctx.Params(":hash") + if !com.IsFile(models.GetCustomIconByHash(hash)) { + ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + return + } + ctx.Redirect(setting.AppSubURL + "/icons/" + hash) +} diff --git a/routers/badge/category.go b/routers/badge/category.go new file mode 100644 index 000000000..71c34e1ba --- /dev/null +++ b/routers/badge/category.go @@ -0,0 +1,50 @@ +package badge + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/badge" + "errors" + "net/http" +) + +func GetBadgeCategoryList(ctx *context.Context) { + page := ctx.QueryInt("page") + pageSize := 50 + n, r, err := badge.GetBadgeCategoryList(models.ListOptions{Page: page, PageSize: pageSize}) + if err != nil { + log.Error("GetCategoryList error.%v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + m := make(map[string]interface{}) + m["List"] = r + m["Total"] = n + m["PageSize"] = pageSize + ctx.JSON(http.StatusOK, response.SuccessWithData(m)) +} + +func OperateBadgeCategory(ctx *context.Context, category models.BadgeCategory4Show) { + action := ctx.Params(":action") + + var err *response.BizError + switch action { + case "edit": + err = badge.EditBadgeCategory(category, ctx.User) + case "new": + err = badge.AddBadgeCategory(category, ctx.User) + case "del": + err = badge.DelBadgeCategory(category.ID, ctx.User) + default: + err = response.NewBizError(errors.New("action type error")) + } + + if err != nil { + log.Error("OperateBadgeCategory error ,%v", err) + ctx.JSON(http.StatusOK, response.ResponseError(err)) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} diff --git a/routers/modelapp/desensitization.go b/routers/modelapp/desensitization.go new file mode 100644 index 000000000..9de33526e --- /dev/null +++ b/routers/modelapp/desensitization.go @@ -0,0 +1,78 @@ +package modelapp + +import ( + "bytes" + "code.gitea.io/gitea/models" + "crypto/tls" + "image" + "image/png" + "net/http" + "strconv" + + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/base" + + "code.gitea.io/gitea/modules/context" + "github.com/go-resty/resty/v2" +) + +var restyClient *resty.Client +var tplExploreUpload base.TplName = "model/tuomin/upload" +var uploadUrl = "/extension/tuomin/upload" +var allowedContentType = []string{"image/jpeg", "image/jpg", "image/png"} + +func ProcessImageUI(ctx *context.Context) { + ctx.HTML(200, tplExploreUpload) +} + +func ProcessImage(ctx *context.Context) { + + file, header, err := ctx.GetFile("file") + if err != nil { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.get_file_fail"))) + return + } + defer file.Close() + + contentType := header.Header.Get("Content-Type") + + if !isInAllowedContentType(contentType) { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.content_type_unsupported"))) + return + } + + client := getRestyClient() + res, err := client.R().SetMultipartField( + "file", header.Filename, contentType, file).Post(setting.ModelApp.DesensitizationUrl + "?mode=" + strconv.Itoa(ctx.QueryInt("mode"))) + if err != nil { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.process_image_fail"))) + return + } + image, _, err := image.Decode(bytes.NewReader(res.Body())) + if err != nil { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.process_image_fail"))) + return + } + + png.Encode(ctx.Resp, image) + return + +} + +func isInAllowedContentType(contentType string) bool { + for _, allowType := range allowedContentType { + if allowType == contentType { + return true + } + } + return false +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + return restyClient +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 957028fc6..5b358b83b 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "regexp" "strings" "code.gitea.io/gitea/models" @@ -85,6 +86,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio accuracy["Precision"] = "" accuracyJson, _ := json.Marshal(accuracy) log.Info("accuracyJson=" + string(accuracyJson)) + aiTask.ContainerIp = "" aiTaskJson, _ := json.Marshal(aiTask) model := &models.AiModelManage{ @@ -635,6 +637,7 @@ func ShowSingleModel(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range models { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -644,6 +647,13 @@ func ShowSingleModel(ctx *context.Context) { ctx.JSON(http.StatusOK, models) } +func removeIpInfo(model *models.AiModelManage) { + reg, _ := regexp.Compile(`[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}`) + taskInfo := model.TrainTaskInfo + taskInfo = reg.ReplaceAllString(taskInfo, "") + model.TrainTaskInfo = taskInfo +} + func queryUserName(intSlice []int64) map[int64]*models.User { keys := make(map[int64]string) uniqueElements := []int64{} @@ -677,6 +687,7 @@ func ShowOneVersionOtherModel(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range aimodels { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -793,6 +804,7 @@ func ShowModelPageInfo(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range modelResult { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -866,6 +878,7 @@ func QueryModelListForPredict(ctx *context.Context) { nameMap := make(map[string][]*models.AiModelManage) for _, model := range modelResult { + removeIpInfo(model) if _, value := nameMap[model.Name]; !value { models := make([]*models.AiModelManage, 0) models = append(models, model) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go new file mode 100644 index 000000000..5102a6722 --- /dev/null +++ b/routers/repo/aisafety.go @@ -0,0 +1,1049 @@ +package repo + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/aisafety" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward/point/account" + uuid "github.com/satori/go.uuid" +) + +const ( + tplModelSafetyTestCreateGrampusGpu = "repo/modelsafety/newgrampusgpu" + tplModelSafetyTestCreateGrampusNpu = "repo/modelsafety/newgrampusnpu" + tplModelSafetyTestCreateGpu = "repo/modelsafety/newgpu" + tplModelSafetyTestCreateNpu = "repo/modelsafety/newnpu" + tplModelSafetyTestShow = "repo/modelsafety/show" +) + +func CloudBrainAiSafetyCreateTest(ctx *context.Context) { + log.Info("start to create CloudBrainAiSafetyCreate") + uuid := uuid.NewV4() + id := uuid.String() + seriaNoParas := ctx.Query("serialNo") + fileName := ctx.Query("fileName") + + //if jobType == string(models.JobTypeBenchmark) { + req := aisafety.TaskReq{ + UnionId: id, + EvalName: "test1", + EvalContent: "test1", + TLPath: "test1", + Indicators: []string{"ACC", "ASS"}, + CDName: "CIFAR10_1000_FGSM", + BDName: "CIFAR10_1000基础数据集", + } + aisafety.GetAlgorithmList() + if seriaNoParas != "" { + aisafety.GetTaskStatus(seriaNoParas) + } else { + jsonStr, err := getJsonContent("http://192.168.207.34:8065/Test_zap1234/openi_aisafety/raw/branch/master/result/" + fileName) + serialNo, err := aisafety.CreateSafetyTask(req, jsonStr) + if err == nil { + log.Info("serialNo=" + serialNo) + time.Sleep(time.Duration(2) * time.Second) + aisafety.GetTaskStatus(serialNo) + } else { + log.Info("CreateSafetyTask error," + err.Error()) + } + } +} + +func GetAiSafetyTaskByJob(job *models.Cloudbrain) { + if job == nil { + log.Error("GetCloudbrainByJobID failed") + return + } + syncAiSafetyTaskStatus(job) +} + +func GetAiSafetyTaskTmpl(ctx *context.Context) { + ctx.Data["id"] = ctx.Params(":id") + ctx.Data["PageIsCloudBrain"] = true + ctx.HTML(200, tplModelSafetyTestShow) +} + +func GetAiSafetyTask(ctx *context.Context) { + var ID = ctx.Params(":id") + job, err := models.GetCloudbrainByIDWithDeleted(ID) + if err != nil { + log.Error("GetCloudbrainByJobID failed:" + err.Error()) + return + } + syncAiSafetyTaskStatus(job) + job, err = models.GetCloudbrainByIDWithDeleted(ID) + job.BenchmarkType = "安全评测" + job.BenchmarkTypeName = "Image Classification" + job.CanModify = cloudbrain.CanModifyJob(ctx, job) + job.CanDel = cloudbrain.CanDeleteJob(ctx, job) + if job.Parameters == "{\"parameter\":[]}" { + job.Parameters = "" + } + s, err := resource.GetCloudbrainSpec(job.ID) + if err == nil { + job.Spec = s + } + user, err := models.GetUserByID(job.UserID) + if err == nil { + tmpUser := &models.User{ + Name: user.Name, + } + job.User = tmpUser + } + + ctx.JSON(200, job) +} + +func StopAiSafetyTask(ctx *context.Context) { + log.Info("start to stop the task.") + var ID = ctx.Params(":id") + task, err := models.GetCloudbrainByIDWithDeleted(ID) + result := make(map[string]interface{}) + result["result_code"] = "-1" + if err != nil { + log.Info("query task error.err=" + err.Error()) + log.Error("GetCloudbrainByJobID failed:" + err.Error()) + result["msg"] = "No such task." + ctx.JSON(200, result) + return + } + if isTaskNotFinished(task.Status) { + if task.Type == models.TypeCloudBrainTwo { + log.Info("start to stop model arts task.") + _, err := modelarts.StopTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Info("stop failed.err=" + err.Error()) + } + task.Status = string(models.JobStopped) + if task.EndTime == 0 { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_success_update_status_fail" + ctx.JSON(200, result) + return + } + //queryTaskStatusFromCloudbrainTwo(job) + } else if task.Type == models.TypeCloudBrainOne { + if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) { + log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Already_stopped" + ctx.JSON(200, result) + return + } + err := cloudbrain.StopJob(task.JobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_failed" + ctx.JSON(200, result) + return + } + task.Status = string(models.JobStopped) + if task.EndTime == 0 { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_success_update_status_fail" + ctx.JSON(200, result) + return + } + } + } else { + if task.Status == string(models.ModelSafetyTesting) { + //修改为Failed + task.Status = string(models.JobStopped) + if task.EndTime == 0 { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_success_update_status_fail" + ctx.JSON(200, result) + return + } + } else { + log.Info("The job is finished. status=" + task.Status) + } + } + result["result_code"] = "0" + result["msg"] = "succeed" + ctx.JSON(200, result) + +} + +func DelAiSafetyTask(ctx *context.Context) { + var ID = ctx.Params(":id") + task, err := models.GetCloudbrainByIDWithDeleted(ID) + + if err != nil { + log.Error("GetCloudbrainByJobID failed:" + err.Error()) + ctx.ServerError("No such task.", err) + return + } + if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) { + log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) + ctx.ServerError("the job("+task.JobName+") has not been stopped", nil) + return + } + if task.Type == models.TypeCloudBrainOne { + DeleteCloudbrainJobStorage(task.JobName, models.TypeCloudBrainOne) + } + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError(err.Error(), err) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") +} + +func syncAiSafetyTaskStatus(job *models.Cloudbrain) { + log.Info("start to query safety task status.") + if isTaskNotFinished(job.Status) { + if job.Type == models.TypeCloudBrainTwo { + queryTaskStatusFromCloudbrainTwo(job) + } else if job.Type == models.TypeCloudBrainOne { + queryTaskStatusFromCloudbrain(job) + } + } else { + if job.Status == string(models.ModelSafetyTesting) { + queryTaskStatusFromModelSafetyTestServer(job) + } else { + log.Info("The job is finished. status=" + job.Status) + } + } +} + +func TimerHandleModelSafetyTestTask() { + log.Info("start to TimerHandleModelSafetyTestTask") + tasks, err := models.GetModelSafetyTestTask() + if err == nil { + if tasks != nil && len(tasks) > 0 { + for _, job := range tasks { + syncAiSafetyTaskStatus(job) + } + } else { + log.Info("query running model safety test task 0.") + } + } else { + log.Info("query running model safety test task err." + err.Error()) + } +} + +func queryTaskStatusFromCloudbrainTwo(job *models.Cloudbrain) { + log.Info("The task not finished,name=" + job.DisplayJobName) + result, err := modelarts.GetTrainJob(job.JobID, strconv.FormatInt(job.VersionID, 10)) + if err != nil { + log.Info("query train job error." + err.Error()) + return + } + + job.Status = modelarts.TransTrainJobStatus(result.IntStatus) + job.Duration = result.Duration / 1000 + job.TrainJobDuration = result.TrainJobDuration + + if job.StartTime == 0 && result.StartTime > 0 { + job.StartTime = timeutil.TimeStamp(result.StartTime / 1000) + } + job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) + if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { + job.EndTime = job.StartTime.Add(job.Duration) + } + job.CorrectCreateUnix() + + if job.Status != string(models.ModelArtsTrainJobCompleted) { + log.Info("CloudbrainTwo task status=" + job.Status) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } else { + log.Info("start to deal ModelSafetyTesting, task status=" + job.Status) + job.Status = string(models.ModelSafetyTesting) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + //send msg to beihang + sendNPUInferenceResultToTest(job) + } + +} + +func queryTaskStatusFromCloudbrain(job *models.Cloudbrain) { + + log.Info("The task not finished,name=" + job.DisplayJobName) + jobResult, err := cloudbrain.GetJob(job.JobID) + + result, err := models.ConvertToJobResultPayload(jobResult.Payload) + if err != nil { + log.Error("ConvertToJobResultPayload failed:", err) + return + } + job.Status = result.JobStatus.State + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + job.Status = taskRes.TaskStatuses[0].State + } + models.ParseAndSetDurationFromCloudBrainOne(result, job) + //updateCloudBrainOneJobTime(job) + log.Info("cloud brain one job status=" + job.Status) + if result.JobStatus.State != string(models.JobSucceeded) { + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } else { + // + job.Status = string(models.ModelSafetyTesting) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + //send msg to beihang + sendGPUInferenceResultToTest(job) + } +} + +func queryTaskStatusFromModelSafetyTestServer(job *models.Cloudbrain) { + result, err := aisafety.GetTaskStatus(job.PreVersionName) + if err == nil { + if result.Code == "0" { + if result.Data.Status == 1 { + log.Info("The task is running....") + } else { + if result.Data.Code == 0 { + job.ResultJson = result.Data.StandardJson + job.Status = string(models.JobSucceeded) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } else { + job.ResultJson = result.Data.Msg + job.Status = string(models.JobFailed) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + } + } else { + log.Info("The task is failed.") + job.Status = string(models.JobFailed) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + } else { + log.Info("The task not found.....") + } +} + +func getAisafetyTaskReq(job *models.Cloudbrain) aisafety.TaskReq { + datasetname := job.DatasetName + datasetnames := strings.Split(datasetname, ";") + indicator := job.LabelName + EvalContent := "test1" + if job.Description != "" { + EvalContent = job.Description + } + req := aisafety.TaskReq{ + UnionId: job.JobID, + EvalName: job.DisplayJobName, + EvalContent: EvalContent, + TLPath: "test1", + Indicators: strings.Split(indicator, ";"), + CDName: strings.Split(datasetnames[1], ".")[0], + BDName: strings.Split(datasetnames[0], ".")[0] + "基础数据集", + } + log.Info("CDName=" + req.CDName) + log.Info("BDName=" + req.BDName) + return req +} + +func sendGPUInferenceResultToTest(job *models.Cloudbrain) { + log.Info("send sendGPUInferenceResultToTest") + req := getAisafetyTaskReq(job) + resultDir := "/result" + prefix := setting.CBCodePathPrefix + job.JobName + resultDir + files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") + if err != nil { + log.Error("query cloudbrain one model failed: %v", err) + return + } + jsonContent := "" + for _, file := range files { + if strings.HasSuffix(file.FileName, "result.json") { + path := storage.GetMinioPath(job.JobName+resultDir+"/", file.FileName) + log.Info("path=" + path) + reader, err := os.Open(path) + defer reader.Close() + if err == nil { + r := bufio.NewReader(reader) + for { + line, error := r.ReadString('\n') + jsonContent += line + if error == io.EOF { + log.Info("read file completed.") + break + } + if error != nil { + log.Info("read file error." + error.Error()) + break + } + } + } + break + } + } + if jsonContent != "" { + sendHttpReqToBeihang(job, jsonContent, req) + } else { + updateJobFailed(job, "推理生成的Json数据为空，无法进行评测。") + } +} + +func sendNPUInferenceResultToTest(job *models.Cloudbrain) { + log.Info("start to sendNPUInferenceResultToTest") + req := getAisafetyTaskReq(job) + jsonContent := "" + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + resultPath := modelarts.JobPath + job.JobName + modelarts.ResultPath + VersionOutputPath + "/result.json" + resultPath = resultPath[1:] + log.Info("bucket=" + setting.Bucket + " resultPath=" + resultPath) + body, err := storage.ObsDownloadAFile(setting.Bucket, resultPath) + if err != nil { + log.Info("ObsDownloadAFile error." + err.Error() + " resultPath=" + resultPath) + } else { + defer body.Close() + var data []byte + p := make([]byte, 4096) + var readErr error + var readCount int + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + data = append(data, p[:readCount]...) + } + if readErr != nil || readCount == 0 { + break + } + } + jsonContent = string(data) + } + + if jsonContent != "" { + sendHttpReqToBeihang(job, jsonContent, req) + } else { + updateJobFailed(job, "推理生成的Json数据为空，无法进行评测。") + } +} +func updateJobFailed(job *models.Cloudbrain, msg string) { + log.Info("The json is null. so set it failed.") + //update task failed. + job.Status = string(models.ModelArtsTrainJobFailed) + job.ResultJson = msg + err := models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } +} +func sendHttpReqToBeihang(job *models.Cloudbrain, jsonContent string, req aisafety.TaskReq) { + log.Info("start to send beihang ...") + serialNo, err := aisafety.CreateSafetyTask(req, jsonContent) + if err == nil { + //update serial no to db + job.PreVersionName = serialNo + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } +} + +func isTaskNotFinished(status string) bool { + if status == string(models.ModelArtsTrainJobRunning) || status == string(models.ModelArtsTrainJobWaiting) { + return true + } + if status == string(models.JobWaiting) || status == string(models.JobRunning) { + return true + } + + if status == string(models.ModelArtsTrainJobUnknown) || status == string(models.ModelArtsTrainJobInit) { + return true + } + if status == string(models.ModelArtsTrainJobImageCreating) || status == string(models.ModelArtsTrainJobSubmitTrying) { + return true + } + return false +} + +func AiSafetyCreateForGetGPU(ctx *context.Context) { + t := time.Now() + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["IsCreate"] = true + ctx.Data["type"] = models.TypeCloudBrainOne + ctx.Data["compute_resource"] = models.GPUResource + ctx.Data["datasetType"] = models.TypeCloudBrainOne + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.GPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.GPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.GPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.GPUCombatDataSetUUID + log.Info("GPUBaseDataSetName=" + setting.ModelSafetyTest.GPUBaseDataSetName) + log.Info("GPUBaseDataSetUUID=" + setting.ModelSafetyTest.GPUBaseDataSetUUID) + log.Info("GPUCombatDataSetName=" + setting.ModelSafetyTest.GPUCombatDataSetName) + log.Info("GPUCombatDataSetUUID=" + setting.ModelSafetyTest.GPUCombatDataSetUUID) + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["display_job_name"] = displayJobName + prepareCloudbrainOneSpecs(ctx) + queuesDetail, _ := cloudbrain.GetQueuesDetail() + if queuesDetail != nil { + ctx.Data["QueuesDetail"] = queuesDetail + reqPara, _ := json.Marshal(queuesDetail) + log.Warn("The GPU WaitCount json:", string(reqPara)) + } else { + log.Info("The GPU WaitCount not get") + } + ctx.HTML(200, tplModelSafetyTestCreateGpu) +} + +func AiSafetyCreateForGetNPU(ctx *context.Context) { + t := time.Now() + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["IsCreate"] = true + ctx.Data["type"] = models.TypeCloudBrainTwo + ctx.Data["compute_resource"] = models.NPUResource + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["display_job_name"] = displayJobName + ctx.Data["datasetType"] = models.TypeCloudBrainTwo + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.NPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.NPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.NPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.NPUCombatDataSetUUID + + log.Info("NPUBaseDataSetName=" + setting.ModelSafetyTest.NPUBaseDataSetName) + log.Info("NPUBaseDataSetUUID=" + setting.ModelSafetyTest.NPUBaseDataSetUUID) + log.Info("NPUCombatDataSetName=" + setting.ModelSafetyTest.NPUCombatDataSetName) + log.Info("NPUCombatDataSetUUID=" + setting.ModelSafetyTest.NPUCombatDataSetUUID) + var resourcePools modelarts.ResourcePool + if err := json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err := json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engine_versions"] = versionInfos.Version + + prepareCloudbrainTwoInferenceSpecs(ctx) + waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") + ctx.Data["WaitCount"] = waitCount + log.Info("The NPU WaitCount is " + fmt.Sprint(waitCount)) + ctx.HTML(200, tplModelSafetyTestCreateNpu) +} + +func AiSafetyCreateForPost(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + displayJobName := ctx.Query("display_job_name") + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + + taskType := ctx.QueryInt("type") + description := ctx.Query("description") + ctx.Data["type"] = taskType + ctx.Data["displayJobName"] = displayJobName + ctx.Data["description"] = description + + repo := ctx.Repo.Repository + + tpname := tplCloudBrainModelSafetyNewNpu + if taskType == models.TypeCloudBrainOne { + tpname = tplCloudBrainModelSafetyNewGpu + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeModelSafety), displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr("the job name did already exist", tpname, nil) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tpname, nil) + return + } + } + + if !jobNamePattern.MatchString(jobName) { + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpname, nil) + return + } + + count, err := models.GetModelSafetyCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tpname, nil) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpname, nil) + return + } + } + BootFile := ctx.Query("boot_file") + BootFile = strings.TrimSpace(BootFile) + bootFileExist, err := ctx.Repo.FileExists(BootFile, cloudbrain.DefaultBranchName) + if err != nil || !bootFileExist { + log.Error("Get bootfile error:", err) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpname, nil) + return + } + if taskType == models.TypeCloudBrainTwo { + err = createForNPU(ctx, jobName) + } else if taskType == models.TypeCloudBrainOne { + err = createForGPU(ctx, jobName) + } + if err != nil { + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(err.Error(), tpname, nil) + } else { + log.Info("to redirect...") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") + } +} + +func createForNPU(ctx *context.Context, jobName string) error { + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + BootFile := ctx.Query("boot_file") + BootFile = strings.TrimSpace(BootFile) + displayJobName := ctx.Query("display_job_name") + description := ctx.Query("description") + + srcDataset := ctx.Query("src_dataset") //uuid + combatDataset := ctx.Query("combat_dataset") //uuid + evaluationIndex := ctx.Query("evaluation_index") + Params := ctx.Query("run_para_list") + specId := ctx.QueryInt64("spec_id") + + engineID := ctx.QueryInt("engine_id") + log.Info("engine_id=" + fmt.Sprint(engineID)) + poolID := ctx.Query("pool_id") + repo := ctx.Repo.Repository + + trainUrl := ctx.Query("train_url") + modelName := ctx.Query("model_name") + modelVersion := ctx.Query("model_version") + ckptName := ctx.Query("ckpt_name") + ckptUrl := "/" + trainUrl + ckptName + log.Info("ckpt url:" + ckptUrl) + + FlavorName := ctx.Query("flaver_names") + EngineName := ctx.Query("engine_names") + + isLatestVersion := modelarts.IsLatestVersion + VersionCount := modelarts.VersionCountOne + + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + resultObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.ResultPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" + log.Info("ckpt url:" + ckptUrl) + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobTypeInference, + ComputeResource: models.NPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainTwo}) + if err != nil || spec == nil { + + //ctx.RenderWithErr("Resource specification not available", tplCloudBrainModelSafetyNewNpu, nil) + return errors.New("Resource specification not available") + } + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) + return errors.New(ctx.Tr("points.insufficient_points_balance")) + } + + //todo: del the codeLocalPath + _, err = ioutil.ReadDir(codeLocalPath) + if err == nil { + os.RemoveAll(codeLocalPath) + } + + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(cloudbrain.DefaultBranchName) + + if err := downloadCode(repo, codeLocalPath, cloudbrain.DefaultBranchName); err != nil { + log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) + return errors.New(ctx.Tr("cloudbrain.load_code_failed")) + } + + //todo: upload code (send to file_server todo this work?) + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.ResultPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_result: %s (%v)", repo.FullName(), err) + return errors.New("Failed to obsMkdir_result") + } + + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) + return errors.New("Failed to obsMkdir_log") + } + + if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) + return errors.New(ctx.Tr("cloudbrain.load_code_failed")) + } + + var parameters models.Parameters + param := make([]models.Parameter, 0) + param = append(param, models.Parameter{ + Label: modelarts.ResultUrl, + Value: "s3:/" + resultObsPath, + }, models.Parameter{ + Label: modelarts.CkptUrl, + Value: "s3:/" + ckptUrl, + }) + uuid := srcDataset + ";" + combatDataset + datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid) + if err != nil { + return err + } + dataPath := dataUrl + jsondatas, err := json.Marshal(datasUrlList) + if err != nil { + log.Error("Failed to Marshal: %v", err) + return err + } + if isMultiDataset { + param = append(param, models.Parameter{ + Label: modelarts.MultiDataUrl, + Value: string(jsondatas), + }) + } + + existDeviceTarget := false + if len(Params) != 0 { + err := json.Unmarshal([]byte(Params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", Params, err) + return errors.New("运行参数错误") + } + + for _, parameter := range parameters.Parameter { + if parameter.Label == modelarts.DeviceTarget { + existDeviceTarget = true + } + if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl { + param = append(param, models.Parameter{ + Label: parameter.Label, + Value: parameter.Value, + }) + } + } + } + if !existDeviceTarget { + param = append(param, models.Parameter{ + Label: modelarts.DeviceTarget, + Value: modelarts.Ascend, + }) + } + + req := &modelarts.GenerateInferenceJobReq{ + JobName: jobName, + DisplayJobName: displayJobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + BootFile, + BootFile: BootFile, + TrainUrl: trainUrl, + WorkServerNumber: 1, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Uuid: uuid, + Parameters: param, //modelarts train parameters + CommitID: commitID, + BranchName: cloudbrain.DefaultBranchName, + Params: Params, + FlavorName: FlavorName, + EngineName: EngineName, + LabelName: evaluationIndex, + IsLatestVersion: isLatestVersion, + VersionCount: VersionCount, + TotalVersionCount: modelarts.TotalVersionCount, + ModelName: modelName, + ModelVersion: modelVersion, + CkptName: ckptName, + ResultUrl: resultObsPath, + Spec: spec, + DatasetName: datasetNames, + JobType: string(models.JobTypeModelSafety), + } + + err = modelarts.GenerateInferenceJob(ctx, req) + if err != nil { + log.Error("GenerateTrainJob failed:%v", err.Error()) + return err + } + return nil +} + +func createForGPU(ctx *context.Context, jobName string) error { + BootFile := ctx.Query("boot_file") + BootFile = strings.TrimSpace(BootFile) + displayJobName := ctx.Query("display_job_name") + description := ctx.Query("description") + image := strings.TrimSpace(ctx.Query("image")) + srcDataset := ctx.Query("src_dataset") //uuid + combatDataset := ctx.Query("combat_dataset") //uuid + evaluationIndex := ctx.Query("evaluation_index") + Params := ctx.Query("run_para_list") + specId := ctx.QueryInt64("spec_id") + TrainUrl := ctx.Query("train_url") + CkptName := ctx.Query("ckpt_name") + modelName := ctx.Query("model_name") + modelVersion := ctx.Query("model_version") + + ckptUrl := setting.Attachment.Minio.RealPath + TrainUrl + CkptName + log.Info("ckpt url:" + ckptUrl) + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobTypeBenchmark, + ComputeResource: models.GPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainOne}) + if err != nil || spec == nil { + return errors.New("Resource specification not available") + } + + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) + return errors.New(ctx.Tr("points.insufficient_points_balance")) + } + + repo := ctx.Repo.Repository + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + os.RemoveAll(codePath) + + if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { + log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) + return errors.New("system error") + } + + err = uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + if err != nil { + log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) + return errors.New("system error") + } + + uuid := srcDataset + ";" + combatDataset + datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) + log.Info("uuid=" + uuid) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return errors.New(ctx.Tr("cloudbrain.error.dataset_select")) + } + command, err := getGpuModelSafetyCommand(BootFile, Params, CkptName, displayJobName) + if err != nil { + log.Error("Get Command failed: %v", err, ctx.Data["MsgID"]) + return errors.New(ctx.Tr("cloudbrain.error.dataset_select")) + } + log.Info("Command=" + command) + + req := cloudbrain.GenerateCloudBrainTaskReq{ + Ctx: ctx, + DisplayJobName: displayJobName, + JobName: jobName, + Image: image, + Command: command, + Uuids: uuid, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, + CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + ModelPath: setting.Attachment.Minio.RealPath + TrainUrl, + BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), + Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + JobType: string(models.JobTypeModelSafety), + Description: description, + BranchName: cloudbrain.DefaultBranchName, + BootFile: BootFile, + Params: Params, + CommitID: "", + ModelName: modelName, + ModelVersion: modelVersion, + CkptName: CkptName, + ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), + Spec: spec, + LabelName: evaluationIndex, + } + + err = cloudbrain.GenerateTask(req) + if err != nil { + return err + } + return nil +} + +func getGpuModelSafetyCommand(BootFile string, params string, CkptName string, DisplayJobName string) (string, error) { + var command string + bootFile := strings.TrimSpace(BootFile) + + if !strings.HasSuffix(bootFile, ".py") { + log.Error("bootFile(%s) format error", bootFile) + return command, errors.New("bootFile format error") + } + + var parameters models.Parameters + var param string + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + return command, err + } + + for _, parameter := range parameters.Parameter { + param += " --" + parameter.Label + "=" + parameter.Value + } + } + + param += " --ckpt_url=" + cloudbrain.ModelMountPath + "/" + CkptName + + command += "python /code/" + bootFile + param + " > " + cloudbrain.ResultPath + "/" + DisplayJobName + "-" + cloudbrain.LogFile + + return command, nil +} + +func modelSafetyNewDataPrepare(ctx *context.Context) error { + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["type"] = ctx.QueryInt("type") + ctx.Data["boot_file"] = ctx.Query("boot_file") + ctx.Data["display_job_name"] = ctx.Query("display_job_name") + ctx.Data["description"] = ctx.Query("description") + ctx.Data["image"] = strings.TrimSpace(ctx.Query("image")) + ctx.Data["src_dataset"] = ctx.Query("src_dataset") //uuid + ctx.Data["combat_dataset"] = ctx.Query("combat_dataset") //uuid + ctx.Data["evaluationIndex"] = ctx.Query("evaluation_index") + ctx.Data["run_para_list"] = ctx.Query("run_para_list") + ctx.Data["spec_id"] = ctx.QueryInt64("spec_id") + ctx.Data["train_url"] = ctx.Query("train_url") + ctx.Data["ckpt_name"] = ctx.Query("ckpt_name") + + ctx.Data["train_url"] = ctx.Query("train_url") + ctx.Data["ckpt_name"] = ctx.Query("ckpt_name") + ctx.Data["model_name"] = ctx.Query("model_name") + ctx.Data["model_version"] = ctx.Query("model_version") + + if ctx.QueryInt("type") == models.TypeCloudBrainOne { + ctx.Data["type"] = models.TypeCloudBrainOne + ctx.Data["compute_resource"] = models.GPUResource + ctx.Data["datasetType"] = models.TypeCloudBrainOne + + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.GPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.GPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.GPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.GPUCombatDataSetUUID + prepareCloudbrainOneSpecs(ctx) + + queuesDetail, _ := cloudbrain.GetQueuesDetail() + if queuesDetail != nil { + ctx.Data["QueuesDetail"] = queuesDetail + } + + } else { + ctx.Data["engine_id"] = ctx.QueryInt("engine_id") + ctx.Data["pool_id"] = ctx.Query("pool_id") + + ctx.Data["type"] = models.TypeCloudBrainTwo + ctx.Data["compute_resource"] = models.NPUResource + ctx.Data["datasetType"] = models.TypeCloudBrainTwo + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.NPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.NPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.NPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.NPUCombatDataSetUUID + + var engines modelarts.Engine + if err := json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engine_versions"] = versionInfos.Version + prepareCloudbrainTwoInferenceSpecs(ctx) + + waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") + ctx.Data["WaitCount"] = waitCount + } + + return nil +} + +func getJsonContent(url string) (string, error) { + + resp, err := http.Get(url) + if err != nil || resp.StatusCode != 200 { + log.Info("Get organizations url error=" + err.Error()) + return "", err + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + return "", err + } + str := string(bytes) + //log.Info("json str =" + str) + + return str, nil +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 74f6924f4..a2ea7d51b 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -51,6 +51,9 @@ const ( tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new" tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show" + tplCloudBrainModelSafetyNewGpu base.TplName = "repo/modelsafety/newgpu" + tplCloudBrainModelSafetyNewNpu base.TplName = "repo/modelsafety/newnpu" + tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit" tplCloudBrainImageEdit base.TplName = "repo/cloudbrain/image/edit" @@ -140,8 +143,11 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled ctx.Data["datasetType"] = models.TypeCloudBrainOne - - ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode") + defaultMode := ctx.Query("benchmarkMode") + if defaultMode == "" { + defaultMode = "alogrithm" + } + ctx.Data["benchmarkMode"] = defaultMode if ctx.Cloudbrain != nil { ctx.Data["branch_name"] = ctx.Cloudbrain.BranchName @@ -734,7 +740,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo } else { task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id")) } - + if task.JobType == string(models.JobTypeModelSafety) { + GetAiSafetyTaskTmpl(ctx) + return + } if err != nil { log.Info("error:" + err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) @@ -760,8 +769,8 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State + task.ContainerIp = "" task.ContainerID = taskRes.TaskStatuses[0].ContainerID - task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) if task.DeletedAt.IsZero() { //normal record @@ -1827,6 +1836,9 @@ func SyncCloudbrainStatus() { } for _, task := range cloudBrains { + if task.JobType == string(models.JobTypeModelSafety) { + continue + } if task.Type == models.TypeCloudBrainOne { result, err := cloudbrain.GetJob(task.JobID) if err != nil { @@ -2112,14 +2124,14 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, - Type: models.TypeCloudBrainOne, + Type: -1, JobTypes: jobTypes, }) if err != nil { @@ -2166,6 +2178,10 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } } } + if task.JobType == string(models.JobTypeModelSafety) { + ciTasks[i].BenchmarkType = "安全评测" + ciTasks[i].BenchmarkTypeName = "Image Classification" + } } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 40edc4767..b350343db 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -303,7 +303,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo } if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { ctx.Redirect(ctx.Repo.RepoLink + "/src/branch/" + util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(form.TreePath)) } @@ -475,7 +475,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { ctx.Flash.Success(ctx.Tr("repo.editor.file_delete_success", ctx.Repo.TreePath)) if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { treePath := filepath.Dir(ctx.Repo.TreePath) if treePath == "." { @@ -686,7 +686,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { ctx.Redirect(ctx.Repo.RepoLink + "/src/branch/" + util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(form.TreePath)) } diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index abf64281a..b78bdebd3 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -713,6 +713,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain DatasetNames: datasetNames, DatasetInfos: datasetInfos, Spec: spec, + CodeName: strings.ToLower(repo.Name), } if form.ModelName != "" { //使用预训练模型训练 req.ModelName = form.ModelName @@ -837,6 +838,7 @@ func GrampusTrainJobShow(ctx *context.Context) { ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } + task.ContainerIp = "" if task.DeletedAt.IsZero() { //normal record result, err := grampus.GetJob(task.JobID) @@ -976,8 +978,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" - command += commandDownload + //no need to download code & dataset by internet } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) @@ -986,8 +987,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //unzip code & dataset if processorType == grampus.ProcessorTypeNPU { - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" - command += commandUnzip + //no need to process } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand @@ -1024,7 +1024,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo var commandCode string if processorType == grampus.ProcessorTypeNPU { - commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" + commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py /tmp/log/train.log" + paramCode + ";" } else if processorType == grampus.ProcessorTypeGPU { if pretrainModelFileName != "" { paramCode += " --ckpt_url" + "=" + workDir + "pretrainmodel/" + pretrainModelFileName diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f70d84208..6e44b3cd2 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1795,7 +1795,7 @@ func TrainJobShow(ctx *context.Context) { datasetList = append(datasetList, GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)) VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) - + VersionListTasks[i].ContainerIp = "" //add spec s, err := resource.GetCloudbrainSpec(task.Cloudbrain.ID) if err != nil { @@ -2200,6 +2200,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ResultUrl: resultObsPath, Spec: spec, DatasetName: datasetNames, + JobType: string(models.JobTypeInference), } err = modelarts.GenerateInferenceJob(ctx, req) @@ -2737,7 +2738,7 @@ func TrainJobDownloadLogFile(ctx *context.Context) { return } ctx.Resp.Header().Set("Cache-Control", "max-age=0") - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) } func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) { var isMultiDataset bool diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 03d2d832a..2c8c2f45b 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -12,6 +12,7 @@ import ( "path" "regexp" "strings" + "text/template" "unicode/utf8" "code.gitea.io/gitea/modules/validation" @@ -212,7 +213,7 @@ func CreatePost(ctx *context.Context, form auth.CreateRepoForm) { opts := models.GenerateRepoOptions{ Name: form.RepoName, Alias: form.Alias, - Description: form.Description, + Description: template.HTMLEscapeString(form.Description), Private: form.Private, GitContent: form.GitContent, Topics: form.Topics, diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 11efdf275..5fcfb2287 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -8,6 +8,7 @@ package repo import ( "errors" "fmt" + "html/template" "io/ioutil" "net/url" "regexp" @@ -129,7 +130,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { // In case it's just a case change. repo.Name = newRepoName repo.LowerName = strings.ToLower(newRepoName) - repo.Description = form.Description + repo.Description = template.HTMLEscapeString(form.Description) repo.Website = form.Website repo.IsTemplate = form.Template repo.Alias = newAlias diff --git a/routers/repo/setting_protected_branch.go b/routers/repo/setting_protected_branch.go index ab0fd77ee..f1ea17528 100644 --- a/routers/repo/setting_protected_branch.go +++ b/routers/repo/setting_protected_branch.go @@ -5,6 +5,7 @@ package repo import ( + "code.gitea.io/gitea/modules/util" "fmt" "strings" "time" @@ -192,7 +193,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) } if f.RequiredApprovals < 0 { ctx.Flash.Error(ctx.Tr("repo.settings.protected_branch_required_approvals_min")) - ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, branch)) + ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, util.PathEscapeSegments(branch))) } var whitelistUsers, whitelistTeams, mergeWhitelistUsers, mergeWhitelistTeams, approvalsWhitelistUsers, approvalsWhitelistTeams []int64 @@ -263,7 +264,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) return } ctx.Flash.Success(ctx.Tr("repo.settings.update_protect_branch_success", branch)) - ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, branch)) + ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, util.PathEscapeSegments(branch))) } else { if protectBranch != nil { if err := ctx.Repo.Repository.DeleteProtectedBranch(protectBranch.ID); err != nil { diff --git a/routers/response/response_list.go b/routers/response/response_list.go index 6514f3edd..8bdbf375c 100644 --- a/routers/response/response_list.go +++ b/routers/response/response_list.go @@ -3,3 +3,6 @@ package response var RESOURCE_QUEUE_NOT_AVAILABLE = &BizError{Code: 1001, Err: "resource queue not available"} var SPECIFICATION_NOT_EXIST = &BizError{Code: 1002, Err: "specification not exist"} var SPECIFICATION_NOT_AVAILABLE = &BizError{Code: 1003, Err: "specification not available"} + +var CATEGORY_STILL_HAS_BADGES = &BizError{Code: 1004, Err: "Please delete badges in the category first"} +var BADGES_STILL_HAS_USERS = &BizError{Code: 1005, Err: "Please delete users of badge first"} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index e52686ee4..be603785f 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -12,6 +12,14 @@ import ( "text/template" "time" + "code.gitea.io/gitea/routers/badge" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" + badge_service "code.gitea.io/gitea/services/badge" + "code.gitea.io/gitea/services/reward" + + "code.gitea.io/gitea/routers/modelapp" + "code.gitea.io/gitea/routers/reward/point" "code.gitea.io/gitea/routers/task" "code.gitea.io/gitea/services/reward" @@ -194,6 +202,14 @@ func NewMacaron() *macaron.Macaron { ExpiresAfter: setting.StaticCacheTime, }, )) + m.Use(public.StaticHandler( + setting.IconUploadPath, + &public.Options{ + Prefix: "icons", + SkipLogging: setting.DisableRouterLog, + ExpiresAfter: setting.StaticCacheTime, + }, + )) m.Use(public.StaticHandler( setting.RepositoryAvatarUploadPath, &public.Options{ @@ -347,6 +363,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/user/login/kanban", user.SignInPostAPI) m.Get("/home/term", routers.HomeTerm) m.Get("/home/privacy", routers.HomePrivacy) + m.Get("/extension/tuomin/upload", modelapp.ProcessImageUI) + m.Post("/extension/tuomin/upload", reqSignIn, modelapp.ProcessImage) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") @@ -519,6 +537,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/avatar/:hash", user.AvatarByEmailHash) + m.Get("/show/icon/:hash", badge.GetIcon) + adminReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, AdminRequired: true}) // ***** START: Admin ***** @@ -664,6 +684,23 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/add/batch", bindIgnErr(models.BatchLimitConfigVO{}), task.BatchAddTaskConfig) m.Post("/^:action(new|edit|del)$", bindIgnErr(models.TaskConfigWithLimit{}), task.OperateTaskConfig) }) + + m.Group("/badge", func() { + m.Group("/category", func() { + m.Get("/list", badge.GetBadgeCategoryList) + m.Post("/^:action(new|edit|del)$", bindIgnErr(models.BadgeCategory4Show{}), badge.OperateBadgeCategory) + }) + m.Group("/customize", func() { + m.Get("/list", badge.GetCustomizeBadgeList) + }) + m.Group("/users", func() { + m.Get("", badge.GetBadgeUsers) + m.Post("/add", bindIgnErr(models.AddBadgeUsersReq{}), badge.AddOperateBadgeUsers) + m.Post("/del", bindIgnErr(models.DelBadgeUserReq{}), badge.DelBadgeUsers) + }) + m.Post("/^:action(new|edit|del)$", bindIgnErr(models.BadgeOperateReq{}), badge.OperateBadge) + }) + m.Post("/icon/upload", bindIgnErr(badge_service.IconUploadForm{}), badge.UploadIcon) }, operationReq) // ***** END: Operation ***** @@ -1200,7 +1237,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/modelmanage", func() { m.Post("/create_model", repo.SaveModel) - m.Post("/create_model_convert", reqRepoModelManageWriter, repo.SaveModelConvert) + m.Post("/create_model_convert", reqWechatBind, reqRepoModelManageWriter, repo.SaveModelConvert) m.Post("/create_new_model", repo.SaveNewNameModel) m.Delete("/delete_model", repo.DeleteModel) m.Post("/delete_model_convert/:id", repo.DeleteModelConvert) @@ -1227,6 +1264,18 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/downloadall", repo.DownloadMultiModelFile) }, context.RepoRef()) + m.Group("/modelsafety", func() { + m.Group("/:id", func() { + m.Get("/show", reqRepoCloudBrainWriter, repo.GetAiSafetyTaskTmpl) + m.Get("", reqRepoCloudBrainWriter, repo.GetAiSafetyTask) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.StopAiSafetyTask) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.DelAiSafetyTask) + }) + m.Get("/create_gpu", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.AiSafetyCreateForGetGPU) + m.Get("/create_npu", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.AiSafetyCreateForGetNPU) + m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.AiSafetyCreateForPost) + }, context.RepoRef()) + m.Group("/debugjob", func() { m.Get("", reqRepoCloudBrainReader, repo.DebugJobIndex) }, context.RepoRef()) diff --git a/routers/user/home.go b/routers/user/home.go index 991c80328..1a20c26e2 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -777,7 +777,7 @@ func Cloudbrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/user/profile.go b/routers/user/profile.go index 42cdfd1a8..66a480b7f 100755 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -6,6 +6,7 @@ package user import ( + "code.gitea.io/gitea/services/badge" "errors" "fmt" "path" @@ -90,10 +91,25 @@ func Profile(ctx *context.Context) { return } + // Show user badges + badges, err := badge.GetUserBadges(ctxUser.ID, models.ListOptions{Page: 1, PageSize: 5}) + if err != nil { + ctx.ServerError("GetUserBadges", err) + return + } + // Count user badges + cnt, err := badge.CountUserBadges(ctxUser.ID) + if err != nil { + ctx.ServerError("CountUserBadges", err) + return + } + ctx.Data["Title"] = ctxUser.DisplayName() ctx.Data["PageIsUserProfile"] = true ctx.Data["Owner"] = ctxUser ctx.Data["OpenIDs"] = openIDs + ctx.Data["RecentBadges"] = badges + ctx.Data["TotalBadges"] = cnt ctx.Data["EnableHeatmap"] = setting.Service.EnableUserHeatmap ctx.Data["HeatmapUser"] = ctxUser.Name showPrivate := ctx.IsSigned && (ctx.User.IsAdmin || ctx.User.ID == ctxUser.ID) @@ -297,6 +313,13 @@ func Profile(ctx *context.Context) { } total = int(count) + case "badge": + allBadges, err := badge.GetUserAllBadges(ctxUser.ID) + if err != nil { + ctx.ServerError("GetUserAllBadges", err) + return + } + ctx.Data["AllBadges"] = allBadges default: ctx.ServerError("tab error", errors.New("tab error")) return diff --git a/services/admin/operate_log/operate_log.go b/services/admin/operate_log/operate_log.go index 7b72ec2e2..f52950351 100644 --- a/services/admin/operate_log/operate_log.go +++ b/services/admin/operate_log/operate_log.go @@ -4,6 +4,13 @@ import ( "code.gitea.io/gitea/models" ) +type LogBizType string + +const ( + BadgeCategoryOperate LogBizType = "BadgeCategoryOperate" + BadgeOperate LogBizType = "BadgeOperate" +) + func Log(log models.AdminOperateLog) error { _, err := models.InsertAdminOperateLog(log) return err @@ -12,3 +19,34 @@ func Log(log models.AdminOperateLog) error { func NewLogValues() *models.LogValues { return &models.LogValues{Params: make([]models.LogValue, 0)} } + +func Log4Add(bizType LogBizType, newValue interface{}, doerId int64, comment string) { + Log(models.AdminOperateLog{ + BizType: string(bizType), + OperateType: "add", + NewValue: NewLogValues().Add("new", newValue).JsonString(), + CreatedBy: doerId, + Comment: comment, + }) +} + +func Log4Edit(bizType LogBizType, oldValue interface{}, newValue interface{}, doerId int64, comment string) { + Log(models.AdminOperateLog{ + BizType: string(bizType), + OperateType: "edit", + NewValue: NewLogValues().Add("new", newValue).JsonString(), + OldValue: NewLogValues().Add("old", oldValue).JsonString(), + CreatedBy: doerId, + Comment: comment, + }) +} + +func Log4Del(bizType LogBizType, oldValue interface{}, doerId int64, comment string) { + Log(models.AdminOperateLog{ + BizType: string(bizType), + OperateType: "del", + OldValue: NewLogValues().Add("old", oldValue).JsonString(), + CreatedBy: doerId, + Comment: comment, + }) +} diff --git a/services/badge/badge.go b/services/badge/badge.go new file mode 100644 index 000000000..c6f833f65 --- /dev/null +++ b/services/badge/badge.go @@ -0,0 +1,80 @@ +package badge + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/admin/operate_log" + "errors" +) + +func GetBadgeList(opts models.GetBadgeOpts) (int64, []*models.Badge4AdminShow, error) { + total, list, err := models.GetBadgeList(opts) + if err != nil { + return 0, nil, err + } + if len(list) == 0 { + return 0, nil, nil + } + r := make([]*models.Badge4AdminShow, len(list)) + for i := 0; i < len(list); i++ { + r[i] = list[i].ToShow() + } + + return total, r, nil +} + +func AddBadge(m models.BadgeOperateReq, doer *models.User) *response.BizError { + _, err := models.GetBadgeCategoryById(m.CategoryId) + + if err != nil { + if models.IsErrRecordNotExist(err) { + return response.NewBizError(errors.New("badge category is not available")) + } + return response.NewBizError(err) + } + _, err = models.AddBadge(m.ToDTO()) + if err != nil { + return response.NewBizError(err) + } + operate_log.Log4Add(operate_log.BadgeOperate, m, doer.ID, "新增了勋章") + return nil +} + +func EditBadge(m models.BadgeOperateReq, doer *models.User) *response.BizError { + if m.ID == 0 { + log.Error(" EditBadge param error") + return response.NewBizError(errors.New("param error")) + } + old, err := models.GetBadgeById(m.ID) + if err != nil { + return response.NewBizError(err) + } + _, err = models.UpdateBadgeById(m.ID, m.ToDTO()) + if err != nil { + return response.NewBizError(err) + } + operate_log.Log4Edit(operate_log.BadgeOperate, old, m.ToDTO(), doer.ID, "修改了勋章") + return nil +} + +func DelBadge(id int64, doer *models.User) *response.BizError { + if id == 0 { + log.Error(" DelBadge param error") + return response.NewBizError(errors.New("param error")) + } + old, err := models.GetBadgeById(id) + if err != nil { + return response.NewBizError(err) + } + n, _, err := models.GetBadgeUsers(id, models.ListOptions{PageSize: 1, Page: 1}) + if err != nil { + return response.NewBizError(err) + } + if n > 0 { + return response.BADGES_STILL_HAS_USERS + } + _, err = models.DelBadge(id) + operate_log.Log4Del(operate_log.BadgeOperate, old, doer.ID, "删除了勋章") + return nil +} diff --git a/services/badge/category.go b/services/badge/category.go new file mode 100644 index 000000000..445dedcad --- /dev/null +++ b/services/badge/category.go @@ -0,0 +1,72 @@ +package badge + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/admin/operate_log" + "errors" +) + +func GetBadgeCategoryList(opts models.ListOptions) (int64, []*models.BadgeCategory4Show, error) { + total, list, err := models.GetBadgeCategoryListPaging(opts) + if err != nil { + return 0, nil, err + } + if len(list) == 0 { + return 0, nil, nil + } + r := make([]*models.BadgeCategory4Show, len(list)) + for i := 0; i < len(list); i++ { + r[i] = list[i].ToShow() + } + + return total, r, nil +} + +func AddBadgeCategory(m models.BadgeCategory4Show, doer *models.User) *response.BizError { + _, err := models.AddBadgeCategory(m.ToDTO()) + if err != nil { + return response.NewBizError(err) + } + operate_log.Log4Add(operate_log.BadgeCategoryOperate, m, doer.ID, "新增了勋章分类") + return nil +} + +func EditBadgeCategory(m models.BadgeCategory4Show, doer *models.User) *response.BizError { + if m.ID == 0 { + log.Error(" EditBadgeCategory param error") + return response.NewBizError(errors.New("param error")) + } + old, err := models.GetBadgeCategoryById(m.ID) + if err != nil { + return response.NewBizError(err) + } + _, err = models.UpdateBadgeCategoryById(m.ID, m.ToDTO()) + if err != nil { + return response.NewBizError(err) + } + operate_log.Log4Edit(operate_log.BadgeCategoryOperate, old, m.ToDTO(), doer.ID, "修改了勋章分类") + return nil +} + +func DelBadgeCategory(id int64, doer *models.User) *response.BizError { + if id == 0 { + log.Error(" DelBadgeCategory param error") + return response.NewBizError(errors.New("param error")) + } + old, err := models.GetBadgeCategoryById(id) + if err != nil { + return response.NewBizError(err) + } + badges, err := models.GetBadgeByCategoryId(id) + if err != nil { + return response.NewBizError(err) + } + if len(badges) > 0 { + return response.CATEGORY_STILL_HAS_BADGES + } + _, err = models.DelBadgeCategory(id) + operate_log.Log4Del(operate_log.BadgeCategoryOperate, old, doer.ID, "删除了勋章分类") + return nil +} diff --git a/services/badge/icon.go b/services/badge/icon.go new file mode 100644 index 000000000..fd731b586 --- /dev/null +++ b/services/badge/icon.go @@ -0,0 +1,140 @@ +package badge + +import ( + "bytes" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/setting" + "crypto/md5" + "errors" + "fmt" + "github.com/nfnt/resize" + "github.com/oliamb/cutter" + "image" + "image/png" + "io/ioutil" + "mime/multipart" + "os" +) + +type IconUploader struct { + Config IconUploadConfig +} + +type IconUploadForm struct { + Icon *multipart.FileHeader +} + +type IconUploadConfig struct { + FileMaxSize int64 + FileMaxWidth int + FileMaxHeight int + DefaultSize uint + NeedResize bool + NeedSquare bool +} + +func NewIconUploader(config IconUploadConfig) IconUploader { + return IconUploader{Config: config} +} + +func (u IconUploader) Upload(form IconUploadForm, user *models.User) (string, error) { + if form.Icon == nil || form.Icon.Filename == "" { + return "", errors.New("File or fileName is empty") + } + + fr, err := form.Icon.Open() + if err != nil { + return "", fmt.Errorf("Icon.Open: %v", err) + } + defer fr.Close() + + if form.Icon.Size > u.Config.FileMaxSize { + return "", errors.New("File is too large") + } + + data, err := ioutil.ReadAll(fr) + if err != nil { + return "", fmt.Errorf("ioutil.ReadAll: %v", err) + } + if !base.IsImageFile(data) { + return "", errors.New("File is not a image") + } + iconName, err := u.uploadIcon(data, user.ID) + if err != nil { + return "", fmt.Errorf("uploadIcon: %v", err) + } + return iconName, nil + +} + +func (u IconUploader) uploadIcon(data []byte, userId int64) (string, error) { + m, err := u.prepare(data) + if err != nil { + return "", err + } + + iconName := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", userId, md5.Sum(data))))) + + if err := os.MkdirAll(setting.IconUploadPath, os.ModePerm); err != nil { + return "", fmt.Errorf("uploadIcon. Failed to create dir %s: %v", setting.AvatarUploadPath, err) + } + + fw, err := os.Create(models.GetCustomIconByHash(iconName)) + if err != nil { + return "", fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + if err = png.Encode(fw, *m); err != nil { + return "", fmt.Errorf("Encode: %v", err) + } + + return iconName, nil +} + +func (u IconUploader) prepare(data []byte) (*image.Image, error) { + imgCfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("DecodeConfig: %v", err) + } + if imgCfg.Width > u.Config.FileMaxWidth { + return nil, fmt.Errorf("Image width is too large: %d > %d", imgCfg.Width, setting.AvatarMaxWidth) + } + if imgCfg.Height > u.Config.FileMaxHeight { + return nil, fmt.Errorf("Image height is too large: %d > %d", imgCfg.Height, setting.AvatarMaxHeight) + } + + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("Decode: %v", err) + } + + if u.Config.NeedSquare { + if imgCfg.Width != imgCfg.Height { + var newSize, ax, ay int + if imgCfg.Width > imgCfg.Height { + newSize = imgCfg.Height + ax = (imgCfg.Width - imgCfg.Height) / 2 + } else { + newSize = imgCfg.Width + ay = (imgCfg.Height - imgCfg.Width) / 2 + } + + img, err = cutter.Crop(img, cutter.Config{ + Width: newSize, + Height: newSize, + Anchor: image.Point{ax, ay}, + }) + if err != nil { + return nil, err + } + } + } + + if u.Config.NeedResize && u.Config.DefaultSize > 0 { + img = resize.Resize(u.Config.DefaultSize, u.Config.DefaultSize, img, resize.NearestNeighbor) + } + + return &img, nil +} diff --git a/services/badge/user.go b/services/badge/user.go new file mode 100644 index 000000000..025b10f77 --- /dev/null +++ b/services/badge/user.go @@ -0,0 +1,111 @@ +package badge + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +func GetBadgeUsers(badgeId int64, opts models.ListOptions) (int64, []*models.BadgeUser4SHow, error) { + total, list, err := models.GetBadgeUsers(badgeId, opts) + if err != nil { + return 0, nil, err + } + if len(list) == 0 { + return 0, nil, nil + } + r := make([]*models.BadgeUser4SHow, len(list)) + for i := 0; i < len(list); i++ { + r[i] = list[i].ToShow() + } + + return total, r, nil +} + +func AddBadgeUsers(badgeId int64, userNames []string) (int, error) { + userIds := models.GetUserIdsByUserNames(userNames) + if len(userIds) == 0 { + return 0, nil + } + successCount := 0 + for _, v := range userIds { + m := models.BadgeUser{ + UserId: v, + BadgeId: badgeId, + } + _, err := models.AddBadgeUser(m) + if err != nil { + log.Error("AddBadgeUser err in loop, m=%+v. e=%v", m, err) + continue + } + successCount++ + } + return successCount, nil +} + +func DelBadgeUser(id int64) error { + _, err := models.DelBadgeUser(id) + return err +} + +//GetUserBadges Only Returns badges the user has earned +func GetUserBadges(userId int64, opts models.ListOptions) ([]*models.Badge4UserShow, error) { + badges, err := models.GetUserBadgesPaging(userId, models.GetUserBadgesOpts{ListOptions: opts}) + if err != nil { + return nil, err + } + r := make([]*models.Badge4UserShow, len(badges)) + for i, v := range badges { + r[i] = v.ToUserShow() + } + return r, nil +} + +func CountUserBadges(userId int64) (int64, error) { + return models.CountUserBadges(userId) +} + +func GetUserAllBadges(userId int64) ([]models.UserAllBadgeInCategory, error) { + categoryList, err := models.GetBadgeCategoryList() + if err != nil { + return nil, err + } + r := make([]models.UserAllBadgeInCategory, 0) + for _, v := range categoryList { + badges, err := models.GetBadgeByCategoryId(v.ID) + if badges == nil || len(badges) == 0 { + continue + } + userBadgeMap, err := getUserBadgesMap(userId, v.ID) + if err != nil { + return nil, err + } + t := models.UserAllBadgeInCategory{ + CategoryName: v.Name, + CategoryId: v.ID, + LightedNum: len(userBadgeMap), + } + bArray := make([]*models.BadgeShowWithStatus, len(badges)) + for j, v := range badges { + b := &models.BadgeShowWithStatus{Badge: v.ToUserShow()} + if _, has := userBadgeMap[v.ID]; has { + b.IsLighted = true + } + bArray[j] = b + } + t.Badges = bArray + r = append(r, t) + } + return r, nil +} + +func getUserBadgesMap(userId, categoryId int64) (map[int64]*models.Badge, error) { + userBadges, err := models.GetUserBadges(userId, categoryId) + if err != nil { + return nil, err + } + m := make(map[int64]*models.Badge, 0) + for _, v := range userBadges { + m[v.ID] = v + } + return m, nil +} diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index 89649a307..97d968954 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -89,7 +89,7 @@
{{$JobID := '0'}} - {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}} + {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}} {{$JobID = .Cloudbrain.ID}} {{else}} {{$JobID = .JobID}} @@ -124,7 +124,7 @@ {{.DisplayJobName}} - {{else if eq .JobType "BENCHMARK"}} + {{else if eq .JobType "BENCHMARK" "MODELSAFETY"}} @@ -135,14 +135,13 @@
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} + {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -185,6 +183,17 @@ var spanEl = document.querySelector('.card_type_{{.DisplayJobName}}_{{$JobID}}'); spanEl.setAttribute('title', cardType); spanEl.innerText = cardType; + + var cluster = spec.Cluster || '--'; + var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--'; + spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', cluster); + spanEl.innerText = clusterName; + + var aiCenter = spec.AiCenterName || '--'; + spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', aiCenter); + spanEl.innerText = aiCenter; })(); @@ -239,8 +248,19 @@
{{end}} - +
+ {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.stop"}} + +
+ {{else}} {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
{{$.CsrfTokenHtml}} @@ -259,6 +279,7 @@ {{$.i18n.Tr "repo.stop"}} {{end}} + {{end}}
{{if eq .JobType "TRAIN"}} @@ -269,17 +290,30 @@
{{end}} - - {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.delete"}} - - + {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{else}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{end}} diff --git a/templates/admin/cloudbrain/search.tmpl b/templates/admin/cloudbrain/search.tmpl index 72077b8a2..3562672a8 100644 --- a/templates/admin/cloudbrain/search.tmpl +++ b/templates/admin/cloudbrain/search.tmpl @@ -42,6 +42,7 @@ BENCHMARK SNN4IMAGENET BRAINSCORE + MODELSAFETY
diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 3de7b25a0..f9908f6f0 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -32,9 +32,15 @@
- {{.i18n.Tr "custom.head.project"}} {{.i18n.Tr "custom.head.dataset"}} +
+ {{.i18n.Tr "repo.model_manager"}} + +
+ 模型体验 +
+
+ {{else if eq .benchmarkMode "aisafety"}} +
+ {{.CsrfTokenHtml}} + + +
+ +
+ {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} + {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} + 模型安全评测 +
+
+ +
+
+ + {{template "custom/task_wait_count" .}} +
+
+ + + {{.i18n.Tr "repo.cloudbrain_jobname_err"}} +
+
+ + +
+ + +
+
+ +
+ + +
+ +
+ + + {{.i18n.Tr "cloudbrain.view_sample"}} +
+ + +
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+
+
+ {{end}} diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 3c71140bf..a54989df4 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -253,7 +253,7 @@
{{if and ($.canDownload) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 45b6501fd..67548696f 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -265,7 +265,7 @@
{{if and ($.canDownload) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}},EngineName:{{.EngineName}},ComputeResource:{{.ComputeResource}},Type:{{.Type}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index f3929ec60..6683fc94a 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -207,6 +207,18 @@ padding-left: 1rem; padding-top: 0.5rem; } + .dataset_nowrap_two_line{ + word-wrap: break-word; + word-break: break-all; + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + line-clamp: 2; + -webkit-line-clamp: 2; + text-overflow: -o-ellipsis-lastline; + max-height: 50px; + }
{{template "repo/header" .}} @@ -437,15 +449,14 @@ {{range $.datasetDownload}} - - + {{if eq .IsDelete true}} {{.DatasetName}}({{$.i18n.Tr "dataset.file_deleted"}}) {{else}} {{.DatasetName}} {{end}} - {{.DatasetDownloadLink}} +
{{.DatasetDownloadLink}}
{{$.i18n.Tr "dataset.download_copy"}} {{end}} diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 8e2810317..0e1e8eedb 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -261,29 +261,25 @@
- -
- + {{if .WorkNode}} + {{range .WorkNode}} - {{if $.work_server_number}} - {{if eq . $.work_server_number }} - - {{else}} - - {{end}} - {{else}} - - {{end}} - {{end}} + {{if $.work_server_number}} + {{if eq . $.work_server_number }} + + {{else}} + + {{end}} + {{else}} + + {{end}} + {{end}} - {{else}} - - {{end}} + {{else}} + + {{end}}
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 0c9bb3ba4..e95a1233d 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -249,7 +249,7 @@ {{$.CsrfTokenHtml}} {{if and (.CanModify) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}},EngineID:{{.EngineID}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} @@ -783,7 +783,6 @@ } let dirKey="isOnlyDir--:&"; function loadSelectedModelFile(trainJob){ - console.log("trainJob=" + trainJob); $('#choice_file').dropdown('clear') $("#model-file").empty() if(trainJob ==null || trainJob ==""){ @@ -893,11 +892,11 @@ $('#JobName').val(obj.DisplayJobName).addClass('model_disabled') $('input[name="JobId"]').val(obj.JobID) $('input[name="VersionName"]').val(obj.VersionName).addClass('model_disabled') - if(obj.EngineID ==122 || obj.EngineID ==35 || obj.EngineID ==-1){ + if(obj.EngineID ==122 || obj.EngineID ==35 || obj.EngineID ==-1 || obj.EngineID ==37){ $('input[name="Engine_name"]').val("MindSpore").addClass('model_disabled'); $('input[name="Engine"]').val(2); } - if(obj.EngineID ==121){ + if(obj.EngineID ==121 || obj.EngineID ==38){ $('input[name="Engine_name"]').val("TensorFlow").addClass('model_disabled'); $('input[name="Engine"]').val(1); } diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 3c4ac7f69..92eefca2e 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -581,10 +581,10 @@ if(versionList[0].Engine == engineOption){ return "selected=\"selected\""; }else{ - if(versionList[0].Engine==122 && engineOption==2){ + if((versionList[0].Engine==122 || versionList[0].Engine==37) && engineOption==2){ return "selected=\"selected\""; } - if(versionList[0].Engine==121 && engineOption==1){ + if((versionList[0].Engine==121 || versionList[0].Engine==38) && engineOption==1){ return "selected=\"selected\""; } } diff --git a/templates/repo/modelmanage/showinfo.tmpl b/templates/repo/modelmanage/showinfo.tmpl index e18c6bd99..4cc0305ed 100644 --- a/templates/repo/modelmanage/showinfo.tmpl +++ b/templates/repo/modelmanage/showinfo.tmpl @@ -244,9 +244,9 @@ function loadInfo(){ function getEngineName(model){ if(model.Engine == 0){ return "PyTorch"; - }else if(model.Engine == 1 || model.Engine == 121){ + }else if(model.Engine == 1 || model.Engine == 121 || model.Engine == 38){ return "TensorFlow"; - }else if(model.Engine == 2 || model.Engine == 122 || model.Engine == 35){ + }else if(model.Engine == 2 || model.Engine == 122 || model.Engine == 35 || model.Engine == 37){ return "MindSpore"; }else if(model.Engine == 3){ return "Other"; diff --git a/templates/repo/modelsafety/indicators_descr.tmpl b/templates/repo/modelsafety/indicators_descr.tmpl new file mode 100644 index 000000000..a8027659a --- /dev/null +++ b/templates/repo/modelsafety/indicators_descr.tmpl @@ -0,0 +1,63 @@ + + + + + diff --git a/templates/repo/modelsafety/new.tmpl b/templates/repo/modelsafety/new.tmpl new file mode 100644 index 000000000..07bcd311d --- /dev/null +++ b/templates/repo/modelsafety/new.tmpl @@ -0,0 +1,469 @@ +{{template "base/head" .}} + +{{template "custom/global_mask" .}} +
+ {{template "repo/header" .}} +
+ {{$Grampus := (or (eq (index (SubJumpablePath .Link) 1) "create_grampus_gpu") (eq (index (SubJumpablePath .Link) 1) "create_grampus_npu"))}} + + {{template "base/alert" .}} +

+ {{.i18n.Tr "repo.modelarts.evaluate_job.new_job"}} +

+
+ +
+ {{.CsrfTokenHtml}} + + + + +
+ +
+ {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} + {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} + {{.i18n.Tr "modelsafety.model_security_evaluation"}} +
+
+ +
+ +
+ + + + + + CPU/GPU + + + + + + + Ascend NPU +
+
+
+
+ + {{template "custom/task_wait_count" .}} +
+
+ +
+
+ + {{.i18n.Tr "modelsafety.new_model_security_evaluation_tips"}} +
+
+
+
+ + + {{.i18n.Tr "repo.cloudbrain_jobname_err"}} +
+
+ + +
+ +
+
+ + +
+ +
+ +
+
+ +
+ + + +
+ + {{if not $Grampus}} + {{if eq .datasetType 0}} +
+ + {{else}} +
+ +
+ +
+
+ +
+
+ {{end}} + {{else}} + {{if eq .datasetType 0}} +
+ {{else}} +
+ + +
+ {{end}} + {{end}} + +
+ + + + + + {{.i18n.Tr "cloudbrain.view_sample"}} +
+
+ +
+ +
+
+ +
+ +
+
+
+
+ + + + {{template "repo/modelsafety/indicators_descr".}} +
+
+ + + {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} + + +
+
+
+ + + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}} +
+
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+
+
+
+
+
+{{template "base/footer" .}} + + + + diff --git a/templates/repo/modelsafety/newgpu.tmpl b/templates/repo/modelsafety/newgpu.tmpl new file mode 100644 index 000000000..d41c8f816 --- /dev/null +++ b/templates/repo/modelsafety/newgpu.tmpl @@ -0,0 +1 @@ +{{template "repo/modelsafety/new".}} diff --git a/templates/repo/modelsafety/newnpu.tmpl b/templates/repo/modelsafety/newnpu.tmpl new file mode 100644 index 000000000..d41c8f816 --- /dev/null +++ b/templates/repo/modelsafety/newnpu.tmpl @@ -0,0 +1 @@ +{{template "repo/modelsafety/new".}} diff --git a/templates/repo/modelsafety/show.tmpl b/templates/repo/modelsafety/show.tmpl new file mode 100644 index 000000000..6954f6f6b --- /dev/null +++ b/templates/repo/modelsafety/show.tmpl @@ -0,0 +1,893 @@ +{{template "base/head" .}} + +
+
+
+
+
+
+
+
+
+
+ {{template "repo/header" .}} +
+

+
+ + {{$.i18n.Tr "repo.cloudbrain"}} + +
/
+ + {{$.i18n.Tr "repo.modelarts.evaluate_job"}} + +
/
+
+
+

+
+ +
+
+
+ + + +
+ + + + {{$.i18n.Tr "repo.modelarts.status"}}： + + + + + + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： + +
+ + +
+
+
+
+
+
+ {{$.i18n.Tr "repo.modelarts.train_job.config"}} + {{$.i18n.Tr "repo.modelarts.log"}} + {{.i18n.Tr "modelsafety.evaluation_result"}} +
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{$.i18n.Tr "repo.cloudbrain_task"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.status"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.createtime"}} + +
+ +
+
+ {{$.i18n.Tr "repo.cloudbrain.time.starttime"}} + +
+
+ {{$.i18n.Tr "repo.cloudbrain.time.endtime"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}} + +
+
+ {{$.i18n.Tr "cloudbrain.mirror"}} + {{$.i18n.Tr "repo.modelarts.train_job.AI_driver"}} + +
+ + +
+
+ {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_scenes"}} + +
+
+ {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}} + +
+
+ {{.i18n.Tr "modelsafety.base_data_set"}} + +
+
+ {{.i18n.Tr "modelsafety.combat_data_set"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}} + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{$.i18n.Tr "repo.modelarts.code_version"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.description"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.resource_type"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.standard"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.model_name"}} + +
+
+ {{$.i18n.Tr "repo.modelconvert.modelversion"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.infer_job_model_file"}} + +
+
+ {{$.i18n.Tr "repo.cloudbrain_creator"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.start_file"}} + +
+
+ {{.i18n.Tr "modelsafety.evaluation_indicators"}} + +
+
+
+
+
+ +
+
+
+ +
+ + + + + + + +
+
+
+
+ + + 

+                             

+                      
                         
+                    

+                    
                      
+                      
+                      

+                        {{template "repo/modelsafety/indicators_descr".}}
+                      

+                      

+                        
{{.i18n.Tr "modelsafety.no_data"}}

+                      

+                    

+                

+            

+        

+        {{template "base/paginate" .}}
+    

+    
+    

+        

+            

+                 {{$.i18n.Tr "cloudbrain.delete_task"}}
+            

+
+            

+                
{{$.i18n.Tr "cloudbrain.task_delete_confirm"}}

+            

+            

+                

+                     {{$.i18n.Tr "cloudbrain.operate_cancel"}}
+                

+                

+                     {{$.i18n.Tr "cloudbrain.operate_confirm"}}
+                

+            

+        

+    

+

+{{template "base/footer" .}}
+
+
+
+
diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl
index defed27bc..5b7ab6aca 100755
--- a/templates/user/dashboard/cloudbrains.tmpl
+++ b/templates/user/dashboard/cloudbrains.tmpl
@@ -71,7 +71,7 @@
                             

                                 
                                 {{$JobID := '0'}}
-                                {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}}
+                                {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}}
                                 {{$JobID = .Cloudbrain.ID}}
                                 {{else}}
                                 {{$JobID = .JobID}}
@@ -85,7 +85,7 @@
                                         {{.DisplayJobName}}
                                     
-                                    {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}
+                                    {{else if (eq .JobType "SNN4IMAGENET" "BRAINSCORE")}}
                                     
@@ -106,7 +106,7 @@
                                         {{.DisplayJobName}}
                                     
-                                    {{else if eq .JobType "BENCHMARK"}}
+                                    {{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}
                                     
@@ -117,14 +117,13 @@
                                 

                                 
                                 

-                                    {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
+                                    {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
                                 

                                 
                                 

                                     
                                         
                                 

-                                    {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
+                                    {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
                                 

                                 
                                 

@@ -169,6 +167,17 @@
                                         var spanEl = document.querySelector('.card_type_{{.DisplayJobName}}_{{$JobID}}');
                                         spanEl.setAttribute('title', cardType);
                                         spanEl.innerText = cardType;
+                                        
+                                        var cluster = spec.Cluster || '--';
+                                        var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--';
+                                        spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}');
+                                        spanEl.setAttribute('title', cluster);
+                                        spanEl.innerText = clusterName;
+
+                                        var aiCenter = spec.AiCenterName || '--';
+                                        spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}');
+                                        spanEl.setAttribute('title', aiCenter);
+                                        spanEl.innerText = aiCenter;
                                     })();
                                 
                                 
@@ -202,23 +211,35 @@
                                     {{end}}
                                     
                                     

-                                        {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
-                                        

-                                            {{$.CsrfTokenHtml}}
+                                        {{if eq .JobType "MODELSAFETY"}}
+                                            
+                                                {{$.CsrfTokenHtml}}
+                                                
+                                                    {{$.i18n.Tr "repo.stop"}}
+                                                
+                                            

+                                        {{else}}
+                                            {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
+                                            

+                                                {{$.CsrfTokenHtml}}
+                                                
+                                                    {{$.i18n.Tr "repo.stop"}}
+                                                
+                                            

+                                            {{else}}
                                             
+                                                class='ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "STOPPED" "SUCCEEDED" "CREATE_FAILED"}}disabled {{else}} blue {{end}}button'
+                                                data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 1}}modelarts/inference-job{{else}}cloudbrain/train-job{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}modelarts/train-job{{else if eq .Cloudbrain.Type 0}}cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}grampus/train-job{{end}}{{end}}'
+                                                data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
                                                 {{$.i18n.Tr "repo.stop"}}
                                             
-                                        
-                                        {{else}}
-                                        
-                                            {{$.i18n.Tr "repo.stop"}}
-                                        
+                                            {{end}}
                                         {{end}}
                                     

                                     {{if eq .JobType "BENCHMARK"}}
@@ -228,7 +249,6 @@
                                             {{$.i18n.Tr "repo.score"}}
                                         
                                     

-
                                     {{end}}
                                     
                                     {{if eq .JobType "TRAIN"}}
@@ -239,6 +259,18 @@
                                     

                                     {{end}}
                                     
+                                    {{if eq .JobType "MODELSAFETY"}}
+                                    

+                                        {{$.CsrfTokenHtml}}
+                                        
+                                            {{$.i18n.Tr "repo.delete"}}
+                                        
+                                    

+                                    {{else}}
                                     

@@ -250,6 +282,8 @@
                                             {{$.i18n.Tr "repo.delete"}}
                                         
                                     

+                                    {{end}}
+                                    
                                 

                             

                         

diff --git a/templates/user/profile.tmpl b/templates/user/profile.tmpl
index be6ecbaa0..d4e97a961 100755
--- a/templates/user/profile.tmpl
+++ b/templates/user/profile.tmpl
@@ -17,6 +17,17 @@
 						{{if .Owner.FullName}}{{.Owner.FullName}}{{end}}
 						{{.Owner.Name}}
 					

+					
+					

+						{{range $k,$v :=.RecentBadges}}
+						{{if le $k 3}}
+						

+						{{else}}
+						
+						{{end}}
+						{{end}}
+						
+					

 					

 						

 
 				{{if eq .TabName "activity"}}
@@ -201,6 +218,8 @@
 					{{template "explore/dataset_search" .}}
 					{{template "explore/dataset_list" .}}
 					{{template "base/paginate" .}}
+				{{else if eq .TabName "badge"}}
+					{{template "repo/badge" .}}
 				{{else}}
 					{{template "explore/repo_search" .}}
 					{{template "explore/repo_list" .}}
@@ -228,5 +247,8 @@
 	.user.profile .ui.card .extra.content ul {
 		padding: 5px 0;
 	}
+	.ui.secondary.pointing.menu .item{
+		padding: 0.78571429em 0.92857143em;
+	}
 
 
\ No newline at end of file
diff --git a/web_src/js/components/dataset/selectDataset.vue b/web_src/js/components/dataset/selectDataset.vue
index 77d54033f..057298d13 100755
--- a/web_src/js/components/dataset/selectDataset.vue
+++ b/web_src/js/components/dataset/selectDataset.vue
@@ -1011,7 +1011,8 @@ export default {
     if (
       location.href.indexOf("benchmark") !== -1 ||
       location.href.indexOf("train-job") !== -1 ||
-      location.href.indexOf("inference") !== -1
+      location.href.indexOf("inference") !== -1 || 
+      location.href.indexOf("modelsafety") !== -1
     ) {
       this.benchmarkNew = true;
     }
diff --git a/web_src/js/components/images/selectGrampusImages.vue b/web_src/js/components/images/selectGrampusImages.vue
index f7af01e9f..eced8cf88 100755
--- a/web_src/js/components/images/selectGrampusImages.vue
+++ b/web_src/js/components/images/selectGrampusImages.vue
@@ -242,7 +242,8 @@ export default {
     this.getImageListPublic();
     if (
       location.href.indexOf("benchmark") !== -1 ||
-      location.href.indexOf("train-job") !== -1
+      location.href.indexOf("train-job") !== -1 ||
+      location.href.indexOf("modelsafety") !== -1
     ) {
       this.benchmarkNew = true;
     }
diff --git a/web_src/js/components/images/selectImages.vue b/web_src/js/components/images/selectImages.vue
index add4553d0..ca2ab3138 100755
--- a/web_src/js/components/images/selectImages.vue
+++ b/web_src/js/components/images/selectImages.vue
@@ -47,6 +47,7 @@
           type="text"
           :placeholder="i18n.image_search_placeholder"
           v-model="search"
+          @keydown.enter.stop.prevent="searchName"
         />
       
       
@@ -505,7 +506,8 @@ export default {
     if (
       location.href.indexOf("train-job") !== -1 ||
       location.href.indexOf("inference-job") !== -1 ||
-      location.href.indexOf("benchmark") !== -1
+      location.href.indexOf("benchmark") !== -1 ||
+      location.href.indexOf("modelsafety") !== -1
     ) {
       this.benchmarkNew = true;
     }
diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index f451789d1..3aafcc1a6 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -148,6 +148,10 @@ export default async function initCloudrainSow() {
           $(`#${version_name}-log-down`)
             .removeClass("ti-download-file")
             .addClass("disabled");
+        } else {
+          $(`#${version_name}-log-down`)
+            .addClass("ti-download-file")
+            .removeClass("disabled");
         }
         $(`#log${version_name} input[name=end_line]`).val(data.EndLine); //如果变动就改变所对应的值
         $(`#log${version_name} input[name=start_line]`).val(data.StartLine);
diff --git a/web_src/js/features/cloudrbanin.js b/web_src/js/features/cloudrbanin.js
index 2df780233..698523d11 100644
--- a/web_src/js/features/cloudrbanin.js
+++ b/web_src/js/features/cloudrbanin.js
@@ -8,6 +8,7 @@ export default async function initCloudrain() {
   function loadJobStatus() {
     $(".job-status").each((index, job) => {
       const ID = job.dataset.jobid;
+      if (!ID) return;
       const repoPath = job.dataset.repopath;
       // const computeResource = job.dataset.resource
       const versionname = job.dataset.version;
@@ -144,6 +145,7 @@ export default async function initCloudrain() {
   function loadShowJobStatus() {
     $(".ui.accordion.border-according").each((index, job) => {
       const jobID = job.dataset.jobid;
+      if (!jobID) return;
       const repoPath = job.dataset.repopath;
       const versionname = job.dataset.version;
       // ['IMAGE_FAILED','SUBMIT_FAILED','DELETE_FAILED','KILLED','COMPLETED','FAILED','CANCELED','LOST','START_FAILED']
@@ -152,7 +154,7 @@ export default async function initCloudrain() {
       //     || job.textContent.trim() == 'CANCELED' || job.textContent.trim() == 'LOST') {
       //     return
       // }
-      let status = $(`#${versionname}-status-span`).text();
+      let status = $(`#${versionname}-status-span`).text().trim();
 
       if (
         [
diff --git a/web_src/js/index.js b/web_src/js/index.js
index b796d020a..9a1f04296 100755
--- a/web_src/js/index.js
+++ b/web_src/js/index.js
@@ -65,6 +65,7 @@ const { AppSubUrl, StaticUrlPrefix, csrf } = window.config;
 Object.defineProperty(Vue.prototype, "$echarts", {
   value: echarts,
 });
+window.echarts = echarts;
 
 function htmlEncode(text) {
   return jQuery("
").text(text).html();
diff --git a/web_src/js/standalone/cloudbrainNew.js b/web_src/js/standalone/cloudbrainNew.js
index 170344f7a..bea652dfd 100644
--- a/web_src/js/standalone/cloudbrainNew.js
+++ b/web_src/js/standalone/cloudbrainNew.js
@@ -89,7 +89,7 @@
           identifier: "job_name",
           rules: [
             {
-              type: "regExp[/^[a-zA-Z0-9-_]{1,36}$/]",
+              type: "regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]",
             },
           ],
         },
@@ -97,7 +97,7 @@
           identifier: "display_job_name",
           rules: [
             {
-              type: "regExp[/^[a-zA-Z0-9-_]{1,36}$/]",
+              type: "regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]",
             },
           ],
         },
@@ -121,6 +121,7 @@
       },
       onFailure: function (e) {
         isValidate = false;
+        createFlag = false;
         return false;
       },
     });
diff --git a/web_src/less/_user.less b/web_src/less/_user.less
index 6acbb35ee..29ca96255 100644
--- a/web_src/less/_user.less
+++ b/web_src/less/_user.less
@@ -9,11 +9,30 @@
             .username {
                 display: block;
             }
-
+            .badge-wrap {
+                display: flex;
+                justify-content: center;
+                align-items: center;
+                .badge-img-avatar {
+                    width: 32px;
+                    height: 32px;
+                    margin-right: 5px;
+                }
+                .badge-more-icon {
+                    width: 32px;
+                    height: 32px;
+                    display: flex;
+                    justify-content: center;
+                    align-items: center;
+                    border-radius: 50%;
+                    border: 1px #f8f9fa solid;
+                    background: #f8f9fa;
+                }
+            }
             .header {
                 font-weight: 700;
                 font-size: 1.3rem;
-                margin-top: -.2rem;
+                margin-top: -0.2rem;
                 line-height: 1.3rem;
             }
 
@@ -158,3 +177,70 @@
         max-width: 60px;
     }
 }
+.badge-achive {
+    .bagde-section {
+        color: #000;
+        margin-top: 28px;
+        border-bottom: 1px solid #dededf;
+    }
+    .bagde-section:last-child {
+        color: #000;
+        margin-top: 28px;
+        border-bottom: none;
+    }
+    .badge-section-title {
+        position: relative;
+        font-size: 16px;
+        line-height: 24px;
+        padding-left: 8px;
+        margin-bottom: 20px;
+        display: flex;
+        justify-content: space-between;
+    }
+    .badge-section-children {
+        width: 100%;
+    }
+    .badge-honor-badge {
+        margin-bottom: 25px;
+    }
+    .badge-honor-badge-basic {
+        display: flex;
+        align-items: flex-start;
+        flex-wrap: wrap;
+    }
+    .badge-honor-badge-basic-item {
+        text-align: center;
+        font-size: 12px;
+        margin-right: 30px;
+        color: #101010;
+    }
+    .is-not-pointer {
+        cursor: pointer;
+        pointer-events: none;
+    }
+    .badge-honor-badge-basic-img {
+        width: 100px;
+        height: 100px;
+        margin-bottom: 10px;
+    }
+    .badge-honor-badge-basic-txt {
+        line-height: 20px;
+        width: 100px;
+        word-break: break-all;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        display: -webkit-box;
+        -webkit-line-clamp: 2;
+        -webkit-box-orient: vertical;
+    }
+    .badge-section-title:before {
+        content: "";
+        position: absolute;
+        top: 50%;
+        left: 0;
+        transform: translateY(-50%);
+        width: 3px;
+        height: 1em;
+        background-color: #000;
+    }
+}
diff --git a/web_src/vuepages/apis/modules/desensitization.js b/web_src/vuepages/apis/modules/desensitization.js
new file mode 100644
index 000000000..2d0ab7b97
--- /dev/null
+++ b/web_src/vuepages/apis/modules/desensitization.js
@@ -0,0 +1,13 @@
+import service from "../service";
+
+// 数据脱敏接口
+export const postImgDesensitization = (params, data) => {
+  return service({
+    url: "/extension/tuomin/upload",
+    headers: { Accept: "image/png" },
+    responseType: "blob",
+    method: "POST",
+    params: params,
+    data: data,
+  });
+};
diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js
index c60b90f91..9258d1656 100644
--- a/web_src/vuepages/langs/config/en-US.js
+++ b/web_src/vuepages/langs/config/en-US.js
@@ -176,6 +176,22 @@ const en = {
     Activated: 'Activated',
     notActive: 'Not active',
   },
+  tranformImageFailed:'Picture desensitization failed',
+  originPicture:'Origin picture',
+  desensitizationPicture:'Desensitization picture',
+  desensitizationObject:'Desensitization object',
+  example:'Example',
+  startDesensitization:'Start desensitization',
+  all:'All',
+  onlyFace:'Only face',
+  onlyLicensePlate:'Only license plate',
+  dragThePictureHere:'Drag the picture here',
+  or:'or',
+  clickUpload:'Click upload',
+  dataDesensitizationModelExperience:'Data desensitization model experience',
+  dataDesensitizationModelDesc:'Use AI technology to desensitize the face and license plate number in the picture. For more information about this model, please visit the project',
+  limitFilesUpload:'Only jpg/jpeg/png files can be uploaded',
+  limitSizeUpload:'The size of the uploaded file cannot exceed 20M!',
 }
 
 export default en;
diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js
index 2b451e2db..c9d9f7a8c 100644
--- a/web_src/vuepages/langs/config/zh-CN.js
+++ b/web_src/vuepages/langs/config/zh-CN.js
@@ -176,6 +176,22 @@ const zh = {
     Activated: '已激活',
     notActive: '未激活',
   },
+  tranformImageFailed:'图片脱敏失败',
+  originPicture:'原始图片',
+  desensitizationPicture:'脱敏图片',
+  desensitizationObject:'脱敏对象',
+  example:'示例',
+  startDesensitization:'开始处理',
+  all:'全部',
+  onlyFace:'仅人脸',
+  onlyLicensePlate:'仅车牌',
+  dragThePictureHere:'拖动图片到这里',
+  or:'或',
+  clickUpload:'点击上传',
+  dataDesensitizationModelExperience:'数据脱敏模型体验',
+  dataDesensitizationModelDesc:'利用人工智能AI技术，把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目',
+  limitFilesUpload:'只能上传 jpg/jpeg/png 格式的文件',
+  limitSizeUpload:'上传文件大小不能超过 20M !',
 }
 
 export default zh;
diff --git a/web_src/vuepages/pages/model/tuomin/index.vue b/web_src/vuepages/pages/model/tuomin/index.vue
new file mode 100644
index 000000000..6c7813977
--- /dev/null
+++ b/web_src/vuepages/pages/model/tuomin/index.vue
@@ -0,0 +1,380 @@
+
+
+
+
+
diff --git a/web_src/vuepages/pages/model/tuomin/vp-model-tuomin.js b/web_src/vuepages/pages/model/tuomin/vp-model-tuomin.js
new file mode 100644
index 000000000..48e542500
--- /dev/null
+++ b/web_src/vuepages/pages/model/tuomin/vp-model-tuomin.js
@@ -0,0 +1,17 @@
+import Vue from 'vue';
+import ElementUI from 'element-ui';
+import 'element-ui/lib/theme-chalk/index.css';
+import localeEn from 'element-ui/lib/locale/lang/en';
+import localeZh from 'element-ui/lib/locale/lang/zh-CN';
+import { i18n, lang } from '~/langs';
+import App from './index.vue';
+
+Vue.use(ElementUI, {
+  locale: lang === 'zh-CN' ? localeZh : localeEn,
+  size: 'small',
+});
+
+new Vue({
+  i18n,
+  render: (h) => h(App),
+}).$mount('#__vue-root');
diff --git a/web_src/vuepages/pages/resources/specification/index.vue b/web_src/vuepages/pages/resources/specification/index.vue
index 17ca4870c..4dc2657f3 100644
--- a/web_src/vuepages/pages/resources/specification/index.vue
+++ b/web_src/vuepages/pages/resources/specification/index.vue
@@ -22,15 +22,15 @@
     

     

       

-        
-          
-          
+        
+          
+          
           
-          
           
-          
           
           
           
-          
            {
     if (record.SourceType === 'ADMIN_OPERATE') {
       out.remark = record.Remark;
     } else if (record.SourceType === 'ACCOMPLISH_TASK') {
+      if (record.Action?.Cloudbrain?.JobType === 'MODELSAFETY') {
+        record.Action.Cloudbrain.oJobType = 'MODELSAFETY';
+        record.Action.Cloudbrain.JobType = 'BENCHMARK';
+      }
       switch (record?.Action?.TaskType) {
         case 'CreatePublicRepo': // 创建公开项目 - 创建了项目OpenI/aiforge
           out.remark = `${i18n.t('createdRepository')}${record.Action.ShortRepoFullDisplayName}`;