diff --git a/models/cloudbrain.go b/models/cloudbrain.go index b66bed900..ad0a6c570 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,13 +1,14 @@ package models import ( - "code.gitea.io/gitea/modules/util" "encoding/json" "fmt" "strconv" "strings" "time" + "code.gitea.io/gitea/modules/util" + "xorm.io/builder" "xorm.io/xorm" @@ -1362,7 +1363,7 @@ func CloudbrainsVersionList(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int, e func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { cloudbrain.TrainJobDuration = DURATION_STR_ZERO - if _, err = x.Insert(cloudbrain); err != nil { + if _, err = x.NoAutoTime().Insert(cloudbrain); err != nil { return err } return nil @@ -1577,7 +1578,7 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { return err } - if _, err = sess.Insert(new); err != nil { + if _, err = sess.NoAutoTime().Insert(new); err != nil { sess.Rollback() return err } @@ -1588,3 +1589,64 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { return nil } +func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + if (opts.Type) >= 0 { + cond = cond.And( + builder.Eq{"cloudbrain.type": opts.Type}, + ) + } + + var count int64 + var err error + condition := "cloudbrain.user_id = `user`.id" + if len(opts.Keyword) == 0 { + count, err = sess.Where(cond).Count(new(Cloudbrain)) + } else { + lowerKeyWord := strings.ToLower(opts.Keyword) + + cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) + count, err = sess.Table(&Cloudbrain{}).Where(cond). + Join("left", "`user`", condition).Count(new(CloudbrainInfo)) + + } + + if err != nil { + return nil, 0, fmt.Errorf("Count: %v", err) + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + sess.Limit(opts.PageSize, start) + } + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Join("left", "`user`", condition). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + if opts.NeedRepoInfo { + var ids []int64 + for _, task := range cloudbrains { + ids = append(ids, task.RepoID) + } + repositoryMap, err := GetRepositoriesMapByIDs(ids) + if err == nil { + for _, task := range cloudbrains { + task.Repo = repositoryMap[task.RepoID] + } + } + + } + + return cloudbrains, count, nil +} diff --git a/models/dataset.go b/models/dataset.go index c56a8fdc3..95800100c 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -63,19 +63,20 @@ func (datasets DatasetList) loadAttributes(e Engine) error { } set := make(map[int64]struct{}) + userIdSet := make(map[int64]struct{}) datasetIDs := make([]int64, len(datasets)) for i := range datasets { - set[datasets[i].UserID] = struct{}{} + userIdSet[datasets[i].UserID] = struct{}{} set[datasets[i].RepoID] = struct{}{} datasetIDs[i] = datasets[i].ID } // Load owners. - users := make(map[int64]*User, len(set)) + users := make(map[int64]*User, len(userIdSet)) repos := make(map[int64]*Repository, len(set)) if err := e. Where("id > 0"). - In("id", keysInt64(set)). + In("id", keysInt64(userIdSet)). Find(&users); err != nil { return fmt.Errorf("find users: %v", err) } diff --git a/models/org.go b/models/org.go index 8b3e60ef8..2a6528023 100755 --- a/models/org.go +++ b/models/org.go @@ -193,22 +193,22 @@ func (org *User) getOrgStatistics() (int, error) { } func FindTopNStarsOrgs(n int) ([]*OrgScore, error) { - sql := "select a.id,sum(b.num_stars) score from \"user\" a ,repository b where a.id=b.owner_id and a.type=1 group by a.id order by score desc limit " + strconv.Itoa(n) + sql := "select a.id,sum(b.num_stars) score from \"user\" a ,repository b where a.id=b.owner_id and a.type=1 and a.visibility=0 group by a.id order by score desc limit " + strconv.Itoa(n) return findTopNOrgs(sql) } func FindTopNMembersOrgs(n int) ([]*OrgScore, error) { sql := "select id, count(user_id) score from" + - " (select org_id as id, uid as user_id from org_user " + + " (select org_id as id, uid as user_id from org_user o, \"user\" u where o.org_id=u.id and u.visibility=0 " + "union select a.id,b.user_id from \"user\" a,collaboration b,repository c " + - "where a.type=1 and a.id=c.owner_id and b.repo_id=c.id) d " + + "where a.type=1 and a.visibility=0 and a.id=c.owner_id and b.repo_id=c.id) d " + "group by id order by score desc limit " + strconv.Itoa(n) return findTopNOrgs(sql) } func FindTopNOpenIOrgs(n int) ([]*OrgScore, error) { - sql := "select org_id id,num_score score from org_statistic order by num_score desc limit 10" + strconv.Itoa(n) + sql := "select org_id id,num_score score from org_statistic a, \"user\" b where a.org_id=b.id and b.visibility=0 order by num_score desc limit " + strconv.Itoa(n) return findTopNOrgs(sql) } diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index d998638dc..eaf680c65 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -1,6 +1,7 @@ package cloudbrain import ( + "code.gitea.io/gitea/modules/timeutil" "encoding/json" "errors" "strconv" @@ -230,6 +231,7 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, datasetName = attach.Name } + createTime := timeutil.TimeStampNow() jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -330,6 +332,8 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, BootFile: bootFile, DatasetName: datasetName, Parameters: params, + CreatedUnix: createTime, + UpdatedUnix: createTime, }) if err != nil { @@ -377,6 +381,7 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e return errors.New("no such resourceSpec") } + createTime := timeutil.TimeStampNow() jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -468,6 +473,8 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e GpuQueue: task.GpuQueue, ResourceSpecId: task.ResourceSpecId, ComputeResource: task.ComputeResource, + CreatedUnix: createTime, + UpdatedUnix: createTime, } err = models.RestartCloudbrain(task, newTask) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index b1e7b269e..78b40fd56 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,6 +1,7 @@ package modelarts import ( + "code.gitea.io/gitea/modules/timeutil" "encoding/json" "errors" "fmt" @@ -197,6 +198,7 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin if poolInfos == nil { json.Unmarshal([]byte(setting.PoolInfos), &poolInfos) } + createTime := timeutil.TimeStampNow() jobResult, err := CreateJob(models.CreateNotebookParams{ JobName: jobName, Description: description, @@ -235,6 +237,8 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin Type: models.TypeCloudBrainTwo, Uuid: uuid, ComputeResource: models.NPUResource, + CreatedUnix: createTime, + UpdatedUnix: createTime, }) if err != nil { @@ -254,7 +258,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc log.Error("GetNotebookImageName failed: %v", err.Error()) return err } - + createTime := timeutil.TimeStampNow() jobResult, err := createNotebook2(models.CreateNotebook2Params{ JobName: jobName, Description: description, @@ -288,6 +292,8 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc ComputeResource: models.NPUResource, Image: imageName, Description: description, + CreatedUnix: createTime, + UpdatedUnix: createTime, }) if err != nil { @@ -304,6 +310,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc } func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) { + createTime := timeutil.TimeStampNow() jobResult, err := createTrainJob(models.CreateTrainJobParams{ JobName: req.JobName, Description: req.Description, @@ -364,6 +371,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error EngineName: req.EngineName, VersionCount: req.VersionCount, TotalVersionCount: req.TotalVersionCount, + CreatedUnix: createTime, + UpdatedUnix: createTime, }) if err != nil { @@ -375,6 +384,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error } func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, jobId string) (err error) { + createTime := timeutil.TimeStampNow() jobResult, err := createTrainJobVersion(models.CreateTrainJobVersionParams{ Description: req.Description, Config: models.TrainJobVersionConfig{ @@ -451,6 +461,8 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job EngineName: req.EngineName, TotalVersionCount: VersionTaskList[0].TotalVersionCount + 1, VersionCount: VersionListCount + 1, + CreatedUnix: createTime, + UpdatedUnix: createTime, }) if err != nil { log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) @@ -526,6 +538,7 @@ func GetOutputPathByCount(TotalVersionCount int) (VersionOutputPath string) { } func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (err error) { + createTime := timeutil.TimeStampNow() jobResult, err := createInferenceJob(models.CreateInferenceJobParams{ JobName: req.JobName, Description: req.Description, @@ -591,6 +604,8 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e ModelVersion: req.ModelVersion, CkptName: req.CkptName, ResultUrl: req.ResultUrl, + CreatedUnix: createTime, + UpdatedUnix: createTime, }) if err != nil { diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 8af3eaaf4..dceb48f16 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -40,6 +40,7 @@ var ( DisabledRepoUnits []string DefaultRepoUnits []string PrefixArchiveFiles bool + RepoMaxSize int64 // Repository editor settings Editor struct { @@ -54,6 +55,7 @@ var ( AllowedTypes []string `delim:"|"` FileMaxSize int64 MaxFiles int + TotalMaxSize int64 } `ini:"-"` // Repository local settings @@ -104,6 +106,7 @@ var ( DisabledRepoUnits: []string{}, DefaultRepoUnits: []string{}, PrefixArchiveFiles: true, + RepoMaxSize: 1024, // Repository editor settings Editor: struct { @@ -121,12 +124,14 @@ var ( AllowedTypes []string `delim:"|"` FileMaxSize int64 MaxFiles int + TotalMaxSize int64 }{ Enabled: true, TempPath: "data/tmp/uploads", AllowedTypes: []string{}, - FileMaxSize: 3, - MaxFiles: 5, + FileMaxSize: 30, + MaxFiles: 10, + TotalMaxSize: 1024, }, // Repository local settings diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 8e6b4201b..f733eef6c 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -480,7 +480,7 @@ func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { filename = key[comma+1:] } reqParams := make(map[string]string) - filename = url.QueryEscape(filename) + filename = url.PathEscape(filename) reqParams["response-content-disposition"] = "attachment; filename=\"" + filename + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) @@ -503,6 +503,7 @@ func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { input.Bucket = setting.Bucket input.Expires = 60 * 60 + fileName = url.PathEscape(fileName) reqParams := make(map[string]string) reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" input.QueryParams = reqParams diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 57f7aa084..ad7d0bfa7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -919,6 +919,10 @@ cloudbrain_helper=Use GPU/NPU resources to open notebooks, model training tasks, cloudbrain.exitinfo=Exit Information cloudbrain.platform=Platform cloudbrain.endtime=End Time +cloudbrain.runinfo=Task Runtime Information +cloudbrain.time.starttime=Start run time +cloudbrain.time.endtime=End run time +cloudbrain.datasetdownload=Dataset download url model_manager = Model model_noright=No right model_rename=Duplicate model name, please modify model name. @@ -1272,6 +1276,10 @@ editor.cannot_commit_to_protected_branch = Cannot commit to protected branch '%s editor.no_commit_to_branch = Unable to commit directly to branch because: editor.user_no_push_to_branch = User cannot push to branch editor.require_signed_commit = Branch requires a signed commit +editor.repo_too_large = Repository can not exceed %d MB +editor.repo_file_invalid = Upload files are invalid +editor.upload_file_too_much = Can not upload more than %d files at a time + commits.desc = Browse source code change history. commits.commits = Commits @@ -2880,6 +2888,8 @@ uploading = Uploading upload_complete = Uploading complete failed = Upload Failed enable_minio_support = Enable minio support to use the dataset service +max_file_tooltips= Upload a maximum of ? files at a time, each file does not exceed ? MB. +max_size_tooltips= You can only upload a maximum of ? files at a time. The upload limit has been reached, please do not add more files. [notification] notifications = Notifications diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 17852065c..d2bea4899 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -983,6 +983,10 @@ cloudbrain.mirror_description = 镜像描述 cloudbrain.exitinfo=退出信息 cloudbrain.platform=平台 cloudbrain.endtime=结束时间 +cloudbrain.runinfo=任务运行简况 +cloudbrain.time.starttime=开始运行时间 +cloudbrain.time.endtime=结束运行时间 +cloudbrain.datasetdownload=数据集下载地址 record_begintime_get_err=无法获取统计开始时间。 parameter_is_wrong=输入参数错误,请检查输入参数。 total_count_get_error=查询总页数失败。 @@ -1028,7 +1032,9 @@ modelarts.train_job.basic_info=基本信息 modelarts.train_job.job_status=任务状态 modelarts.train_job.job_name=任务名称 modelarts.train_job.version=任务版本 -modelarts.train_job.start_time=开始时间 +modelarts.train_job.start_time=开始运行时间 +modelarts.train_job.end_time=运行结束时间 +modelarts.train_job.wait_time=等待时间 modelarts.train_job.dura_time=运行时长 modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 @@ -1280,6 +1286,9 @@ editor.cannot_commit_to_protected_branch=不可以提交到受保护的分支 '% editor.no_commit_to_branch=无法直接提交分支,因为: editor.user_no_push_to_branch=用户不能推送到分支 editor.require_signed_commit=分支需要签名提交 +editor.repo_too_large = 代码仓总大小不能超过%dMB +editor.repo_file_invalid = 提交的文件非法 +editor.upload_file_too_much = 不能同时提交超过%d个文件 commits.desc=浏览代码修改历史 commits.commits=次代码提交 @@ -2887,6 +2896,8 @@ uploading=正在上传 upload_complete=上传完成 failed=上传失败 enable_minio_support=启用minio支持以使用数据集服务 +max_file_tooltips=单次最多上传?个文件,每个文件不超过? MB。 +max_size_tooltips=一次最多只能上传?个文件, 上传已达到上限,请勿再添加文件。 [notification] notifications=通知 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 9a670c101..2b070a4b8 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -557,6 +557,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday) m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) + //cloudbrain board + m.Group("/cloudbrainboard", func() { + m.Get("/downloadAll", repo.DownloadCloudBrainBoard) + }, operationReq) // Users m.Group("/users", func() { m.Get("/search", user.Search) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go new file mode 100644 index 000000000..b979729a8 --- /dev/null +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -0,0 +1,135 @@ +package repo + +import ( + "net/http" + "net/url" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "github.com/360EntSecGroup-Skylar/excelize/v2" +) + +func DownloadCloudBrainBoard(ctx *context.Context) { + + page := 1 + + pageSize := 300 + + var cloudBrain = ctx.Tr("repo.cloudbrain") + fileName := getCloudbrainFileName(cloudBrain) + + _, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: 1, + }, + Type: models.TypeCloudBrainAll, + NeedRepoInfo: false, + }) + + if err != nil { + log.Warn("Can not get cloud brain info", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) + return + } + + totalPage := getTotalPage(total, pageSize) + + f := excelize.NewFile() + + index := f.NewSheet(cloudBrain) + f.DeleteSheet("Sheet1") + + for k, v := range allCloudbrainHeader(ctx) { + f.SetCellValue(cloudBrain, k, v) + } + + var row = 2 + for i := 0; i < totalPage; i++ { + + pageRecords, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pageSize, + }, + Type: models.TypeCloudBrainAll, + NeedRepoInfo: true, + }) + if err != nil { + log.Warn("Can not get cloud brain info", err) + continue + } + for _, record := range pageRecords { + + for k, v := range allCloudbrainValues(row, record, ctx) { + f.SetCellValue(cloudBrain, k, v) + } + row++ + + } + + page++ + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) +} +func getCloudbrainFileName(baseName string) string { + return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx" + +} +func allCloudbrainHeader(ctx *context.Context) map[string]string { + + return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_task_type"), "C1": ctx.Tr("repo.modelarts.status"), + "D1": ctx.Tr("repo.modelarts.createtime"), "E1": ctx.Tr("repo.modelarts.train_job.wait_time"), "F1": ctx.Tr("repo.modelarts.train_job.dura_time"), + "G1": ctx.Tr("repo.modelarts.train_job.start_time"), + "H1": ctx.Tr("repo.modelarts.train_job.end_time"), "I1": ctx.Tr("repo.modelarts.computing_resources"), + "J1": ctx.Tr("repo.cloudbrain_creator"), "K1": ctx.Tr("repo.repo_name"), "L1": ctx.Tr("repo.cloudbrain_task_name")} + +} +func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { + return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): rs.JobType, getCellName("C", row): rs.Status, + getCellName("D", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("E", row): getBrainWaitTime(rs), + getCellName("F", row): rs.TrainJobDuration, getCellName("G", row): getBrainStartTime(rs), + getCellName("H", row): getBrainEndTime(rs), + getCellName("I", row): rs.ComputeResource, getCellName("J", row): rs.Name, getCellName("K", row): getBrainRepo(rs), + getCellName("L", row): rs.JobName, + } +} +func getBrainRepo(rs *models.CloudbrainInfo) string { + if rs.Repo != nil { + return rs.Repo.OwnerName + "/" + rs.Repo.Alias + } + return "" +} +func getBrainStartTime(rs *models.CloudbrainInfo) string { + timeString := time.Unix(int64(rs.Cloudbrain.StartTime), 0).Format(CREATE_TIME_FORMAT) + if timeString != "1970/01/01 08:00:00" { + return timeString + } else { + return "0" + } + +} +func getBrainEndTime(rs *models.CloudbrainInfo) string { + timeString := time.Unix(int64(rs.Cloudbrain.EndTime), 0).Format(CREATE_TIME_FORMAT) + if timeString != "1970/01/01 08:00:00" { + return timeString + } else { + return "0" + } + +} +func getBrainWaitTime(rs *models.CloudbrainInfo) string { + waitTime := rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix + if waitTime <= 0 { + return "0" + } else { + return models.ConvertDurationToStr(int64(waitTime)) + } +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 544323010..096694a46 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -441,15 +441,29 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo return } - if cloudbrain.ResourceSpecs == nil { - json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) - } - for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec { - if tmp.Id == task.ResourceSpecId { - ctx.Data["GpuNum"] = tmp.GpuNum - ctx.Data["CpuNum"] = tmp.CpuNum - ctx.Data["MemMiB"] = tmp.MemMiB - ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB + if task.JobType == string(models.JobTypeTrain) { + if cloudbrain.TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) + } + for _, tmp := range cloudbrain.TrainResourceSpecs.ResourceSpec { + if tmp.Id == task.ResourceSpecId { + ctx.Data["GpuNum"] = tmp.GpuNum + ctx.Data["CpuNum"] = tmp.CpuNum + ctx.Data["MemMiB"] = tmp.MemMiB + ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB + } + } + } else { + if cloudbrain.ResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) + } + for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec { + if tmp.Id == task.ResourceSpecId { + ctx.Data["GpuNum"] = tmp.GpuNum + ctx.Data["CpuNum"] = tmp.CpuNum + ctx.Data["MemMiB"] = tmp.MemMiB + ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB + } } } diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 2fa7976e0..8e13735df 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -5,6 +5,7 @@ package repo import ( + repo_service "code.gitea.io/gitea/services/repository" "encoding/json" "fmt" "io/ioutil" @@ -614,6 +615,19 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { message += "\n\n" + form.CommitMessage } + if err := repo_service.CheckPushSizeLimit4Web(ctx.Repo.Repository, form.Files); err != nil { + if repo_service.IsRepoTooLargeErr(err) { + ctx.RenderWithErr(ctx.Tr("repo.editor.repo_too_large", setting.Repository.RepoMaxSize), tplUploadFile, &form) + } else if repo_service.IsUploadFileInvalidErr(err) { + ctx.RenderWithErr(ctx.Tr("repo.editor.repo_file_invalid"), tplUploadFile, &form) + } else if repo_service.IsUploadFileTooMuchErr(err) { + ctx.RenderWithErr(ctx.Tr("repo.editor.upload_file_too_much", setting.Repository.Upload.MaxFiles), tplUploadFile, &form) + } else { + ctx.RenderWithErr(err.Error(), tplUploadFile, &form) + } + return + } + if err := repofiles.UploadRepoFiles(ctx.Repo.Repository, ctx.User, &repofiles.UploadRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 933571a0b..8b785a395 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -279,7 +279,7 @@ func NotebookShow(ctx *context.Context) { } } - datasetDownloadLink := "-" + datasetDownloadLink := "" if ctx.IsSigned { if task.Uuid != "" && task.UserID == ctx.User.ID { attachment, err := models.GetAttachmentByUUID(task.Uuid) @@ -439,6 +439,7 @@ func NotebookManage(ctx *context.Context) { param := models.NotebookAction{ Action: action, } + createTime := timeutil.TimeStampNow() res, err := modelarts.ManageNotebook2(task.JobID, param) if err != nil { log.Error("ManageNotebook2(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) @@ -465,6 +466,8 @@ func NotebookManage(ctx *context.Context) { Image: task.Image, ComputeResource: task.ComputeResource, Description: task.Description, + CreatedUnix: createTime, + UpdatedUnix: createTime, } err = models.RestartCloudbrain(task, newTask) diff --git a/services/repository/repository.go b/services/repository/repository.go index cea16516a..d0cd52653 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "code.gitea.io/gitea/models" @@ -172,3 +173,137 @@ func RecommendFromPromote(url string) ([]string, error) { } return result, nil } + +func CheckPushSizeLimit4Web(repo *models.Repository, fileIds []string) error { + if err := CheckRepoNumOnceLimit(len(fileIds)); err != nil { + return err + } + totalSize, err := CountUploadFileSizeByIds(fileIds) + if err != nil { + return UploadFileInvalidErr{} + } + if err := CheckRepoTotalSizeLimit(repo, totalSize); err != nil { + return err + } + return nil +} + +func CheckPushSizeLimit4Http(repo *models.Repository, uploadFileSize int64) error { + if err := CheckRepoOnceTotalSizeLimit(uploadFileSize); err != nil { + return err + } + if err := CheckRepoTotalSizeLimit(repo, uploadFileSize); err != nil { + return err + } + return nil +} + +func CheckRepoTotalSizeLimit(repo *models.Repository, uploadFileSize int64) error { + if repo.Size+uploadFileSize > setting.Repository.RepoMaxSize*1024*1024 { + return RepoTooLargeErr{} + } + return nil +} + +func CheckRepoOnceTotalSizeLimit(uploadFileSize int64) error { + if uploadFileSize > setting.Repository.Upload.TotalMaxSize*1024*1024 { + return UploadFileTooLargeErr{} + } + return nil +} + +func CheckRepoNumOnceLimit(uploadFileNum int) error { + if uploadFileNum > setting.Repository.Upload.MaxFiles { + return UploadFileTooMuchErr{} + } + return nil +} + +func CountUploadFileSizeByIds(fileIds []string) (int64, error) { + if len(fileIds) == 0 { + return 0, nil + } + uploads, err := models.GetUploadsByUUIDs(fileIds) + if err != nil { + return 0, fmt.Errorf("CountUploadFileSizeByIds error [uuids: %v]: %v", fileIds, err) + } + var totalSize int64 + for _, upload := range uploads { + size, err := GetUploadFileSize(upload) + if err != nil { + return 0, err + } + totalSize += size + } + return totalSize, nil +} + +func GetUploadFileSize(upload *models.Upload) (int64, error) { + info, err := os.Lstat(upload.LocalPath()) + + if err != nil { + return 0, err + } + return info.Size(), nil + +} + +type RepoTooLargeErr struct { +} + +func (RepoTooLargeErr) Error() string { + return fmt.Sprintf("Repository can not exceed %d MB. Please remove some unnecessary files and try again", setting.Repository.RepoMaxSize) +} + +func IsRepoTooLargeErr(err error) bool { + _, ok := err.(RepoTooLargeErr) + return ok +} + +type UploadFileTooLargeErr struct { +} + +func (UploadFileTooLargeErr) Error() string { + return fmt.Sprintf("Upload files can not exceed %d MB at a time", setting.Repository.Upload.TotalMaxSize) +} + +func IsUploadFileTooLargeErr(err error) bool { + _, ok := err.(UploadFileTooLargeErr) + return ok +} + +type RepoFileTooLargeErr struct { +} + +func (RepoFileTooLargeErr) Error() string { + return "repository file is too large" +} + +func IsRepoFileTooLargeErr(err error) bool { + _, ok := err.(RepoFileTooLargeErr) + return ok +} + +type UploadFileTooMuchErr struct { +} + +func (UploadFileTooMuchErr) Error() string { + return "upload files are too lmuch" +} + +func IsUploadFileTooMuchErr(err error) bool { + _, ok := err.(UploadFileTooMuchErr) + return ok +} + +type UploadFileInvalidErr struct { +} + +func (UploadFileInvalidErr) Error() string { + return "upload files are invalid" +} + +func IsUploadFileInvalidErr(err error) bool { + _, ok := err.(UploadFileInvalidErr) + return ok +} diff --git a/templates/base/head_notice.tmpl b/templates/base/head_notice.tmpl index 88615cc38..43c581e6e 100644 --- a/templates/base/head_notice.tmpl +++ b/templates/base/head_notice.tmpl @@ -1,7 +1,7 @@ {{if not .IsCourse}} {{ if .notices}}
-
+
{{ $firstTag := true }} {{range .notices.Notices}} @@ -25,7 +25,7 @@
- + {{end}} diff --git a/templates/custom/select_dataset.tmpl b/templates/custom/select_dataset.tmpl index 724ccb396..b5afdd8e7 100644 --- a/templates/custom/select_dataset.tmpl +++ b/templates/custom/select_dataset.tmpl @@ -135,4 +135,4 @@ - \ No newline at end of file + diff --git a/templates/custom/select_dataset_train.tmpl b/templates/custom/select_dataset_train.tmpl new file mode 100644 index 000000000..2771200b6 --- /dev/null +++ b/templates/custom/select_dataset_train.tmpl @@ -0,0 +1,134 @@ + + +
+     + + + {{.i18n.Tr "dataset.select_dataset"}} + +
+ + +
+ + + +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias} ${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ + +
+ +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ +
+ +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ +
+ +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ +
+
+
+ + +
+
+ + +
diff --git a/templates/explore/datasets.tmpl b/templates/explore/datasets.tmpl index ff18a3da3..a1692cf35 100644 --- a/templates/explore/datasets.tmpl +++ b/templates/explore/datasets.tmpl @@ -121,10 +121,10 @@ @@ -184,9 +184,15 @@
+ {{if eq .UserID 0}} + {{else}} + + + + {{end}} 创建于:{{TimeSinceUnix1 .CreatedUnix}}
@@ -215,4 +221,4 @@ -{{template "base/footer" .}} +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/explore/organizations.tmpl b/templates/explore/organizations.tmpl index 58615a8e5..6d308161d 100644 --- a/templates/explore/organizations.tmpl +++ b/templates/explore/organizations.tmpl @@ -83,35 +83,43 @@
-

{{$.i18n.Tr "org.star"}}

+

{{$.i18n.Tr "org.star"}}

  • {{ range $i,$user :=.StarOrgs}} -
      +
        {{if eq $i 0}} -
        - -
        +
      • +
        + +
        +
      • {{else if eq $i 1}} -
        - -
        +
      • +
        + +
        +
      • {{else if eq $i 2}} -
        - -
        +
      • +
        + +
        +
      • {{else }} -
        - {{Add $i 1}} -
        +
      • +
        + {{Add $i 1}} +
        +
      • {{end}}
      • -

        {{$user.Name}}

        + {{$user.Name}}
      • -
          +
          •  

            {{$user.Score}}

          • @@ -122,36 +130,44 @@
  • -
    -

    {{$.i18n.Tr "org.member"}}

    +
    +

    {{$.i18n.Tr "org.member"}}

  • - {{ range $i,$user :=.StarOrgs}} -
      + {{ range $i,$user :=.MemberOrgs}} +
        {{if eq $i 0}} -
        - -
        +
      • +
        + +
        +
      • {{else if eq $i 1}} -
        - -
        +
      • +
        + +
        +
      • {{else if eq $i 2}} -
        - -
        +
      • +
        + +
        +
      • {{else }} -
        - {{Add $i 1}} -
        +
      • +
        + {{Add $i 1}} +
        +
      • {{end}}
      • -

        {{$user.Name}}

        + {{$user.Name}}
      • -
          +
          •  

            {{$user.Score}}

          • @@ -162,34 +178,43 @@
  • -
    -

    {{$.i18n.Tr "org.active"}}

    +
    +

    {{$.i18n.Tr "org.active"}}

  • - {{ range $i,$user :=.StarOrgs}} -
      + {{ range $i,$user :=.ActiveOrgs }} +
        {{if eq $i 0}} -
        - -
        +
      • +
        + +
        +
      • {{else if eq $i 1}} -
        - -
        +
      • +
        + +
        +
      • {{else if eq $i 2}} -
        - -
        +
      • +
        + +
        +
      • {{else }} -
        - {{Add $i 1}} -
        +
      • +
        + {{Add $i 1}} +
        + +
      • {{end}}
      • -

        {{$user.Name}}

        + {{$user.Name}}
      @@ -249,3 +274,6 @@ window.onload = function() { } + \ No newline at end of file diff --git a/templates/org/member/members.tmpl b/templates/org/member/members.tmpl index 87189c015..9c45007e5 100644 --- a/templates/org/member/members.tmpl +++ b/templates/org/member/members.tmpl @@ -4,18 +4,19 @@
      {{template "base/alert" .}} {{template "org/navber" .}} -
      - +
      {{ range .Members}}
      -
      +
      +
      {{.FullName}}
      -
      +
      +
      {{$.i18n.Tr "org.members.membership_visibility"}}
      @@ -29,14 +30,16 @@ {{if or (eq $.SignedUser.ID .ID) $.IsOrganizationOwner}}({{$.i18n.Tr "org.members.private_helper"}}){{end}} {{end}}
      -
      +
      +
      {{$.i18n.Tr "org.members.member_role"}}
      {{if index $.MembersIsUserOrgOwner .ID}}{{svg "octicon-shield-lock" 16}} {{$.i18n.Tr "org.members.owner"}}{{else}}{{$.i18n.Tr "org.members.member"}}{{end}}
      -
      +
      +
      2FA
      @@ -49,7 +52,8 @@ {{end}}
      -
      +
      +
      {{if eq $.SignedUser.ID .ID}}
      diff --git a/templates/org/navber.tmpl b/templates/org/navber.tmpl index f7e88c1e9..772d9f67e 100755 --- a/templates/org/navber.tmpl +++ b/templates/org/navber.tmpl @@ -42,18 +42,15 @@ .active{ color:#0366D6 !important; } - .mleft{ - margin-left: 30% !important; - } .mbom{ margin-bottom: 10px !important; }
      -