Merge branch 'V20221228' into zpz-issue-1228

models/cloudbrain.go

@@ -180,14 +180,14 @@ type Cloudbrain struct {
 	AiCenter        string //grampus ai center: center_id+center_name
 
 	TrainUrl              string      //输出模型的obs路径
-	BranchName            string      //分支名称
+	BranchName            string      `xorm:"varchar(2550)"` //分支名称
 	Parameters            string      //传给modelarts的param参数
-	BootFile              string      //启动文件
+	BootFile              string      `xorm:"varchar(2550)"` //启动文件
 	DataUrl               string      `xorm:"varchar(3500)"` //数据集的obs路径
 	LogUrl                string      //日志输出的obs路径
 	PreVersionId          int64       //父版本的版本id
 	FlavorCode            string      //modelarts上的规格id
-	Description           string      `xorm:"varchar(256)"` //描述
+	Description           string      `xorm:"varchar(2550)"` //描述
 	WorkServerNumber      int         //节点数
 	FlavorName            string      //规格名称
 	EngineName            string      //引擎名称

models/user_analysis_for_activity.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -450,15 +451,30 @@ func QueryUserLoginInfo(userIds []int64) []*UserLoginLog {
 	return loginList
 }
 
+var WeekBonusData = make(map[int64][]int)
+
 func QueryUserAnnualReport(userId int64) *UserSummaryCurrentYear {
 	statictisSess := xStatistic.NewSession()
 	defer statictisSess.Close()
 	log.Info("userId=" + fmt.Sprint(userId))
+	if len(WeekBonusData) == 0 {
+		WeekBonusData = getBonusWeekDataMap()
+	}
 
 	reList := make([]*UserSummaryCurrentYear, 0)
 	err := statictisSess.Select("*").Table(new(UserSummaryCurrentYear)).Where("id=" + fmt.Sprint(userId)).Find(&reList)
 	if err == nil {
 		if len(reList) > 0 {
+			record, ok := WeekBonusData[userId]
+			if ok {
+				bonusInfo := make(map[string]int)
+				bonusInfo["order"] = record[0]
+				bonusInfo["money"] = record[1]
+				bonusInfo["week"] = record[2]
+				bonusInfo["num"] = record[3]
+				bonusInfoJson, _ := json.Marshal(bonusInfo)
+				reList[0].WeekBonusData = string(bonusInfoJson)
+			}
 			return reList[0]
 		}
 	} else {

models/user_business_analysis.go

@@ -880,6 +880,68 @@ func isUserYearData(tableName string) bool {
 	return false
 }
 
+func getBonusWeekDataMap() map[int64][]int {
+	bonusMap := make(map[int64][]int)
+	url := setting.RecommentRepoAddr + "bonus/weekdata/record.txt"
+	content, err := GetContentFromPromote(url)
+	if err == nil {
+		filenames := strings.Split(content, "\n")
+		for i := 0; i < len(filenames); i++ {
+			url = setting.RecommentRepoAddr + "bonus/weekdata/" + filenames[i]
+			csvContent, err1 := GetContentFromPromote(url)
+			if err1 == nil {
+				//read csv
+				lines := strings.Split(csvContent, "\n")
+				for j := 1; j < len(lines); j++ {
+					aLine := strings.Split(lines[j], ",")
+					if len(aLine) < 4 {
+						continue
+					}
+					userId := getInt64Value(aLine[0])
+					order := getIntValue(aLine[2])
+					money := getIntValue(aLine[3])
+					week, num := getWeekAndNum(filenames[i])
+					//email := lines[2]
+					record, ok := bonusMap[userId]
+					if !ok {
+						record = make([]int, 4)
+						record[0] = order
+						record[1] = money
+						record[2] = week
+						record[3] = num
+						bonusMap[userId] = record
+					} else {
+						if record[0] > order {
+							record[0] = order
+							record[1] = money
+							record[2] = week
+							record[3] = num
+						} else {
+							if record[0] == order && record[1] < money {
+								record[1] = money
+								record[2] = week
+								record[3] = num
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return bonusMap
+}
+
+func getWeekAndNum(name string) (int, int) {
+	name = name[0 : len(name)-4]
+	tmp := strings.Split(name, "_")
+	if len(tmp) == 2 {
+		week := getIntValue(tmp[0])
+		num := getIntValue(tmp[1])
+		return week, num
+	}
+	return 0, 0
+}
+
 func getBonusMap() map[string]map[string]int {
 	bonusMap := make(map[string]map[string]int)
 	url := setting.RecommentRepoAddr + "bonus/record.txt"
@@ -923,6 +985,14 @@ func getIntValue(val string) int {
 	return 0
 }
 
+func getInt64Value(val string) int64 {
+	i, err := strconv.ParseInt(val, 10, 64)
+	if err == nil {
+		return i
+	}
+	return 0
+}
+
 func getPlayARoll(bonusMap map[string]map[string]int, userName string, scoreMap map[string]float64) string {
 	bonusInfo := make(map[string]string)
 	record, ok := bonusMap[userName]

models/user_business_struct.go

@@ -18,9 +18,9 @@ type UserSummaryCurrentYear struct {
 	CodeInfo       string `xorm:"varchar(500)"`  //代码提交次数，提交总代码行数，最晚的提交时间
 	CloudBrainInfo string `xorm:"varchar(1000)"` //，创建了XX 个云脑任务，调试任务XX 个，训练任务XX 个，推理任务XX 个，累计运行了XXXX 卡时，累计节省xxxxx 元
 	//这些免费的算力资源分别有，XX% 来自鹏城云脑1，XX% 来自鹏城云脑2，XX% 来自智算网络
-	PlayARoll string `xorm:"varchar(500)"` //你参加了XX 次“我为开源打榜狂”活动，累计上榜XX 次，总共获得了社区XXX 元的激励
-
-	Label string `xorm:"varchar(500)"`
+	PlayARoll     string `xorm:"varchar(500)"` //你参加了XX 次“我为开源打榜狂”活动，累计上榜XX 次，总共获得了社区XXX 元的激励
+	WeekBonusData string `xorm:"-"`
+	Label         string `xorm:"varchar(500)"`
 }
 
 type UserBusinessAnalysisCurrentYear struct {

modules/storage/obs.go

@@ -614,7 +614,7 @@ func ObsCreateObject(path string) error {
 	return nil
 }
 
-func GetObsLogFileName(prefix string) (string, error) {
+func GetObsLogFileName(prefix string) ([]FileInfo, error) {
 	input := &obs.ListObjectsInput{}
 	input.Bucket = setting.Bucket
 	input.Prefix = prefix
@@ -622,10 +622,26 @@ func GetObsLogFileName(prefix string) (string, error) {
 	output, err := ObsCli.ListObjects(input)
 	if err != nil {
 		log.Error("PutObject failed:", err.Error())
-		return "", err
+		return nil, err
 	}
 	if output == nil || len(output.Contents) == 0 {
-		return "", errors.New("obs log files not exist")
+		return nil, errors.New("obs log files not exist")
+	}
+	fileInfos := make([]FileInfo, 0)
+	for _, val := range output.Contents {
+		//result[num] = c.Key
+		if strings.HasSuffix(val.Key, ".log") {
+			log.Info("log fileName=" + val.Key)
+			fileInfo := FileInfo{
+				ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
+				FileName: val.Key[len(prefix)-3:], //加上  job
+				Size:     val.Size,
+				IsDir:    false,
+				ParenDir: prefix[0 : len(prefix)-3],
+			}
+			fileInfos = append(fileInfos, fileInfo)
+		}
+
 	}
-	return output.Contents[0].Key, nil
+	return fileInfos, nil
 }

options/locale/locale_en-US.ini

@@ -1063,6 +1063,8 @@ model_rename=Duplicate model name, please modify model name.
 
 notebook_file_not_exist=Notebook file does not exist.
 notebook_select_wrong=Please select a Notebook(.ipynb) file first.
+notebook_path_too_long=The total length of selected file or files path exceed 255 characters, please select a shorter path file or change the file path.
+notebook_branch_name_too_long=The total  length of branch or branches name exceed 255 characters, please select a file in other branch.
 notebook_file_no_right=You have no right to access the Notebook(.ipynb) file.
 notebook_repo_conflict=The files in different branches of the same repository can not run together.
 debug_again_fail=Fail to restart debug task, please try again later.

options/locale/locale_zh-CN.ini

@@ -1062,6 +1062,8 @@ model_rename=模型名称重复，请修改模型名称
 
 notebook_file_not_exist=Notebook文件不存在。
 notebook_select_wrong=请先选择Notebook（.ipynb）文件。
+notebook_path_too_long=选择的一个或多个Notebook文件路径总长度超过255个字符，请选择路径较短的文件或调整文件路径。
+notebook_branch_name_too_long=选择的一个或多个Notebook文件分支名总长度超过255个字符，请选择其他分支的文件。
 notebook_file_no_right=您没有这个Notebook文件的读权限。
 notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。
 debug_again_fail=再次调试失败，请稍后再试。

routers/api/v1/repo/attachments.go

@@ -1,22 +1,67 @@
 package repo
 
 import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
 	routeRepo "code.gitea.io/gitea/routers/repo"
 )
 
 func GetSuccessChunks(ctx *context.APIContext) {
+	if errStr := checkDatasetPermission(ctx); errStr != "" {
+		ctx.JSON(http.StatusForbidden, ctx.Tr(errStr))
+	}
+
 	routeRepo.GetSuccessChunks(ctx.Context)
 }
 
+func checkDatasetPermission(ctx *context.APIContext) string {
+	datasetId := ctx.QueryInt64("dataset_id")
+
+	dataset, err := models.GetDatasetByID(datasetId)
+	if err != nil {
+		log.Warn("can not find dataset", err)
+
+		return "dataset.query_dataset_fail"
+	}
+	repo, err := models.GetRepositoryByID(dataset.RepoID)
+	if err != nil {
+		log.Warn("can not find repo", err)
+		return "dataset.query_dataset_fail"
+	}
+
+	permission, err := models.GetUserRepoPermission(repo, ctx.User)
+	if err != nil {
+		log.Warn("can not find repo permission for user", err)
+		return "dataset.query_dataset_fail"
+	}
+	if !permission.CanWrite(models.UnitTypeDatasets) {
+
+		return "error.no_right"
+	}
+	return ""
+}
+
 func NewMultipart(ctx *context.APIContext) {
+	if errStr := checkDatasetPermission(ctx); errStr != "" {
+		ctx.JSON(http.StatusForbidden, ctx.Tr(errStr))
+	}
 	routeRepo.NewMultipart(ctx.Context)
 }
 func GetMultipartUploadUrl(ctx *context.APIContext) {
+	if errStr := checkDatasetPermission(ctx); errStr != "" {
+		ctx.JSON(http.StatusForbidden, ctx.Tr(errStr))
+	}
 	routeRepo.GetMultipartUploadUrl(ctx.Context)
 }
 
 func CompleteMultipart(ctx *context.APIContext) {
+	if errStr := checkDatasetPermission(ctx); errStr != "" {
+		ctx.JSON(http.StatusForbidden, ctx.Tr(errStr))
+	}
 	routeRepo.CompleteMultipart(ctx.Context)
 
 }

routers/repo/modelarts.go

@@ -483,7 +483,7 @@ func getFileUrl(url string, filename string) string {
 		}
 	}
 
-	return url + middle + filename
+	return url + middle + filename + "?reset"
 }
 
 func NotebookRestart(ctx *context.Context) {
@@ -2893,15 +2893,19 @@ func TrainJobDownloadLogFile(ctx *context.Context) {
 		ctx.ServerError("GetObsLogFileName", err)
 		return
 	}
-
-	url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key)
-	if err != nil {
-		log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"])
-		ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err)
-		return
+	if len(key) > 1 {
+		ObsDownloadManyFile(prefix[0:len(prefix)-3], ctx, task.DisplayJobName+".zip", key)
+	} else {
+		url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key[0].ParenDir+key[0].FileName)
+		if err != nil {
+			log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"])
+			ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err)
+			return
+		}
+		ctx.Resp.Header().Set("Cache-Control", "max-age=0")
+		http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
 	}
-	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
-	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
+
 }
 func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) {
 	var isMultiDataset bool

services/cloudbrain/cloudbrainTask/notebook.go

@@ -35,6 +35,7 @@ const NoteBookExtension = ".ipynb"
 const CPUType = 0
 const GPUType = 1
 const NPUType = 2
+const CharacterLength = 2550
 
 func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) {
 
@@ -46,6 +47,14 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
 		ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong")))
 		return
 	}
+	if len(getBootFile(option.File, option.OwnerName, option.ProjectName)) > CharacterLength {
+		ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long")))
+		return
+	}
+	if len(option.BranchName) > CharacterLength {
+		ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_branch_name_too_long")))
+		return
+	}
 
 	isNotebookFileExist, _ := isNoteBookFileExist(ctx, option)
 	if !isNotebookFileExist {
@@ -105,14 +114,29 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
 					err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName)
 					if err != nil {
 						log.Error("download code failed", err)
-						ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
-						return
+						if !strings.Contains(err.Error(), "already exists and is not an empty directory") {
+							ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
+							return
+						}
 					}
 				}
 				if !isRepoFileMatch(option, noteBook) {
-					noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName)
-					noteBook.BranchName += ";" + option.BranchName
-					noteBook.Description += ";" + getDescription(option)
+					if len(noteBook.BootFile)+len(getBootFile(option.File, option.OwnerName, option.ProjectName))+1 <= CharacterLength {
+						noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName)
+					} else {
+						ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long")))
+						return
+					}
+					if len(noteBook.BranchName)+len(option.BranchName)+1 <= CharacterLength {
+						noteBook.BranchName += ";" + option.BranchName
+					} else {
+						ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_branch_name_too_long")))
+						return
+					}
+
+					if len(noteBook.Description)+len(getDescription(option))+1 <= CharacterLength {
+						noteBook.Description += ";" + getDescription(option)
+					}
 
 					err := models.UpdateJob(noteBook)
 					if err != nil {
@@ -431,7 +455,11 @@ func getCodePath(jobName string, repo *models.Repository) string {
 }
 
 func getDescription(option api.CreateFileNotebookJobOption) string {
-	return option.OwnerName + "/" + option.ProjectName + "/" + option.File
+	des := option.OwnerName + "/" + option.ProjectName + "/" + option.File
+	if len(des) <= CharacterLength {
+		return des
+	}
+	return ""
 }
 
 func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) {