Merge branch 'liuzx_trainjob' of https://git.openi.org.cn/OpenI/aiforge into liuzx_trainjob

models/cloudbrain.go

@@ -87,7 +87,7 @@ type Cloudbrain struct {
 	LogUrl            string //日志输出的obs路径
 	PreVersionId      int64  //父版本的版本id
 	FlavorCode        string //modelarts上的规格id
-	Description       string //描述
+	Description       string `xorm:"varchar(256)"` //描述
 	WorkServerNumber  int    //节点数
 	FlavorName        string //规格名称
 	EngineName        string //引擎名称

models/models.go

@@ -139,6 +139,7 @@ func init() {
 		new(RepoStatistic),
 		new(SummaryStatistic),
 		new(UserBusinessAnalysis),
+		new(UserBusinessAnalysisAll),
 		new(UserLoginLog),
 	)
 

models/user_business_analysis.go

@@ -4,13 +4,80 @@ import (
 	"encoding/json"
 	"fmt"
 	"sort"
+	"strconv"
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/timeutil"
 	"xorm.io/builder"
+	"xorm.io/xorm"
 )
 
+type UserBusinessAnalysisAll struct {
+	ID int64 `xorm:"pk"`
+
+	CountDate int64 `xorm:"pk"`
+
+	//action :ActionMergePullRequest                         // 11
+	CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//action :ActionCommitRepo                               // 5
+	CommitCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//action :ActionCreateIssue                             // 10
+	IssueCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//comment table current date
+	CommentCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//watch table current date
+	FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//star table current date
+	StarRepoCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//follow table
+	WatchedCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	// user table
+	GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"`
+
+	//
+	CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"`
+
+	//attachement table
+	CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"`
+
+	//0
+	CommitModelCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//issue, issueassignees
+	SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//baike
+	EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//user
+	RegistDate timeutil.TimeStamp `xorm:"NOT NULL"`
+
+	//repo
+	CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//login count, from elk
+	LoginCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//openi index
+	OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"`
+
+	//user
+	Email string `xorm:"NOT NULL"`
+
+	//user
+	Name string `xorm:"NOT NULL"`
+
+	DataDate string `xorm:"NULL"`
+}
+
 type UserBusinessAnalysis struct {
 	ID int64 `xorm:"pk"`
 
@@ -93,46 +160,12 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool {
 	return ulist[i].ID > ulist[j].ID
 }
 
-func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis {
-	log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime))
-	statictisSess := xStatistic.NewSession()
-	defer statictisSess.Close()
-
-	statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc")
-
-	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
-	statictisSess.Find(&userBusinessAnalysisList)
-
-	resultMap := make(map[int64]*UserBusinessAnalysis)
-	log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
-	for _, userRecord := range userBusinessAnalysisList {
-		if _, ok := resultMap[userRecord.ID]; !ok {
-			resultMap[userRecord.ID] = userRecord
-		} else {
-			resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
-			resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
-			resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
-			resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
-			resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
-			resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
-			resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
-			resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
-			resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
-			resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
-			resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
-			resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
-			resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
-			resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
-		}
-	}
+type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll
 
-	userBusinessAnalysisReturnList := UserBusinessAnalysisList{}
-	for _, v := range resultMap {
-		userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v)
-	}
-	sort.Sort(userBusinessAnalysisReturnList)
-	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
-	return userBusinessAnalysisReturnList
+func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] }
+func (ulist UserBusinessAnalysisAllList) Len() int      { return len(ulist) }
+func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool {
+	return ulist[i].ID > ulist[j].ID
 }
 
 func getLastCountDate() int64 {
@@ -153,71 +186,36 @@ func getLastCountDate() int64 {
 	return pageStartTime.Unix()
 }
 
-func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) {
+func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) {
 	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll))
 
 	statictisSess := xStatistic.NewSession()
 	defer statictisSess.Close()
 
-	resultMap := make(map[int64]*UserBusinessAnalysis)
-
-	var newAndCond = builder.NewCond()
-	if len(opts.UserName) > 0 {
-		newAndCond = newAndCond.And(
-			builder.Like{"name", opts.UserName},
-		)
-	}
-	if !opts.IsAll {
-		newAndCond = newAndCond.And(
-			builder.Gte{"count_date": opts.StartTime},
-		)
-		newAndCond = newAndCond.And(
-			builder.Lte{"count_date": opts.EndTime},
-		)
-	}
-
-	allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis))
+	allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll))
 	if err != nil {
 		log.Info("query error." + err.Error())
 		return nil, 0
 	}
 	log.Info("query return total:" + fmt.Sprint(allCount))
-	pageSize := 200
+	if allCount == 0 {
+		RefreshUserStaticAllTabel()
+	}
+	pageSize := 1000
 	totalPage := int(allCount) / pageSize
-
+	userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{}
 	for i := 0; i <= int(totalPage); i++ {
-		userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
-		if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize).
-			Find(&userBusinessAnalysisList); err != nil {
+		userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0)
+		if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize).
+			Find(&userBusinessAnalysisAllList); err != nil {
 			return nil, 0
 		}
-		log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
-		for _, userRecord := range userBusinessAnalysisList {
-			if _, ok := resultMap[userRecord.ID]; !ok {
-				resultMap[userRecord.ID] = userRecord
-			} else {
-				resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
-				resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
-				resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
-				resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
-				resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
-				resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
-				resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
-				resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
-				resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
-				resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
-				resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
-				resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
-				resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
-				resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
-			}
+		log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList)))
+		for _, userRecord := range userBusinessAnalysisAllList {
+			userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord)
 		}
 	}
 
-	userBusinessAnalysisReturnList := UserBusinessAnalysisList{}
-	for _, v := range resultMap {
-		userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v)
-	}
 	sort.Sort(userBusinessAnalysisReturnList)
 	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
 	return userBusinessAnalysisReturnList, allCount
@@ -338,6 +336,50 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
 	return userBusinessAnalysisReturnList, count
 }
 
+func RefreshUserStaticAllTabel() {
+
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+	log.Info("delete all data from table: user_business_analysis_all")
+	statictisSess.Exec("delete from user_business_analysis_all")
+
+	currentTimeNow := time.Now()
+	pageStartTime := getLastCountDate()
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix()
+
+	var cond = builder.NewCond()
+	cond = cond.And(
+		builder.Gte{"count_date": pageStartTime},
+	)
+	cond = cond.And(
+		builder.Lte{"count_date": pageEndTime},
+	)
+	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
+	if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc").
+		Find(&userBusinessAnalysisList); err != nil {
+		return
+	}
+	log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList)))
+	for _, userRecord := range userBusinessAnalysisList {
+		log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID))
+		allData := getAllData(userRecord.ID, statictisSess)
+		allData.ID = userRecord.ID
+		allData.CountDate = 0
+		allData.DataDate = userRecord.DataDate
+		allData.Email = userRecord.Email
+		allData.OpenIIndex = userRecord.OpenIIndex
+		allData.GiteaAgeMonth = userRecord.GiteaAgeMonth
+		allData.Name = userRecord.Name
+		allData.RegistDate = userRecord.RegistDate
+
+		_, err := statictisSess.Insert(&allData)
+		if err != nil {
+			log.Info("insert all data failed." + err.Error())
+		}
+	}
+	log.Info("refresh all data finished.")
+}
+
 func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error {
 
 	log.Info("start to count other user info data")
@@ -491,10 +533,99 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 			log.Info("insert daterecord failed." + err.Error())
 			return err
 		}
+
+		if isExistUserInAllTable(dateRecord.ID, statictisSess) {
+			updateCurrentData(dateRecord.ID, statictisSess, dateRecord)
+		} else {
+			log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID))
+			allData := getAllData(dateRecord.ID, statictisSess)
+			allData.ID = dateRecord.ID
+			allData.CountDate = 0
+			allData.DataDate = dateRecord.DataDate
+			allData.Email = dateRecord.Email
+			allData.OpenIIndex = dateRecord.OpenIIndex
+			allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth
+			allData.Name = dateRecord.Name
+			allData.RegistDate = dateRecord.RegistDate
+
+			_, err = statictisSess.Insert(&allData)
+			if err != nil {
+				log.Info("insert all data failed." + err.Error())
+				return err
+			}
+		}
+
 	}
 	return nil
 }
 
+func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) {
+
+	_, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) +
+		",commit_count+=" + fmt.Sprint(currentData.CommitCount) +
+		",issue_count+=" + fmt.Sprint(currentData.IssueCount) +
+		",comment_count+=" + fmt.Sprint(currentData.CommentCount) +
+		",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) +
+		",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) +
+		",watched_count+=" + fmt.Sprint(currentData.WatchedCount) +
+		",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) +
+		",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) +
+		",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) +
+		",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) +
+		",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) +
+		",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) +
+		",login_count+=" + fmt.Sprint(currentData.LoginCount) +
+		" where id=" + fmt.Sprint(userId))
+
+	if err != nil {
+		log.Info("update table failed." + err.Error())
+	}
+
+}
+
+func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool {
+
+	allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll))
+	if err != nil {
+		return false
+	}
+	return allCount > 0
+}
+
+func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll {
+	var dateRecord UserBusinessAnalysisAll
+
+	rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id")
+	if err == nil {
+		for i, row := range rows {
+			log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"]))
+			dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"]))
+			dateRecord.CommitCount = getInt(string(row["commit_count"]))
+			dateRecord.IssueCount = getInt(string(row["issue_count"]))
+			dateRecord.CommentCount = getInt(string(row["comment_count"]))
+			dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"]))
+			dateRecord.StarRepoCount = getInt(string(row["star_repo_count"]))
+			dateRecord.WatchedCount = getInt(string(row["watched_count"]))
+			dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"]))
+			dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"]))
+			dateRecord.CommitModelCount = getInt(string(row["commit_model_count"]))
+			dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"]))
+			dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"]))
+			dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"]))
+			dateRecord.LoginCount = getInt(string(row["login_count"]))
+		}
+	}
+	return dateRecord
+}
+
+func getInt(str string) int {
+	re, err := strconv.ParseInt(str, 10, 32)
+	if err != nil {
+		return 0
+	}
+	return int(re)
+}
+
 func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
 	CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false)
 }

modules/storage/obs.go

@@ -176,10 +176,10 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
 	}
 }
 
-func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) {
+func GetObsListObject(jobName, parentDir string, versionOutputPath string) ([]FileInfo, error) {
 	input := &obs.ListObjectsInput{}
 	input.Bucket = setting.Bucket
-	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/")
+	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, versionOutputPath, parentDir), "/")
 	strPrefix := strings.Split(input.Prefix, "/")
 	output, err := ObsCli.ListObjects(input)
 	fileInfos := make([]FileInfo, 0)

routers/api/v1/repo/modelarts.go

@@ -303,8 +303,7 @@ func ModelList(ctx *context.APIContext) {
 		return
 	}
 	VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount)
-	parentDir = VersionOutputPath + "/" + parentDir
-	models, err := storage.GetObsListObject(task.JobName, parentDir)
+	models, err := storage.GetObsListObject(task.JobName, parentDir, VersionOutputPath)
 	if err != nil {
 		log.Info("get TrainJobListModel failed:", err)
 		ctx.ServerError("GetObsListObject:", err)

routers/repo/modelarts.go

@@ -380,6 +380,85 @@ func trainJobNewDataPrepare(ctx *context.Context) error {
 	return nil
 }
 
+func ErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error {
+	ctx.Data["PageIsCloudBrain"] = true
+
+	//can, err := canUserCreateTrainJob(ctx.User.ID)
+	//if err != nil {
+	//	ctx.ServerError("canUserCreateTrainJob", err)
+	//	return
+	//}
+	//
+	//if !can {
+	//	log.Error("the user can not create train-job")
+	//	ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job"))
+	//	return
+	//}
+
+	t := time.Now()
+	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
+	ctx.Data["job_name"] = jobName
+
+	attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
+	if err != nil {
+		ctx.ServerError("GetAllUserAttachments failed:", err)
+		return err
+	}
+	ctx.Data["attachments"] = attachs
+
+	var resourcePools modelarts.ResourcePool
+	if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["resource_pools"] = resourcePools.Info
+
+	var engines modelarts.Engine
+	if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["engines"] = engines.Info
+
+	var versionInfos modelarts.VersionInfo
+	if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["engine_versions"] = versionInfos.Version
+
+	var flavorInfos modelarts.Flavor
+	if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["flavor_infos"] = flavorInfos.Info
+
+	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
+	ctx.Data["train_url"] = outputObsPath
+
+	Branches, err := ctx.Repo.GitRepo.GetBranches()
+	if err != nil {
+		ctx.ServerError("GetBranches error:", err)
+		return err
+	}
+	ctx.Data["Branches"] = Branches
+	ctx.Data["BranchesCount"] = len(Branches)
+
+	configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
+	if err != nil {
+		ctx.ServerError("getConfigList failed:", err)
+		return err
+	}
+	ctx.Data["config_list"] = configList.ParaConfigs
+	ctx.Data["bootFile"] = form.BootFile
+	ctx.Data["uuid"] = form.Attachment
+	ctx.Data["params"] = form.Params
+	ctx.Data["branch_name"] = form.BranchName
+
+	return nil
+}
+
 func TrainJobNewVersion(ctx *context.Context) {
 	err := trainJobNewVersionDataPrepare(ctx)
 	if err != nil {
@@ -477,6 +556,94 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
 	return nil
 }
 
+func VersionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error {
+	ctx.Data["PageIsCloudBrain"] = true
+	var jobID = ctx.Params(":jobid")
+	// var versionName = ctx.Params(":version-name")
+	var versionName = ctx.Query("version_name")
+
+	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
+	if err != nil {
+		log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error())
+		return err
+	}
+
+	t := time.Now()
+	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
+	ctx.Data["job_name"] = task.JobName
+
+	attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
+	if err != nil {
+		ctx.ServerError("GetAllUserAttachments failed:", err)
+		return err
+	}
+	ctx.Data["attachments"] = attachs
+
+	var resourcePools modelarts.ResourcePool
+	if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["resource_pools"] = resourcePools.Info
+
+	var engines modelarts.Engine
+	if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["engines"] = engines.Info
+
+	var versionInfos modelarts.VersionInfo
+	if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["engine_versions"] = versionInfos.Version
+
+	var flavorInfos modelarts.Flavor
+	if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["flavor_infos"] = flavorInfos.Info
+
+	var Parameters modelarts.Parameters
+	if err = json.Unmarshal([]byte(form.Params), &Parameters); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return err
+	}
+	ctx.Data["params"] = Parameters.Parameter
+
+	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
+	ctx.Data["train_url"] = outputObsPath
+
+	Branches, err := ctx.Repo.GitRepo.GetBranches()
+	if err != nil {
+		ctx.ServerError("GetBranches error:", err)
+		return err
+	}
+	ctx.Data["branches"] = Branches
+	ctx.Data["description"] = form.Description
+	ctx.Data["dataset_name"] = task.DatasetName
+	ctx.Data["work_server_number"] = form.WorkServerNumber
+	ctx.Data["flavor_name"] = form.FlavorName
+	ctx.Data["engine_name"] = form.EngineName
+	ctx.Data["flavor_code"] = task.FlavorCode
+	ctx.Data["engine_id"] = task.EngineID
+
+	ctx.Data["bootFile"] = form.BootFile
+	ctx.Data["uuid"] = form.Attachment
+	ctx.Data["branch_name"] = form.BranchName
+	configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
+	if err != nil {
+		ctx.ServerError("getConfigList failed:", err)
+		return err
+	}
+	ctx.Data["config_list"] = configList.ParaConfigs
+
+	return nil
+}
+
 func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
 	ctx.Data["PageIsTrainJob"] = true
 	VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(modelarts.TotalVersionCount)
@@ -504,7 +671,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 
 	if err := paramCheckCreateTrainJob(form); err != nil {
 		log.Error("paramCheckCreateTrainJob failed:(%v)", err)
-		trainJobNewDataPrepare(ctx)
+		ErrorNewDataPrepare(ctx, form)
 		ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
 		return
 	}
@@ -529,13 +696,6 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 	}); err != nil {
 		log.Error("创建任务失败，服务器超时！: %s (%v)", repo.FullName(), err)
 		trainJobNewDataPrepare(ctx)
-
-		ctx.Data["bootFile"] = form.BootFile
-		ctx.Data["uuid"] = form.Attachment
-		ctx.Data["datasetName"] = attach.Name
-		ctx.Data["params"] = form.Params
-		ctx.Data["branch_name"] = branch_name
-		trainJobNewDataPrepare(ctx)
 		ctx.RenderWithErr("创建任务失败，服务器超时！", tplModelArtsTrainJobNew, &form)
 		return
 	}
@@ -555,7 +715,9 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		return
 	}
 
+	// parentDir := VersionOutputPath + "/"
 	if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
+		// if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil {
 		log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
 		trainJobNewDataPrepare(ctx)
 		ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobNew, &form)
@@ -651,6 +813,13 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		TotalVersionCount: modelarts.TotalVersionCount,
 	}
 
+	//将params转换Parameters.Parameter，出错时返回给前端
+	var Parameters modelarts.Parameters
+	if err := json.Unmarshal([]byte(params), &Parameters); err != nil {
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return
+	}
+
 	err = modelarts.GenerateTrainJob(ctx, req)
 	if err != nil {
 		log.Error("GenerateTrainJob failed:%v", err.Error())
@@ -658,7 +827,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		ctx.Data["bootFile"] = form.BootFile
 		ctx.Data["uuid"] = form.Attachment
 		ctx.Data["datasetName"] = attach.Name
-		ctx.Data["params"] = form.Params
+		ctx.Data["params"] = Parameters.Parameter
 		ctx.Data["branch_name"] = branch_name
 		ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
 		return
@@ -689,7 +858,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	isSaveParam := form.IsSaveParam
 	repo := ctx.Repo.Repository
 	codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
-	codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
+	codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/"
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/"
 	logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
 	dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
@@ -701,16 +870,16 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 
 	if err := paramCheckCreateTrainJob(form); err != nil {
 		log.Error("paramCheckCreateTrainJob failed:(%v)", err)
-		trainJobNewVersionDataPrepare(ctx)
+		VersionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
 
-	attach, err := models.GetAttachmentByUUID(uuid)
-	if err != nil {
-		log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error())
-		return
-	}
+	// attach, err := models.GetAttachmentByUUID(uuid)
+	// if err != nil {
+	// 	log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error())
+	// 	return
+	// }
 
 	//todo: del the codeLocalPath
 	_, err = ioutil.ReadDir(codeLocalPath)
@@ -724,13 +893,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		Branch: branch_name,
 	}); err != nil {
 		log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err)
-		trainJobNewVersionDataPrepare(ctx)
-
-		ctx.Data["bootFile"] = form.BootFile
-		ctx.Data["uuid"] = form.Attachment
-		ctx.Data["datasetName"] = attach.Name
-		ctx.Data["params"] = form.Params
-		ctx.Data["branch_name"] = branch_name
+		VersionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
@@ -738,21 +901,23 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	//todo: upload code (send to file_server todo this work?)
 	if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil {
 		log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err)
-		trainJobNewVersionDataPrepare(ctx)
+		VersionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
 
 	if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil {
 		log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err)
-		trainJobNewVersionDataPrepare(ctx)
+		VersionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
 
-	if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
+	parentDir := VersionOutputPath + "/"
+	// if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
+	if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil {
 		log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
-		trainJobNewVersionDataPrepare(ctx)
+		VersionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
@@ -772,7 +937,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		err := json.Unmarshal([]byte(params), &parameters)
 		if err != nil {
 			log.Error("Failed to Unmarshal params: %s (%v)", params, err)
-			trainJobNewVersionDataPrepare(ctx)
+			VersionErrorDataPrepare(ctx, form)
 			ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobVersionNew, &form)
 			return
 		}
@@ -791,7 +956,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	if isSaveParam == "on" {
 		if form.ParameterTemplateName == "" {
 			log.Error("ParameterTemplateName is empty")
-			trainJobNewVersionDataPrepare(ctx)
+			VersionErrorDataPrepare(ctx, form)
 			ctx.RenderWithErr("保存作业参数时，作业参数名称不能为空", tplModelArtsTrainJobVersionNew, &form)
 			return
 		}
@@ -815,7 +980,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 
 		if err != nil {
 			log.Error("Failed to CreateTrainJobConfig: %v", err)
-			trainJobNewVersionDataPrepare(ctx)
+			VersionErrorDataPrepare(ctx, form)
 			ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
 			return
 		}
@@ -862,15 +1027,11 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	err = modelarts.GenerateTrainJobVersion(ctx, req, jobID)
 	if err != nil {
 		log.Error("GenerateTrainJob failed:%v", err.Error())
-		trainJobNewVersionDataPrepare(ctx)
-		ctx.Data["bootFile"] = form.BootFile
-		ctx.Data["uuid"] = form.Attachment
-		ctx.Data["datasetName"] = attach.Name
-		ctx.Data["params"] = form.Params
+		VersionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
-	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
+	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job/" + jobID)
 	// ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
 }
 
@@ -963,7 +1124,6 @@ func paramCheckCreateTrainJob(form auth.CreateModelArtsTrainJobForm) error {
 
 func TrainJobShow(ctx *context.Context) {
 	ctx.Data["PageIsCloudBrain"] = true
-
 	var jobID = ctx.Params(":jobid")
 
 	repo := ctx.Repo.Repository

routers/repo/user_data_analysis.go

@@ -16,17 +16,6 @@ import (
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
 )
 
-func QueryUserStaticData(ctx *context.Context) {
-	startDate := ctx.Query("startDate")
-	endDate := ctx.Query("endDate")
-	log.Info("startDate=" + startDate + "  endDate=" + endDate)
-	startTime, _ := time.Parse("2006-01-02", startDate)
-	endTime, _ := time.Parse("2006-01-02", endDate)
-	endTime = endTime.AddDate(0, 0, 1)
-	log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
-	ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix()))
-}
-
 func QueryUserStaticDataPage(ctx *context.Context) {
 	startDate := ctx.Query("startDate")
 	endDate := ctx.Query("endDate")
@@ -131,8 +120,8 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 			formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
 			xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3])
 
-			formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05")
-			xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3])
+			formatTime = userRecord.DataDate
+			xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01")
 		}
 
 		//设置默认打开的表单
@@ -171,6 +160,12 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 }
 
 func TimingCountDataByDateAndReCount(date string, isReCount bool) {
+
+	if date == "refreshAll" {
+		models.RefreshUserStaticAllTabel()
+		return
+	}
+
 	t, _ := time.Parse("2006-01-02", date)
 	startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
 
@@ -223,6 +218,10 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) {
 		log.Error("count user info error." + err.Error())
 		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
 	}
+
+	if isReCount {
+		models.RefreshUserStaticAllTabel()
+	}
 }
 
 func TimingCountDataByDate(date string) {

routers/routes/routes.go

@@ -792,7 +792,6 @@ func RegisterRoutes(m *macaron.Macaron) {
 	}, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef())
 
 	m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action)
-	m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData)
 	m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage)
 	// Grouping for those endpoints not requiring authentication
 	m.Group("/:username/:reponame", func() {