From ab16e888d269704de98def56db03957f346b2c88 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 09:07:24 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 43 ++++++++++++++++----------------- modules/setting/setting.go | 6 ----- routers/home.go | 42 -------------------------------- routers/repo/ai_model_manage.go | 29 +++++++++++++++++----- routers/routes/routes.go | 2 -- 5 files changed, 44 insertions(+), 78 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index af96444ac..ace51002d 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -11,30 +11,30 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string `xorm:"NULL"` + ID string `xorm:"pk"` + Name string `xorm:"INDEX NOT NULL"` + Version string `xorm:"NOT NULL"` + VersionCount int `xorm:"NOT NULL DEFAULT 0"` + New int `xorm:"NOT NULL"` + Type int `xorm:"NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Description string `xorm:"varchar(2000)"` + Label string `xorm:"varchar(1000)"` + Path string `xorm:"varchar(400) NOT NULL"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0"` + Engine int64 `xorm:"NOT NULL DEFAULT 0"` + Status int `xorm:"NOT NULL DEFAULT 0"` + Accuracy string `xorm:"varchar(1000)"` + AttachmentId string `xorm:"NULL"` + RepoId int64 `xorm:"INDEX NULL"` + CodeBranch string `xorm:"varchar(400) NULL"` + CodeCommitID string `xorm:"NULL"` + UserId int64 `xorm:"NOT NULL"` + UserName string UserRelAvatarLink string `xorm:"NULL"` TrainTaskInfo string `xorm:"text NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` IsCanOper bool } @@ -197,7 +197,6 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { Find(&aiModelManages); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } - sess.Close() return aiModelManages, count, nil } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2d70e47b1..dd51623c1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,9 +433,6 @@ var ( AuthUser string AuthPassword string - //home page - RecommentRepoAddr string - //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -1232,9 +1229,6 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") - sec = Cfg.Section("homepage") - RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") - sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") diff --git a/routers/home.go b/routers/home.go index 7cc353ed8..d2c93c771 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,8 +7,6 @@ package routers import ( "bytes" - "fmt" - "io/ioutil" "net/http" "strings" @@ -513,43 +511,3 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } -func RecommendOrgFromPromote(ctx *context.Context) { - url := setting.RecommentRepoAddr + "organizations" - recommendFromPromote(ctx, url) -} - -func recommendFromPromote(ctx *context.Context, url string) { - resp, err := http.Get(url) - if err != nil { - log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return - } - bytes, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return - } - - allLineStr := string(bytes) - lines := strings.Split(allLineStr, "\n") - result := make([]string, len(lines)) - for i, line := range lines { - - tmpIndex := strings.Index(line, ".") - log.Info("i=" + fmt.Sprint(i) + " line=" + line + " tmpIndex=" + fmt.Sprint(tmpIndex)) - if tmpIndex == -1 { - result[i] = strings.Trim(line, " ") - } else { - result[i] = strings.Trim(line[tmpIndex+1:], " ") - } - } - ctx.JSON(http.StatusOK, result) -} - -func RecommendRepoFromPromote(ctx *context.Context) { - url := setting.RecommentRepoAddr + "projects" - recommendFromPromote(ctx, url) -} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index c6ec6c6ae..d3afe61a4 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -28,7 +28,6 @@ const ( func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, ctx *context.Context) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) - //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { log.Info("query task error." + err.Error()) return err @@ -71,7 +70,6 @@ func saveModelByParameters(jobId string, versionName string, name string, versio log.Info("accuracyJson=" + string(accuracyJson)) aiTaskJson, _ := json.Marshal(aiTask) - //taskConfigInfo,err := models.GetCloudbrainByJobIDAndVersionName(jobId,aiTask.VersionName) model := &models.AiModelManage{ ID: id, Version: version, @@ -86,7 +84,6 @@ func saveModelByParameters(jobId string, versionName string, name string, versio AttachmentId: aiTask.Uuid, RepoId: aiTask.RepoID, UserId: ctx.User.ID, - UserName: ctx.User.Name, UserRelAvatarLink: ctx.User.RelAvatarLink(), CodeBranch: aiTask.BranchName, CodeCommitID: aiTask.CommitID, @@ -399,11 +396,22 @@ func ShowOneVersionOtherModel(ctx *context.Context) { repoId := ctx.Repo.Repository.ID name := ctx.Query("name") aimodels := models.QueryModelByName(name, repoId) - for _, model := range aimodels { + + userIds := make([]int64, len(aimodels)) + for i, model := range aimodels { log.Info("model=" + model.Name) log.Info("model.UserId=" + fmt.Sprint(model.UserId)) model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + + userNames, err := models.GetUserNamesByIDs(userIds) + if err == nil { + for i, model := range aimodels { + model.UserName = userNames[i] + } } + if len(aimodels) > 0 { ctx.JSON(200, aimodels[1:]) } else { @@ -431,7 +439,7 @@ func isOper(ctx *context.Context, modelUserId int64) bool { if ctx.User == nil { return false } - if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { + if ctx.User.IsAdmin || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { return true } return false @@ -463,10 +471,19 @@ func ShowModelPageInfo(ctx *context.Context) { return } - for _, model := range modelResult { + userIds := make([]int64, len(modelResult)) + for i, model := range modelResult { log.Info("model=" + model.Name) log.Info("model.UserId=" + fmt.Sprint(model.UserId)) model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + + userNames, err := models.GetUserNamesByIDs(userIds) + if err == nil { + for i, model := range modelResult { + model.UserName = userNames[i] + } } mapInterface := make(map[string]interface{}) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a18fe422c..a8f820dba 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,8 +315,6 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) - m.Get("/recommend/org", routers.RecommendOrgFromPromote) - m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") From 9599b2dfd0dceaf4218ee67f9b0cb88ada148538 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 09:56:35 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 1 - routers/repo/ai_model_manage.go | 43 ++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 15f67e858..cf0065282 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -649,7 +649,6 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { } } return resultMap - } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index d3afe61a4..0fdf698c0 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -55,7 +55,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio cloudType = aiTask.Type //download model zip //train type if cloudType == models.TypeCloudBrainTwo { - modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "", aiTask.TrainUrl) if err != nil { log.Info("download model from CloudBrainTwo faild." + err.Error()) return err @@ -141,9 +141,12 @@ func SaveModel(ctx *context.Context) { log.Info("save model end.") } -func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { - +func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + if trainUrl != "" { + objectkey = trainUrl[len(setting.Bucket)+1:] + } + modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { @@ -392,6 +395,24 @@ func ShowSingleModel(ctx *context.Context) { ctx.JSON(http.StatusOK, models) } +func queryUserName(intSlice []int64) map[int64]string { + keys := make(map[int64]string) + uniqueElements := []int64{} + for _, entry := range intSlice { + if _, value := keys[entry]; !value { + keys[entry] = "" + uniqueElements = append(uniqueElements, entry) + } + } + userNames, err := models.GetUserNamesByIDs(uniqueElements) + if err == nil { + for i, userName := range userNames { + keys[uniqueElements[i]] = userName + } + } + return keys +} + func ShowOneVersionOtherModel(ctx *context.Context) { repoId := ctx.Repo.Repository.ID name := ctx.Query("name") @@ -404,12 +425,10 @@ func ShowOneVersionOtherModel(ctx *context.Context) { model.IsCanOper = isOper(ctx, model.UserId) userIds[i] = model.UserId } + userNameMap := queryUserName(userIds) - userNames, err := models.GetUserNamesByIDs(userIds) - if err == nil { - for i, model := range aimodels { - model.UserName = userNames[i] - } + for _, model := range aimodels { + model.UserName = userNameMap[model.UserId] } if len(aimodels) > 0 { @@ -479,11 +498,9 @@ func ShowModelPageInfo(ctx *context.Context) { userIds[i] = model.UserId } - userNames, err := models.GetUserNamesByIDs(userIds) - if err == nil { - for i, model := range modelResult { - model.UserName = userNames[i] - } + userNameMap := queryUserName(userIds) + for _, model := range modelResult { + model.UserName = userNameMap[model.UserId] } mapInterface := make(map[string]interface{}) From 1a6c5c02983a03aceb6531b5867fb9fcf644f4f9 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 10:04:40 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 0fdf698c0..02aa94c39 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -144,7 +144,7 @@ func SaveModel(ctx *context.Context) { func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") if trainUrl != "" { - objectkey = trainUrl[len(setting.Bucket)+1:] + objectkey = trainUrl[len(setting.Bucket)+2:] } modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") From f0e9abeeaa79dc132d554bcf54aa43603908b5b6 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 10:18:15 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 02aa94c39..5103c2a5e 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -144,7 +144,7 @@ func SaveModel(ctx *context.Context) { func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") if trainUrl != "" { - objectkey = trainUrl[len(setting.Bucket)+2:] + objectkey = strings.Trim(trainUrl[len(setting.Bucket)+1:], "/") } modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") From cd05bc1cdd1a1114e1c203dce1bcac674a2ebe00 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 10:28:15 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E6=81=A2=E5=A4=8D=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/setting/setting.go | 6 ++++++ routers/home.go | 42 ++++++++++++++++++++++++++++++++++++++ routers/routes/routes.go | 2 ++ 3 files changed, 50 insertions(+) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index dd51623c1..2d70e47b1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,6 +433,9 @@ var ( AuthUser string AuthPassword string + //home page + RecommentRepoAddr string + //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -1229,6 +1232,9 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") + sec = Cfg.Section("homepage") + RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") + sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") diff --git a/routers/home.go b/routers/home.go index d2c93c771..7cc353ed8 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,6 +7,8 @@ package routers import ( "bytes" + "fmt" + "io/ioutil" "net/http" "strings" @@ -511,3 +513,43 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } +func RecommendOrgFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "organizations" + recommendFromPromote(ctx, url) +} + +func recommendFromPromote(ctx *context.Context, url string) { + resp, err := http.Get(url) + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + + allLineStr := string(bytes) + lines := strings.Split(allLineStr, "\n") + result := make([]string, len(lines)) + for i, line := range lines { + + tmpIndex := strings.Index(line, ".") + log.Info("i=" + fmt.Sprint(i) + " line=" + line + " tmpIndex=" + fmt.Sprint(tmpIndex)) + if tmpIndex == -1 { + result[i] = strings.Trim(line, " ") + } else { + result[i] = strings.Trim(line[tmpIndex+1:], " ") + } + } + ctx.JSON(http.StatusOK, result) +} + +func RecommendRepoFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "projects" + recommendFromPromote(ctx, url) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a8f820dba..a18fe422c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) + m.Get("/recommend/org", routers.RecommendOrgFromPromote) + m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos")