diff --git a/modules/repository/elk_pagedata.go b/modules/repository/elk_pagedata.go index bb027726d..03a61be1f 100644 --- a/modules/repository/elk_pagedata.go +++ b/modules/repository/elk_pagedata.go @@ -4,10 +4,10 @@ import ( "bytes" "encoding/base64" "encoding/json" - "fmt" "io/ioutil" "net/http" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) @@ -99,8 +99,8 @@ type ResultInfo struct { //elk输出的json结构end -//发送post请求到elk -func SendReqToElk(jsonStr []byte) (content string) { +//处理返回的elk数据,只保留totalView,即访问量;loaded是分片载入次数,用来判断返回的数据是否准确 +func GetResultFromElk(resultInfo ResultInfo, jsonStr []byte) (loaded int, totalView int, err error) { ElkBase64Init := setting.ElkUser + ":" + setting.ElkPassword ElkBase64 := base64.StdEncoding.EncodeToString([]byte(ElkBase64Init)) BasicElkBase64 := "Basic" + " " + ElkBase64 @@ -117,15 +117,11 @@ func SendReqToElk(jsonStr []byte) (content string) { } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) - return string(body) -} -//处理返回的elk数据,只保留totalView,即访问量;loaded是分片载入次数,用来判断返回的数据是否准确 -func GetResultFromElk(resultinfo ResultInfo, jobResult string) (loaded int, totalView int) { - var resultTest ResultInfo - errs := json.Unmarshal([]byte(jobResult), &resultTest) - fmt.Println(errs) - return resultTest.Result.Loaded, resultTest.Result.RawResponse.Hits.Total + errs := json.Unmarshal([]byte(string(body)), &resultInfo) + log.Info("Get resultJson failed", errs) + + return resultInfo.Result.Loaded, resultInfo.Result.RawResponse.Hits.Total, err } //初始化传给elk的数据结构,给定用户名和项目名,查询的起止时间,返回初始化后的结构 @@ -211,23 +207,23 @@ func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (pr } //向elk发送请求,将获取的结果只保留访问量,输入是初始化后的数据结构,返回访问量 -func ViewInfo(viewInfo InputInfo) (totalView int) { +func ViewInfo(viewInfo InputInfo) (totalView int, err error) { jsons, errs := json.Marshal(viewInfo) if errs != nil { - fmt.Println("errs:", errs.Error()) + log.Info("errs:", errs) } - // fmt.Println("viewInfoInit:",string(jsons)) var jsonStr = []byte(jsons) var resultInfo ResultInfo - loaded, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr)) + loaded, totalView, err := GetResultFromElk(resultInfo, jsonStr) + time := 0 for { if loaded == 0 { - loaded_next, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr)) + loaded_next, totalView, err := GetResultFromElk(resultInfo, jsonStr) time++ + log.Info("time:", time) if loaded_next != 0 && time < 100 { - fmt.Println("totalView:", totalView) - return totalView + return totalView, err } if time > 100 { break @@ -236,20 +232,20 @@ func ViewInfo(viewInfo InputInfo) (totalView int) { break } } - fmt.Println("loaded:", loaded) - return totalView + return totalView, err } -// @title ProjectView +// @title AppointProjectView // @description 获取指定用户和项目的访问量 // @param User string "用户名" // @param Project string "项目名" // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) // @param Lte string "结束时间" 如time.Now().Format(time.RFC3339) // @return totalView int "访问量" -func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int) { - InitInfo := ProjectViewInit(User, Project, Gte, Lte) - return ViewInfo(InitInfo) +func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int, err error) { + ProjectViewInitInfo := ProjectViewInit(User, Project, Gte, Lte) + ProjectTotalView, err := ViewInfo(ProjectViewInitInfo) + return ProjectTotalView, err } //统计项目相关页面的访问量 @@ -287,26 +283,44 @@ type ProjectInfo struct { Project_forks int } +type ErrorInfo struct { + Project_dataset_type_0 error + Project_dataset_type_1 error + Project_issues error + Project_labels error + Project_milestones error + Project_pulls error + Project_release error + Project_wiki error + Project_activity error + Project_cloudbrain error + Project_modelarts error + Project_blockchain error + Project_watchers error + Project_stars error + Project_forks error +} + // @title AllProjectView // @description 获取指定用户和项目的访问量 // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) // @param Lte string "结束时间" // @return projectInfo ProjectInfo "统计所有项目中页面的浏览情况,不需要区分项目" -func AllProjectView(Gte string, Lte string) (projectInfo ProjectInfo) { - projectInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte)) - projectInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte)) - return projectInfo +func AllProjectView(Gte string, Lte string) (projectViewInfo ProjectInfo, errorInfo ErrorInfo) { + projectViewInfo.Project_dataset_type_0, errorInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_dataset_type_1, errorInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_issues, errorInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_labels, errorInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte)) + projectViewInfo.Project_milestones, errorInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_pulls, errorInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_release, errorInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_wiki, errorInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_activity, errorInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_cloudbrain, errorInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_modelarts, errorInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_blockchain, errorInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_watchers, errorInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_stars, errorInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_forks, errorInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte)) + return projectViewInfo, errorInfo } diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index ceb410958..6ee5c9a59 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" ) //auto daily or manually @@ -62,9 +63,12 @@ func RepoStatisticDaily(date string) { continue } - //beginTime, endTime := getStatTime(date) - //numVisits := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) - numVisits := 0 + beginTime, endTime := getStatTime(date) + numVisits, err := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) + if err != nil { + log.Error("Get numVisits failed", err) + numVisits = 0 + } repoStat := models.RepoStatistic{ RepoID: repo.ID,