From 5cecfa5159029e19f841bc4507c6a5da1a5d30e5 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 28 Apr 2022 15:37:28 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81=E3=80=82?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=BB=9F=E8=AE=A1=E6=96=B0=E5=A2=9E=E5=B1=9E?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/dbsql/repo_foreigntable_for_es.sql | 10 +- models/user_business_analysis.go | 144 +++++++++++++++++++++- models/user_business_struct.go | 17 ++- options/locale/locale_zh-CN.ini | 13 ++ routers/repo/user_data_analysis.go | 82 +++++++++++- 5 files changed, 247 insertions(+), 19 deletions(-) diff --git a/models/dbsql/repo_foreigntable_for_es.sql b/models/dbsql/repo_foreigntable_for_es.sql index 7e06fd99e..e927eb7c2 100644 --- a/models/dbsql/repo_foreigntable_for_es.sql +++ b/models/dbsql/repo_foreigntable_for_es.sql @@ -523,17 +523,21 @@ DROP TRIGGER IF EXISTS es_udpate_repository_lang on public.language_stat; CREATE OR REPLACE FUNCTION public.udpate_repository_lang() RETURNS trigger AS $def$ + DECLARE + privateValue bigint; BEGIN if (TG_OP = 'UPDATE') then - update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; + select into privateValue updated_unix from public.repository where id=NEW.repo_id; + update public.repository_es SET updated_unix=privateValue,lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; elsif (TG_OP = 'INSERT') then - update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; + select into privateValue updated_unix from public.repository where id=NEW.repo_id; + update public.repository_es SET updated_unix=privateValue,lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; elsif (TG_OP = 'DELETE') then if exists(select 1 from public.repository where id=OLD.repo_id) then update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=OLD.repo_id) where id=OLD.repo_id; end if; end if; - return null; + return NEW; END; $def$ LANGUAGE plpgsql; diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 2d7592baf..7e9e877c2 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" "code.gitea.io/gitea/modules/log" @@ -103,6 +104,8 @@ type UserBusinessAnalysisAll struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + HasActivity int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysis struct { @@ -190,6 +193,8 @@ type UserBusinessAnalysis struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + HasActivity int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisQueryOptions struct { @@ -227,15 +232,93 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } -func QueryMetrics(start int64, end int64) ([]*UserMetrics, int64) { +func QueryMetricsPage(start int64, end int64, page int, pageSize int) ([]UserMetrics, int64) { + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + cond := "count_date >" + fmt.Sprint(start) + " and count_date<" + fmt.Sprint(end) + allCount, err := statictisSess.Where(cond).Count(new(UserMetrics)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + userMetricsList := make([]UserMetrics, 0) + if err := statictisSess.Table(new(UserMetrics)).Where(cond).Limit(pageSize, page*pageSize).OrderBy("count_date desc"). + Find(&userMetricsList); err != nil { + return nil, 0 + } + return userMetricsList, allCount + +} + +func QueryMetrics(start int64, end int64) ([]UserMetrics, int) { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - userMetricsList := make([]*UserMetrics, 0) + userMetricsList := make([]UserMetrics, 0) if err := statictisSess.Table(new(UserMetrics)).Where("count_date >" + fmt.Sprint(start) + " and count_date<" + fmt.Sprint(end)).OrderBy("count_date desc"). Find(&userMetricsList); err != nil { return nil, 0 } - return userMetricsList, int64(len(userMetricsList)) + return userMetricsList, len(userMetricsList) +} + +func QueryMetricsForAll() []UserMetrics { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + userMetricsList := make([]UserMetrics, 0) + if err := statictisSess.Table(new(UserMetrics)).OrderBy("count_date desc"). + Find(&userMetricsList); err != nil { + return nil + } + return makeResultForMonth(userMetricsList, len(userMetricsList)) +} + +func QueryMetricsForYear() []UserMetrics { + currentTimeNow := time.Now() + currentYearEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + currentYearStartTime := time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location()) + allUserInfo, count := QueryMetrics(currentYearStartTime.Unix(), currentYearEndTime.Unix()) + + return makeResultForMonth(allUserInfo, count) +} + +func makeResultForMonth(allUserInfo []UserMetrics, count int) []UserMetrics { + monthMap := make(map[string]UserMetrics) + if count > 0 { + for _, userMetrics := range allUserInfo { + dateTime := time.Unix(userMetrics.CountDate, 0) + month := fmt.Sprint(dateTime.Year()) + "-" + fmt.Sprint(int(dateTime.Month())) + if _, ok := monthMap[month]; !ok { + var monthUserMetrics UserMetrics + monthUserMetrics.DisplayDate = month + monthUserMetrics.ActivateRegistUser = userMetrics.ActivateRegistUser + monthUserMetrics.NotActivateRegistUser = userMetrics.NotActivateRegistUser + monthUserMetrics.TotalUser = userMetrics.TotalUser + monthUserMetrics.TotalActivateRegistUser = userMetrics.TotalActivateRegistUser + monthUserMetrics.TotalHasActivityUser = userMetrics.TotalHasActivityUser + monthUserMetrics.HasActivityUser = userMetrics.HasActivityUser + monthUserMetrics.DaysForMonth = 1 + monthMap[month] = monthUserMetrics + } else { + value := monthMap[month] + value.ActivateRegistUser += userMetrics.ActivateRegistUser + value.NotActivateRegistUser += userMetrics.NotActivateRegistUser + value.TotalUser += userMetrics.TotalUser + value.TotalActivateRegistUser += userMetrics.TotalActivateRegistUser + value.TotalHasActivityUser += userMetrics.TotalHasActivityUser + value.HasActivityUser += userMetrics.HasActivityUser + value.DaysForMonth += 1 + } + } + } + result := make([]UserMetrics, 0) + for _, value := range monthMap { + result = append(result, value) + } + sort.Slice(result, func(i, j int) bool { + return strings.Compare(result[i].DisplayDate, result[j].DisplayDate) > 0 + }) + return result } func QueryRankList(key string, tableName string, limit int) ([]*UserBusinessAnalysisAll, int64) { @@ -540,6 +623,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS if minUserIndex > dateRecordAll.UserIndexPrimitive { minUserIndex = dateRecordAll.UserIndexPrimitive } + dateRecordBatch = append(dateRecordBatch, dateRecordAll) if len(dateRecordBatch) >= BATCH_INSERT_SIZE { insertTable(dateRecordBatch, tableName, statictisSess) @@ -695,7 +779,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("query user error. return.") return err } - + userNewAddActivity := make(map[int64]map[int64]int64) ParaWeight := getParaWeight() userMetrics := make(map[string]int) var indexTotal int64 @@ -767,6 +851,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.UserIndexPrimitive = getUserIndex(dateRecord, ParaWeight) setUserMetrics(userMetrics, userRecord, start_unix, end_unix, dateRecord) + if getUserActivate(dateRecord) > 0 { + addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID) + } _, err = statictisSess.Insert(&dateRecord) if err != nil { log.Info("insert daterecord failed." + err.Error()) @@ -792,11 +879,58 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, useMetrics.NotActivateRegistUser = getMapKeyStringValue("NotActivateRegistUser", userMetrics) useMetrics.TotalActivateRegistUser = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) useMetrics.TotalHasActivityUser = getMapKeyStringValue("TotalHasActivityUser", userMetrics) - statictisSess.Insert(&useMetrics) + count, err = sess.Count(new(User)) + if err != nil { + log.Info("query user error. return.") + } + useMetrics.TotalUser = int(count) + useMetrics.ActivateIIndex = float64(useMetrics.ActivateRegistUser) / float64(useMetrics.ActivateRegistUser+useMetrics.NotActivateRegistUser) + statictisSess.Insert(&useMetrics) + //update new user activity + updateNewUserAcitivity(userNewAddActivity, statictisSess) return nil } +func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) { + for key, value := range currentUserActivity { + useMetrics := &UserMetrics{CountDate: key} + has, err := statictisSess.Get(useMetrics) + if err == nil && has { + userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",") + for _, userIdStr := range userIdArrays { + userIdInt, err := strconv.ParseInt(userIdStr, 10, 64) + if err == nil { + value[userIdInt] = userIdInt + } + } + userIdArray := "" + for _, tmpValue := range value { + userIdArray += fmt.Sprint(tmpValue) + "," + } + useMetrics.HasActivityUser = len(value) + if len(userIdArray) > 0 { + useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] + + } + statictisSess.Update(useMetrics) + } + } +} + +func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64) { + CountDateTime := time.Date(registDate.Year(), registDate.AsTime().Month(), registDate.AsTime().Day(), 0, 1, 0, 0, registDate.AsTime().Location()) + CountDate := CountDateTime.Unix() + if _, ok := currentUserActivity[CountDate]; !ok { + userIdMap := make(map[int64]int64, 0) + userIdMap[userId] = userId + currentUserActivity[CountDate] = userIdMap + } else { + currentUserActivity[CountDate][userId] = userId + } + +} + func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, end_time int64, dateRecord UserBusinessAnalysis) { //ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` //NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 86aecd545..5560cb19a 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -400,10 +400,15 @@ type UserAnalysisPara struct { } type UserMetrics struct { - CountDate int64 `xorm:"pk"` - ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` - TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - TotalHasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + CountDate int64 `xorm:"pk"` + ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + ActivateIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + TotalUser int `xorm:"NOT NULL DEFAULT 0"` + TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + TotalHasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + DisplayDate string `xorm:"-"` + DaysForMonth int `xorm:"NOT NULL DEFAULT 0"` + HasActivityUserJson string `xorm:"NULL"` } diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 474b96c3f..da70e984d 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -530,6 +530,19 @@ static.public.user_business_analysis_last30_day=近30天 static.public.user_business_analysis_last_month=上月 static.public.user_business_analysis_yesterday=昨天 static.public.user_business_analysis_all=所有 + +metrics.sheetname=用户趋势分析 +metrics.date=日期 +metrics.newregistuser=新增注册用户 +metrics.newregistandactiveuser=新增已激活 +metrics.hasactivateuser=新增有贡献活动 +metrics.newregistnotactiveuser=新增未激活 +metrics.averageuser=平均新增用户 +metrics.newuseractiveindex=新增用户激活率 +metrics.totalregistuser=累计注册用户 +metrics.totalregistuser=累计已激活 +metrics.totalregistuser=累计有贡献活动 + [settings] profile=个人信息 account=账号 diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2280e8288..240613566 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -19,6 +19,53 @@ const ( PAGE_SIZE = 2000 ) +func getUserMetricsExcelHeader(ctx *context.Context) map[string]string { + excelHeader := make([]string, 0) + excelHeader = append(excelHeader, ctx.Tr("user.static.id")) + excelHeader = append(excelHeader, ctx.Tr("user.static.name")) + excelHeader = append(excelHeader, ctx.Tr("user.static.UserIndex")) + excelHeader = append(excelHeader, ctx.Tr("user.static.UserIndexPrimitive")) + excelHeader = append(excelHeader, ctx.Tr("user.static.codemergecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commitcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.issuecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commentcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.focusrepocount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.starrepocount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.logincount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.watchedcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commitcodesize")) + excelHeader = append(excelHeader, ctx.Tr("user.static.solveissuecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.encyclopediascount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.createrepocount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.openiindex")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainTaskNum")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) + + excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectedDataset")) + excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendDataset")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectImage")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectedImage")) + excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendImage")) + + excelHeader = append(excelHeader, ctx.Tr("user.static.registdate")) + excelHeader = append(excelHeader, ctx.Tr("user.static.countdate")) + + excelHeaderMap := make(map[string]string, 0) + var i byte + i = 0 + for _, value := range excelHeader { + excelColumn := getColumn(i) + fmt.Sprint(1) + log.Info("excelColumn=" + excelColumn) + excelHeaderMap[excelColumn] = value + i++ + } + return excelHeaderMap +} + func getExcelHeader(ctx *context.Context) map[string]string { excelHeader := make([]string, 0) excelHeader = append(excelHeader, ctx.Tr("user.static.id")) @@ -205,11 +252,36 @@ func QueryMetrics(ctx *context.Context) { endDate := ctx.Query("endDate") startTime, _ := time.ParseInLocation("2006-01-02", startDate, time.Local) endTime, _ := time.ParseInLocation("2006-01-02", endDate, time.Local) - result, count := models.QueryMetrics(startTime.Unix(), endTime.Unix()) - mapInterface := make(map[string]interface{}) - mapInterface["data"] = result - mapInterface["count"] = count - ctx.JSON(http.StatusOK, mapInterface) + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + IsReturnFile := ctx.QueryBool("IsReturnFile") + + if IsReturnFile { + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.metrics.sheetname") + //index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + dataHeader := getUserMetricsExcelHeader(ctx) + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + } else { + result, count := models.QueryMetricsPage(startTime.Unix(), endTime.Unix(), page, pageSize) + mapInterface := make(map[string]interface{}) + mapInterface["data"] = result + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) + } + } func QueryRankingList(ctx *context.Context) {