diff --git a/models/dbsql/dataset_foreigntable_for_es.sql b/models/dbsql/dataset_foreigntable_for_es.sql index 5bb71348a..1ba386b3e 100644 --- a/models/dbsql/dataset_foreigntable_for_es.sql +++ b/models/dbsql/dataset_foreigntable_for_es.sql @@ -57,36 +57,41 @@ DELETE FROM public.dataset_es; CREATE OR REPLACE FUNCTION public.insert_dataset_data() RETURNS trigger AS $def$ + DECLARE + privateValue boolean=false; BEGIN - INSERT INTO public.dataset_es( - id, - title, - status, - category, - description, - download_times, - license, task, - release_id, - user_id, - repo_id, - created_unix, - updated_unix) - VALUES ( - NEW.id, - NEW.title, - NEW.status, - NEW.category, - NEW.description, - NEW.download_times, - NEW.license, - NEW.task, - NEW.release_id, - NEW.user_id, - NEW.repo_id, - NEW.created_unix, - NEW.updated_unix - ); - RETURN NEW; + select into privateValue is_private from public.repository where id=NEW.repo_id; + if not privateValue then + INSERT INTO public.dataset_es( + id, + title, + status, + category, + description, + download_times, + license, task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix) + VALUES ( + NEW.id, + NEW.title, + NEW.status, + NEW.category, + NEW.description, + NEW.download_times, + NEW.license, + NEW.task, + NEW.release_id, + NEW.user_id, + NEW.repo_id, + NEW.created_unix, + NEW.updated_unix + ); + end if; + RETURN NEW; END; $def$ LANGUAGE plpgsql; diff --git a/models/dbsql/issue_foreigntable_for_es.sql b/models/dbsql/issue_foreigntable_for_es.sql index 14d08171c..012cf9701 100644 --- a/models/dbsql/issue_foreigntable_for_es.sql +++ b/models/dbsql/issue_foreigntable_for_es.sql @@ -86,53 +86,57 @@ INSERT INTO public.issue_es( CREATE OR REPLACE FUNCTION public.insert_issue_data() RETURNS trigger AS $def$ + DECLARE + privateValue boolean=false; BEGIN - INSERT INTO public.issue_es( - id, - repo_id, - index, - poster_id, - original_author, - original_author_id, - name, - content, - milestone_id, - priority, - is_closed, - is_pull, - num_comments, - ref, - deadline_unix, - created_unix, - updated_unix, - closed_unix, - is_locked, - amount, - is_transformed) - VALUES ( - NEW.id, - NEW.repo_id, - NEW.index, - NEW.poster_id, - NEW.original_author, - NEW.original_author_id, - NEW.name, - NEW.content, - NEW.milestone_id, - NEW.priority, - NEW.is_closed, - NEW.is_pull, - NEW.num_comments, - NEW.ref, - NEW.deadline_unix, - NEW.created_unix, - NEW.updated_unix, - NEW.closed_unix, - NEW.is_locked, - NEW.amount, - NEW.is_transformed - ); - + select into privateValue is_private from public.repository where id=NEW.repo_id; + if not privateValue then + INSERT INTO public.issue_es( + id, + repo_id, + index, + poster_id, + original_author, + original_author_id, + name, + content, + milestone_id, + priority, + is_closed, + is_pull, + num_comments, + ref, + deadline_unix, + created_unix, + updated_unix, + closed_unix, + is_locked, + amount, + is_transformed) + VALUES ( + NEW.id, + NEW.repo_id, + NEW.index, + NEW.poster_id, + NEW.original_author, + NEW.original_author_id, + NEW.name, + NEW.content, + NEW.milestone_id, + NEW.priority, + NEW.is_closed, + NEW.is_pull, + NEW.num_comments, + NEW.ref, + NEW.deadline_unix, + NEW.created_unix, + NEW.updated_unix, + NEW.closed_unix, + NEW.is_locked, + NEW.amount, + NEW.is_transformed + ); + end if; RETURN NEW; END; $def$ diff --git a/models/dbsql/pr_foreigntable_for_es.sql b/models/dbsql/pr_foreigntable_for_es.sql index 96c47de79..6d28b7651 100644 --- a/models/dbsql/pr_foreigntable_for_es.sql +++ b/models/dbsql/pr_foreigntable_for_es.sql @@ -78,49 +78,53 @@ delete from public.pull_request_es; CREATE OR REPLACE FUNCTION public.insert_pull_request_data() RETURNS trigger AS $def$ + DECLARE + privateValue boolean=false; BEGIN - INSERT INTO public.pull_request_es( - id, - type, - status, - conflicted_files, - commits_ahead, - commits_behind, - issue_id, - index, - head_repo_id, - base_repo_id, - head_branch, - base_branch, - merge_base, - has_merged, - merged_commit_id, - merger_id, - merged_unix, - is_transformed, - amount) - VALUES ( - NEW.id, - NEW.type, - NEW.status, - NEW.conflicted_files, - NEW.commits_ahead, - NEW.commits_behind, - NEW.issue_id, - NEW.index, - NEW.head_repo_id, - NEW.base_repo_id, - NEW.head_branch, - NEW.base_branch, - NEW.merge_base, - NEW.has_merged, - NEW.merged_commit_id, - NEW.merger_id, - NEW.merged_unix, - NEW.is_transformed, - NEW.amount - ); - + select into privateValue is_private from public.repository where id=NEW.repo_id; + if not privateValue then + INSERT INTO public.pull_request_es( + id, + type, + status, + conflicted_files, + commits_ahead, + commits_behind, + issue_id, + index, + head_repo_id, + base_repo_id, + head_branch, + base_branch, + merge_base, + has_merged, + merged_commit_id, + merger_id, + merged_unix, + is_transformed, + amount) + VALUES ( + NEW.id, + NEW.type, + NEW.status, + NEW.conflicted_files, + NEW.commits_ahead, + NEW.commits_behind, + NEW.issue_id, + NEW.index, + NEW.head_repo_id, + NEW.base_repo_id, + NEW.head_branch, + NEW.base_branch, + NEW.merge_base, + NEW.has_merged, + NEW.merged_commit_id, + NEW.merger_id, + NEW.merged_unix, + NEW.is_transformed, + NEW.amount + ); + end if; RETURN NEW; END; $def$ diff --git a/models/dbsql/repo_foreigntable_for_es.sql b/models/dbsql/repo_foreigntable_for_es.sql index a1e5a877e..094fd4e75 100644 --- a/models/dbsql/repo_foreigntable_for_es.sql +++ b/models/dbsql/repo_foreigntable_for_es.sql @@ -1,3 +1,4 @@ +-- 要处理项目从私有变为公有,并且从公有变成私有的情况 DROP FOREIGN table if exists public.repository_es; CREATE FOREIGN TABLE public.repository_es ( id bigint NOT NULL, @@ -55,7 +56,7 @@ OPTIONS ) ; delete from public.repository_es; - INSERT INTO public.repository_es (id, +INSERT INTO public.repository_es (id, owner_id, owner_name, lower_name, @@ -138,13 +139,14 @@ delete from public.repository_es; clone_cnt, num_commit, git_clone_cnt,(select string_agg(language, ',') from public.language_stat a where a.repo_id=b.id) - FROM public.repository b; + FROM public.repository b where b.is_private=false; CREATE OR REPLACE FUNCTION public.insert_repository_data() RETURNS trigger AS $def$ BEGIN - INSERT INTO public.repository_es (id, + if not NEW.is_private then + INSERT INTO public.repository_es (id, owner_id, owner_name, lower_name, @@ -226,6 +228,7 @@ $def$ NEW.clone_cnt, NEW.num_commit, NEW.git_clone_cnt); + end if; RETURN NEW; END; $def$ @@ -242,6 +245,14 @@ CREATE TRIGGER es_insert_repository CREATE OR REPLACE FUNCTION public.update_repository() RETURNS trigger AS $def$ BEGIN + if OLD.is_private != NEW.is_private then + if OLD.is_private and not NEW.is_private then + --delete + + end if; + + end if; + update public.repository_es SET description=NEW.description, name=NEW.name, lower_name=NEW.lower_name, diff --git a/routers/search.go b/routers/search.go index fd6de2183..dce250748 100644 --- a/routers/search.go +++ b/routers/search.go @@ -3,8 +3,10 @@ package routers import ( "encoding/json" "fmt" + "strconv" "strings" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -35,9 +37,10 @@ func Search(ctx *context.Context) { if Page <= 0 { Page = 1 } - if PageSize <= 0 { + if PageSize <= 0 || PageSize > 200 { PageSize = setting.UI.IssuePagingNum } + if TableName == "repository" { searchRepo(ctx, "repository-es-index", Key, Page, PageSize) return @@ -80,11 +83,12 @@ func searchRepoByLabel(ctx *context.Context, TableName string, Key string, Page log.Info("query searchRepoByLabel start") if Key != "" { boolQ := elastic.NewBoolQuery() - topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1) + topicsQuery := elastic.NewMatchQuery("topics", Key) boolQ.Should(topicsQuery) res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context()) if err == nil { - ctx.JSON(200, res) + result := makeRepoResult(res, "") + ctx.JSON(200, result) return } else { log.Info("query es error," + err.Error()) @@ -115,9 +119,9 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa log.Info("query searchRepo start") if Key != "" { boolQ := elastic.NewBoolQuery() - nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first") - descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("desc_second") - topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("topics_third") + nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first") + descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second") + topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, descriptionQuery, topicsQuery) res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context()) if err == nil { @@ -146,16 +150,10 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string) *SearchRes { result := make([]map[string]interface{}, 0) for i, hit := range sRes.Hits.Hits { - log.Info("this is " + fmt.Sprint(i) + " result.") + log.Info("this is repo query " + fmt.Sprint(i) + " result.") recordSource := make(map[string]interface{}) source, err := hit.Source.MarshalJSON() - var isNeedToDealText bool - isNeedToDealText = false - if len(hit.MatchedQueries) > 0 && Key != "" { - if hit.MatchedQueries[0] == "desc_second" || hit.MatchedQueries[0] == "topics_third" { - isNeedToDealText = true - } - } + if err == nil { err = json.Unmarshal(source, &recordSource) if err == nil { @@ -163,35 +161,16 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string) *SearchRes { record["name"] = recordSource["name"] record["owner_name"] = recordSource["owner_name"] desc := recordSource["description"].(string) - stringlen := len(desc) - if isNeedToDealText && stringlen > 200 { - index := strings.Index(desc, Key) - if index > 0 { - start := index - 50 - if start < 0 { - start = 0 - } - end := index + 150 - if end >= stringlen { - end = stringlen - } - record["description"] = desc[start:end] - } else { - record["description"] = desc[0:200] - } - } else { - if stringlen > 200 { - record["description"] = desc[0:200] - } else { - record["description"] = desc - } - } + + record["description"] = dealLongText(desc, Key, hit.MatchedQueries) record["num_watches"] = recordSource["num_watches"] record["num_stars"] = recordSource["num_stars"] record["num_forks"] = recordSource["num_forks"] record["topics"] = recordSource["topics"] - record["avatar"] = recordSource["avatar"] + if recordSource["avatar"] != nil { + record["avatar"] = setting.AppSubURL + "/repo-avatars/" + recordSource["avatar"].(string) + } record["updated_unix"] = recordSource["updated_unix"] record["lang"] = recordSource["lang"] result = append(result, record) @@ -211,6 +190,40 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string) *SearchRes { return returnObj } +func dealLongText(text string, Key string, MatchedQueries []string) string { + var isNeedToDealText bool + isNeedToDealText = false + if len(MatchedQueries) > 0 && Key != "" { + if MatchedQueries[0] == "f_second" || MatchedQueries[0] == "f_third" { + isNeedToDealText = true + } + } + stringlen := len(text) + if isNeedToDealText && stringlen > 200 { + index := strings.Index(text, Key) + if index > 0 { + start := index - 50 + if start < 0 { + start = 0 + } + end := index + 150 + if end >= stringlen { + end = stringlen + } + return text[start:end] + } else { + return text[0:200] + } + } else { + if stringlen > 200 { + return text[0:200] + } else { + return text + } + } + +} + func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page int, PageSize int, IsQueryUser bool) { /* 用户或者组织 ES名称: user-es-index @@ -229,9 +242,9 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in ascending := ctx.QueryBool("Ascending") boolQ := elastic.NewBoolQuery() if Key != "" { - nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first") - full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("fullname_second") - descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("desc_third") + nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first") + full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("f_second") + descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, full_nameQuery, descriptionQuery) } typeValue := 1 @@ -243,13 +256,64 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context()) if err == nil { - ctx.JSON(200, res) + result := makeUserOrOrgResult(res, Key, ctx) + ctx.JSON(200, result) } else { log.Info("query es error," + err.Error()) ctx.JSON(200, "") } } +func makeUserOrOrgResult(sRes *elastic.SearchResult, Key string, ctx *context.Context) *SearchRes { + total := sRes.Hits.TotalHits.Value + result := make([]map[string]interface{}, 0) + + for i, hit := range sRes.Hits.Hits { + log.Info("this is user query " + fmt.Sprint(i) + " result.") + recordSource := make(map[string]interface{}) + source, err := hit.Source.MarshalJSON() + + if err == nil { + err = json.Unmarshal(source, &recordSource) + if err == nil { + record := make(map[string]interface{}) + record["name"] = recordSource["name"] + record["full_name"] = recordSource["full_name"] + desc := recordSource["description"].(string) + + record["description"] = dealLongText(desc, Key, hit.MatchedQueries) + if ctx.User != nil { + record["email"] = recordSource["email"] + } else { + record["email"] = "" + } + + record["location"] = recordSource["location"] + record["website"] = recordSource["website"] + record["num_repos"] = recordSource["num_repos"] + record["num_teams"] = recordSource["num_teams"] + record["num_members"] = recordSource["num_members"] + + record["avatar"] = strings.TrimRight(setting.AppSubURL, "/") + "/user/avatar/" + recordSource["name"].(string) + "/" + strconv.Itoa(-1) + record["updated_unix"] = recordSource["updated_unix"] + record["created_unix"] = recordSource["created_unix"] + result = append(result, record) + } else { + log.Info("deal source error," + err.Error()) + } + } else { + log.Info("deal source error," + err.Error()) + } + } + + returnObj := &SearchRes{ + Total: total, + Result: result, + } + + return returnObj +} + func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, PageSize int) { /* 数据集,ES名称:dataset-es-index @@ -278,22 +342,70 @@ func searchIssue(ctx *context.Context, TableName string, Key string, Page int, P boolQ := elastic.NewBoolQuery() if Key != "" { - nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first") - contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("content_second") - commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("comment_third") + nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first") + contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second") + commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, contentQuery, commentQuery) } isIssueQuery := elastic.NewTermQuery("is_pull", false) boolQ.Must(isIssueQuery) res, err := client.Search(TableName).Query(boolQ).Sort("updated_unix.keyword", false).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context()) if err == nil { - ctx.JSON(200, res) + result := makeIssueResult(res, Key) + ctx.JSON(200, result) } else { log.Info("query es error," + err.Error()) } } +func makeIssueResult(sRes *elastic.SearchResult, Key string) *SearchRes { + total := sRes.Hits.TotalHits.Value + result := make([]map[string]interface{}, 0) + + for i, hit := range sRes.Hits.Hits { + log.Info("this is issue query " + fmt.Sprint(i) + " result.") + recordSource := make(map[string]interface{}) + source, err := hit.Source.MarshalJSON() + + if err == nil { + err = json.Unmarshal(source, &recordSource) + if err == nil { + record := make(map[string]interface{}) + record["id"] = recordSource["id"] + record["repo_id"] = recordSource["repo_id"] + repo, errRepo := models.GetRepositoryByID(recordSource["repo_id"].(int64)) + if errRepo == nil { + record["repoUrl"] = repo.FullName() + } + record["name"] = recordSource["name"] + desc := recordSource["content"].(string) + record["content"] = dealLongText(desc, Key, hit.MatchedQueries) + if recordSource["pr_id"] != nil { + record["pr_id"] = recordSource["pr_id"] + } + + desc = recordSource["comment"].(string) + record["comment"] = dealLongText(desc, Key, hit.MatchedQueries) + record["num_comments"] = recordSource["num_comments"] + record["updated_unix"] = recordSource["updated_unix"] + result = append(result, record) + } else { + log.Info("deal source error," + err.Error()) + } + } else { + log.Info("deal source error," + err.Error()) + } + } + + returnObj := &SearchRes{ + Total: total, + Result: result, + } + + return returnObj +} + func searchPR(ctx *context.Context, TableName string, Key string, Page int, PageSize int) { /* @@ -308,16 +420,17 @@ func searchPR(ctx *context.Context, TableName string, Key string, Page int, Page boolQ := elastic.NewBoolQuery() if Key != "" { - nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first") - contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("content_second") - commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("comment_third") + nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first") + contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second") + commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, contentQuery, commentQuery) } isIssueQuery := elastic.NewTermQuery("is_pull", true) boolQ.Must(isIssueQuery) res, err := client.Search(TableName).Query(boolQ).Sort("updated_unix.keyword", false).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context()) if err == nil { - ctx.JSON(200, res) + result := makeIssueResult(res, Key) + ctx.JSON(200, result) } else { log.Info("query es error," + err.Error()) }