wangwei
/
aiforge
Not watched
1
0
Fork 0
Code
Releases 128 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

提交代码。 

Signed-off-by: zouap <zouap@pcl.ac.cn>
tags/v1.22.3.2^2
zouap 4 years ago
parent
37838ce367
commit
a2e154de25
5 changed files with 307 additions and 170 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +34
    -29
      models/dbsql/dataset_foreigntable_for_es.sql
  2. +50
    -46
      models/dbsql/issue_foreigntable_for_es.sql
  3. +46
    -42
      models/dbsql/pr_foreigntable_for_es.sql
  4. +14
    -3
      models/dbsql/repo_foreigntable_for_es.sql
  5. +163
    -50
      routers/search.go

+ 34
- 29
models/dbsql/dataset_foreigntable_for_es.sql View File

@@ -57,36 +57,41 @@ DELETE FROM public.dataset_es;

CREATE OR REPLACE FUNCTION public.insert_dataset_data() RETURNS trigger AS
$def$
DECLARE
privateValue boolean=false;
BEGIN
INSERT INTO public.dataset_es(
id,
title,
status,
category,
description,
download_times,
license, task,
release_id,
user_id,
repo_id,
created_unix,
updated_unix)
VALUES (
NEW.id,
NEW.title,
NEW.status,
NEW.category,
NEW.description,
NEW.download_times,
NEW.license,
NEW.task,
NEW.release_id,
NEW.user_id,
NEW.repo_id,
NEW.created_unix,
NEW.updated_unix
);
RETURN NEW;
select into privateValue is_private from public.repository where id=NEW.repo_id;
if not privateValue then
INSERT INTO public.dataset_es(
id,
title,
status,
category,
description,
download_times,
license, task,
release_id,
user_id,
repo_id,
created_unix,
updated_unix)
VALUES (
NEW.id,
NEW.title,
NEW.status,
NEW.category,
NEW.description,
NEW.download_times,
NEW.license,
NEW.task,
NEW.release_id,
NEW.user_id,
NEW.repo_id,
NEW.created_unix,
NEW.updated_unix
);
end if;
RETURN NEW;
END;
$def$
LANGUAGE plpgsql;


+ 50
- 46
models/dbsql/issue_foreigntable_for_es.sql View File

@@ -86,53 +86,57 @@ INSERT INTO public.issue_es(

CREATE OR REPLACE FUNCTION public.insert_issue_data() RETURNS trigger AS
$def$
DECLARE
privateValue boolean=false;
BEGIN
INSERT INTO public.issue_es(
id,
repo_id,
index,
poster_id,
original_author,
original_author_id,
name,
content,
milestone_id,
priority,
is_closed,
is_pull,
num_comments,
ref,
deadline_unix,
created_unix,
updated_unix,
closed_unix,
is_locked,
amount,
is_transformed)
VALUES (
NEW.id,
NEW.repo_id,
NEW.index,
NEW.poster_id,
NEW.original_author,
NEW.original_author_id,
NEW.name,
NEW.content,
NEW.milestone_id,
NEW.priority,
NEW.is_closed,
NEW.is_pull,
NEW.num_comments,
NEW.ref,
NEW.deadline_unix,
NEW.created_unix,
NEW.updated_unix,
NEW.closed_unix,
NEW.is_locked,
NEW.amount,
NEW.is_transformed
);
select into privateValue is_private from public.repository where id=NEW.repo_id;
if not privateValue then
INSERT INTO public.issue_es(
id,
repo_id,
index,
poster_id,
original_author,
original_author_id,
name,
content,
milestone_id,
priority,
is_closed,
is_pull,
num_comments,
ref,
deadline_unix,
created_unix,
updated_unix,
closed_unix,
is_locked,
amount,
is_transformed)
VALUES (
NEW.id,
NEW.repo_id,
NEW.index,
NEW.poster_id,
NEW.original_author,
NEW.original_author_id,
NEW.name,
NEW.content,
NEW.milestone_id,
NEW.priority,
NEW.is_closed,
NEW.is_pull,
NEW.num_comments,
NEW.ref,
NEW.deadline_unix,
NEW.created_unix,
NEW.updated_unix,
NEW.closed_unix,
NEW.is_locked,
NEW.amount,
NEW.is_transformed
);
end if;
RETURN NEW;
END;
$def$


+ 46
- 42
models/dbsql/pr_foreigntable_for_es.sql View File

@@ -78,49 +78,53 @@ delete from public.pull_request_es;

CREATE OR REPLACE FUNCTION public.insert_pull_request_data() RETURNS trigger AS
$def$
DECLARE
privateValue boolean=false;
BEGIN
INSERT INTO public.pull_request_es(
id,
type,
status,
conflicted_files,
commits_ahead,
commits_behind,
issue_id,
index,
head_repo_id,
base_repo_id,
head_branch,
base_branch,
merge_base,
has_merged,
merged_commit_id,
merger_id,
merged_unix,
is_transformed,
amount)
VALUES (
NEW.id,
NEW.type,
NEW.status,
NEW.conflicted_files,
NEW.commits_ahead,
NEW.commits_behind,
NEW.issue_id,
NEW.index,
NEW.head_repo_id,
NEW.base_repo_id,
NEW.head_branch,
NEW.base_branch,
NEW.merge_base,
NEW.has_merged,
NEW.merged_commit_id,
NEW.merger_id,
NEW.merged_unix,
NEW.is_transformed,
NEW.amount
);
select into privateValue is_private from public.repository where id=NEW.repo_id;
if not privateValue then
INSERT INTO public.pull_request_es(
id,
type,
status,
conflicted_files,
commits_ahead,
commits_behind,
issue_id,
index,
head_repo_id,
base_repo_id,
head_branch,
base_branch,
merge_base,
has_merged,
merged_commit_id,
merger_id,
merged_unix,
is_transformed,
amount)
VALUES (
NEW.id,
NEW.type,
NEW.status,
NEW.conflicted_files,
NEW.commits_ahead,
NEW.commits_behind,
NEW.issue_id,
NEW.index,
NEW.head_repo_id,
NEW.base_repo_id,
NEW.head_branch,
NEW.base_branch,
NEW.merge_base,
NEW.has_merged,
NEW.merged_commit_id,
NEW.merger_id,
NEW.merged_unix,
NEW.is_transformed,
NEW.amount
);
end if;
RETURN NEW;
END;
$def$


+ 14
- 3
models/dbsql/repo_foreigntable_for_es.sql View File

@@ -1,3 +1,4 @@
-- 要处理项目从私有变为公有，并且从公有变成私有的情况
DROP FOREIGN table if exists public.repository_es;
CREATE FOREIGN TABLE public.repository_es (
id bigint NOT NULL,
@@ -55,7 +56,7 @@ OPTIONS
)
;
delete from public.repository_es;
INSERT INTO public.repository_es (id,
INSERT INTO public.repository_es (id,
owner_id,
owner_name,
lower_name,
@@ -138,13 +139,14 @@ delete from public.repository_es;
clone_cnt,
num_commit,
git_clone_cnt,(select string_agg(language, ',') from public.language_stat a where a.repo_id=b.id)
FROM public.repository b;
FROM public.repository b where b.is_private=false;


CREATE OR REPLACE FUNCTION public.insert_repository_data() RETURNS trigger AS
$def$
BEGIN
INSERT INTO public.repository_es (id,
if not NEW.is_private then
INSERT INTO public.repository_es (id,
owner_id,
owner_name,
lower_name,
@@ -226,6 +228,7 @@ $def$
NEW.clone_cnt,
NEW.num_commit,
NEW.git_clone_cnt);
end if;
RETURN NEW;
END;
$def$
@@ -242,6 +245,14 @@ CREATE TRIGGER es_insert_repository
CREATE OR REPLACE FUNCTION public.update_repository() RETURNS trigger AS
$def$
BEGIN
if OLD.is_private != NEW.is_private then
if OLD.is_private and not NEW.is_private then
--delete
end if;

end if;

update public.repository_es SET description=NEW.description,
name=NEW.name,
lower_name=NEW.lower_name,


+ 163
- 50
routers/search.go View File

@@ -3,8 +3,10 @@ package routers
import (
"encoding/json"
"fmt"
"strconv"
"strings"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@@ -35,9 +37,10 @@ func Search(ctx *context.Context) {
if Page <= 0 {
Page = 1
}
if PageSize <= 0 {
if PageSize <= 0 || PageSize > 200 {
PageSize = setting.UI.IssuePagingNum
}

if TableName == "repository" {
searchRepo(ctx, "repository-es-index", Key, Page, PageSize)
return
@@ -80,11 +83,12 @@ func searchRepoByLabel(ctx *context.Context, TableName string, Key string, Page
log.Info("query searchRepoByLabel start")
if Key != "" {
boolQ := elastic.NewBoolQuery()
topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1)
topicsQuery := elastic.NewMatchQuery("topics", Key)
boolQ.Should(topicsQuery)
res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
if err == nil {
ctx.JSON(200, res)
result := makeRepoResult(res, "")
ctx.JSON(200, result)
return
} else {
log.Info("query es error," + err.Error())
@@ -115,9 +119,9 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa
log.Info("query searchRepo start")
if Key != "" {
boolQ := elastic.NewBoolQuery()
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first")
descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("desc_second")
topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("topics_third")
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second")
topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third")
boolQ.Should(nameQuery, descriptionQuery, topicsQuery)
res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
if err == nil {
@@ -146,16 +150,10 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string) *SearchRes {
result := make([]map[string]interface{}, 0)

for i, hit := range sRes.Hits.Hits {
log.Info("this is " + fmt.Sprint(i) + " result.")
log.Info("this is repo query " + fmt.Sprint(i) + " result.")
recordSource := make(map[string]interface{})
source, err := hit.Source.MarshalJSON()
var isNeedToDealText bool
isNeedToDealText = false
if len(hit.MatchedQueries) > 0 && Key != "" {
if hit.MatchedQueries[0] == "desc_second" || hit.MatchedQueries[0] == "topics_third" {
isNeedToDealText = true
}
}

if err == nil {
err = json.Unmarshal(source, &recordSource)
if err == nil {
@@ -163,35 +161,16 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string) *SearchRes {
record["name"] = recordSource["name"]
record["owner_name"] = recordSource["owner_name"]
desc := recordSource["description"].(string)
stringlen := len(desc)
if isNeedToDealText && stringlen > 200 {
index := strings.Index(desc, Key)
if index > 0 {
start := index - 50
if start < 0 {
start = 0
}
end := index + 150
if end >= stringlen {
end = stringlen
}
record["description"] = desc[start:end]
} else {
record["description"] = desc[0:200]
}
} else {
if stringlen > 200 {
record["description"] = desc[0:200]
} else {
record["description"] = desc
}
}

record["description"] = dealLongText(desc, Key, hit.MatchedQueries)

record["num_watches"] = recordSource["num_watches"]
record["num_stars"] = recordSource["num_stars"]
record["num_forks"] = recordSource["num_forks"]
record["topics"] = recordSource["topics"]
record["avatar"] = recordSource["avatar"]
if recordSource["avatar"] != nil {
record["avatar"] = setting.AppSubURL + "/repo-avatars/" + recordSource["avatar"].(string)
}
record["updated_unix"] = recordSource["updated_unix"]
record["lang"] = recordSource["lang"]
result = append(result, record)
@@ -211,6 +190,40 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string) *SearchRes {
return returnObj
}

func dealLongText(text string, Key string, MatchedQueries []string) string {
var isNeedToDealText bool
isNeedToDealText = false
if len(MatchedQueries) > 0 && Key != "" {
if MatchedQueries[0] == "f_second" || MatchedQueries[0] == "f_third" {
isNeedToDealText = true
}
}
stringlen := len(text)
if isNeedToDealText && stringlen > 200 {
index := strings.Index(text, Key)
if index > 0 {
start := index - 50
if start < 0 {
start = 0
}
end := index + 150
if end >= stringlen {
end = stringlen
}
return text[start:end]
} else {
return text[0:200]
}
} else {
if stringlen > 200 {
return text[0:200]
} else {
return text
}
}

}

func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page int, PageSize int, IsQueryUser bool) {
/*
用户或者组织 ES名称： user-es-index
@@ -229,9 +242,9 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in
ascending := ctx.QueryBool("Ascending")
boolQ := elastic.NewBoolQuery()
if Key != "" {
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first")
full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("fullname_second")
descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("desc_third")
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("f_second")
descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("f_third")
boolQ.Should(nameQuery, full_nameQuery, descriptionQuery)
}
typeValue := 1
@@ -243,13 +256,64 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in

res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
if err == nil {
ctx.JSON(200, res)
result := makeUserOrOrgResult(res, Key, ctx)
ctx.JSON(200, result)
} else {
log.Info("query es error," + err.Error())
ctx.JSON(200, "")
}
}

func makeUserOrOrgResult(sRes *elastic.SearchResult, Key string, ctx *context.Context) *SearchRes {
total := sRes.Hits.TotalHits.Value
result := make([]map[string]interface{}, 0)

for i, hit := range sRes.Hits.Hits {
log.Info("this is user query " + fmt.Sprint(i) + " result.")
recordSource := make(map[string]interface{})
source, err := hit.Source.MarshalJSON()

if err == nil {
err = json.Unmarshal(source, &recordSource)
if err == nil {
record := make(map[string]interface{})
record["name"] = recordSource["name"]
record["full_name"] = recordSource["full_name"]
desc := recordSource["description"].(string)

record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
if ctx.User != nil {
record["email"] = recordSource["email"]
} else {
record["email"] = ""
}

record["location"] = recordSource["location"]
record["website"] = recordSource["website"]
record["num_repos"] = recordSource["num_repos"]
record["num_teams"] = recordSource["num_teams"]
record["num_members"] = recordSource["num_members"]

record["avatar"] = strings.TrimRight(setting.AppSubURL, "/") + "/user/avatar/" + recordSource["name"].(string) + "/" + strconv.Itoa(-1)
record["updated_unix"] = recordSource["updated_unix"]
record["created_unix"] = recordSource["created_unix"]
result = append(result, record)
} else {
log.Info("deal source error," + err.Error())
}
} else {
log.Info("deal source error," + err.Error())
}
}

returnObj := &SearchRes{
Total: total,
Result: result,
}

return returnObj
}

func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, PageSize int) {
/*
数据集，ES名称：dataset-es-index
@@ -278,22 +342,70 @@ func searchIssue(ctx *context.Context, TableName string, Key string, Page int, P

boolQ := elastic.NewBoolQuery()
if Key != "" {
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first")
contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("content_second")
commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("comment_third")
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second")
commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third")
boolQ.Should(nameQuery, contentQuery, commentQuery)
}
isIssueQuery := elastic.NewTermQuery("is_pull", false)
boolQ.Must(isIssueQuery)
res, err := client.Search(TableName).Query(boolQ).Sort("updated_unix.keyword", false).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
if err == nil {
ctx.JSON(200, res)
result := makeIssueResult(res, Key)
ctx.JSON(200, result)
} else {
log.Info("query es error," + err.Error())
}

}

func makeIssueResult(sRes *elastic.SearchResult, Key string) *SearchRes {
total := sRes.Hits.TotalHits.Value
result := make([]map[string]interface{}, 0)

for i, hit := range sRes.Hits.Hits {
log.Info("this is issue query " + fmt.Sprint(i) + " result.")
recordSource := make(map[string]interface{})
source, err := hit.Source.MarshalJSON()

if err == nil {
err = json.Unmarshal(source, &recordSource)
if err == nil {
record := make(map[string]interface{})
record["id"] = recordSource["id"]
record["repo_id"] = recordSource["repo_id"]
repo, errRepo := models.GetRepositoryByID(recordSource["repo_id"].(int64))
if errRepo == nil {
record["repoUrl"] = repo.FullName()
}
record["name"] = recordSource["name"]
desc := recordSource["content"].(string)
record["content"] = dealLongText(desc, Key, hit.MatchedQueries)
if recordSource["pr_id"] != nil {
record["pr_id"] = recordSource["pr_id"]
}

desc = recordSource["comment"].(string)
record["comment"] = dealLongText(desc, Key, hit.MatchedQueries)
record["num_comments"] = recordSource["num_comments"]
record["updated_unix"] = recordSource["updated_unix"]
result = append(result, record)
} else {
log.Info("deal source error," + err.Error())
}
} else {
log.Info("deal source error," + err.Error())
}
}

returnObj := &SearchRes{
Total: total,
Result: result,
}

return returnObj
}

func searchPR(ctx *context.Context, TableName string, Key string, Page int, PageSize int) {

/*
@@ -308,16 +420,17 @@ func searchPR(ctx *context.Context, TableName string, Key string, Page int, Page

boolQ := elastic.NewBoolQuery()
if Key != "" {
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("name_first")
contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("content_second")
commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("comment_third")
nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second")
commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third")
boolQ.Should(nameQuery, contentQuery, commentQuery)
}
isIssueQuery := elastic.NewTermQuery("is_pull", true)
boolQ.Must(isIssueQuery)
res, err := client.Search(TableName).Query(boolQ).Sort("updated_unix.keyword", false).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
if err == nil {
ctx.JSON(200, res)
result := makeIssueResult(res, Key)
ctx.JSON(200, result)
} else {
log.Info("query es error," + err.Error())
}


Write Preview
Loading…
Cancel
Save