You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

search.go 24 kB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769
  1. package routers
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "path/filepath"
  6. "strconv"
  7. "strings"
  8. "code.gitea.io/gitea/models"
  9. "code.gitea.io/gitea/modules/context"
  10. "code.gitea.io/gitea/modules/log"
  11. "code.gitea.io/gitea/modules/setting"
  12. "github.com/olivere/elastic/v7"
  13. )
  14. type SearchRes struct {
  15. Total int64
  16. Result []map[string]interface{}
  17. PrivateTotal int64
  18. PrivatePage int
  19. PublicPage int
  20. PublicTotal int64
  21. }
  22. var client *elastic.Client
  23. func InitESClient() {
  24. ESSearchUrl := setting.ESSearchURL
  25. var err error
  26. client, err = elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(ESSearchUrl))
  27. if err != nil {
  28. log.Info("es init error.")
  29. //panic(err)
  30. }
  31. }
  32. func Search(ctx *context.Context) {
  33. keyword := strings.Trim(ctx.Query("q"), " ")
  34. ctx.Data["Keyword"] = keyword
  35. ctx.Data["SortType"] = "newest"
  36. ctx.HTML(200, "explore/search_new")
  37. }
  38. func SearchApi(ctx *context.Context) {
  39. TableName := ctx.Query("TableName")
  40. Key := ctx.Query("Key")
  41. Page := ctx.QueryInt("Page")
  42. PageSize := ctx.QueryInt("PageSize")
  43. OnlyReturnNum := ctx.QueryBool("OnlyReturnNum")
  44. OnlySearchLabel := ctx.QueryBool("OnlySearchLabel")
  45. if Page <= 0 {
  46. Page = 1
  47. }
  48. if PageSize <= 0 || PageSize > 200 {
  49. PageSize = setting.UI.IssuePagingNum
  50. }
  51. if Key != "" && !OnlyReturnNum {
  52. models.SaveSearchKeywordToDb(Key)
  53. }
  54. if TableName == "repository" {
  55. if OnlySearchLabel {
  56. searchRepoByLabel(ctx, Key, Page, PageSize)
  57. } else {
  58. searchRepo(ctx, "repository-es-index", Key, Page, PageSize, OnlyReturnNum)
  59. }
  60. return
  61. } else if TableName == "issue" {
  62. searchIssue(ctx, "issue-es-index", Key, Page, PageSize, OnlyReturnNum)
  63. return
  64. } else if TableName == "user" {
  65. searchUserOrOrg(ctx, "user-es-index", Key, Page, PageSize, true, OnlyReturnNum)
  66. return
  67. } else if TableName == "org" {
  68. searchUserOrOrg(ctx, "user-es-index", Key, Page, PageSize, false, OnlyReturnNum)
  69. return
  70. } else if TableName == "dataset" {
  71. searchDataSet(ctx, "dataset-es-index", Key, Page, PageSize, OnlyReturnNum)
  72. return
  73. } else if TableName == "pr" {
  74. searchPR(ctx, "issue-es-index", Key, Page, PageSize, OnlyReturnNum)
  75. return
  76. }
  77. }
  78. func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) {
  79. /*
  80. 项目, ES名称: repository-es-index
  81. 搜索:
  82. name character varying(255) , 项目名称
  83. description text, 项目描述
  84. topics json, 标签
  85. 排序:
  86. updated_unix
  87. num_watches,
  88. num_stars,
  89. num_forks,
  90. */
  91. SortBy := ctx.Query("SortBy")
  92. if SortBy == "" {
  93. SortBy = "updated_unix.keyword"
  94. }
  95. ascending := ctx.QueryBool("Ascending")
  96. log.Info("query searchRepoByLabel start")
  97. if Key != "" {
  98. boolQ := elastic.NewBoolQuery()
  99. topicsQuery := elastic.NewMatchQuery("topics", Key)
  100. boolQ.Should(topicsQuery)
  101. res, err := client.Search("repository-es-index").Query(boolQ).SortBy(elastic.NewScoreSort(), elastic.NewFieldSort(SortBy).Order(ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("topics")).Do(ctx.Req.Context())
  102. if err == nil {
  103. searchJson, _ := json.Marshal(res)
  104. log.Info("searchJson=" + string(searchJson))
  105. result := makeRepoResult(res, "", false)
  106. ctx.JSON(200, result)
  107. } else {
  108. log.Info("query es error," + err.Error())
  109. ctx.JSON(200, "")
  110. }
  111. } else {
  112. ctx.JSON(200, "")
  113. }
  114. }
  115. func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) {
  116. /*
  117. 项目, ES名称: repository-es-index
  118. 搜索:
  119. name character varying(255) , 项目名称
  120. description text, 项目描述
  121. topics json, 标签
  122. 排序:
  123. updated_unix
  124. num_watches,
  125. num_stars,
  126. num_forks,
  127. */
  128. SortBy := ctx.Query("SortBy")
  129. if SortBy == "" {
  130. SortBy = "updated_unix.keyword"
  131. }
  132. ascending := ctx.QueryBool("Ascending")
  133. orderBy := models.SearchOrderByRecentUpdated
  134. switch SortBy {
  135. case "updated_unix.keyword":
  136. orderBy = models.SearchOrderByRecentUpdated
  137. case "num_stars":
  138. orderBy = models.SearchOrderByStarsReverse
  139. case "num_forks":
  140. orderBy = models.SearchOrderByForksReverse
  141. case "num_watches":
  142. orderBy = models.SearchOrderByWatches
  143. }
  144. repos, count, err := models.SearchRepository(&models.SearchRepoOptions{
  145. ListOptions: models.ListOptions{
  146. Page: Page,
  147. PageSize: PageSize,
  148. },
  149. Actor: ctx.User,
  150. OrderBy: orderBy,
  151. Private: true,
  152. OnlyPrivate: true,
  153. Keyword: Key,
  154. IncludeDescription: setting.UI.SearchRepoDescription,
  155. })
  156. if err != nil {
  157. ctx.ServerError("SearchRepository", err)
  158. return
  159. }
  160. privateRe := &SearchRes{}
  161. privateRe.PrivateTotal = count
  162. privateRe.Result = make([]map[string]interface{}, 0)
  163. if count > 0 {
  164. log.Info("Query private repo number is:" + fmt.Sprint(count))
  165. makePrivateRepo(repos, privateRe, Key)
  166. }
  167. log.Info("query searchRepo start")
  168. if Key != "" {
  169. boolQ := elastic.NewBoolQuery()
  170. nameQuery := elastic.NewMatchQuery("alias", Key).Boost(1024).QueryName("f_first")
  171. descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second")
  172. topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third")
  173. boolQ.Should(nameQuery, descriptionQuery, topicsQuery)
  174. res, err := client.Search(TableName).Query(boolQ).SortBy(elastic.NewScoreSort(), elastic.NewFieldSort(SortBy).Order(ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context())
  175. if err == nil {
  176. searchJson, _ := json.Marshal(res)
  177. log.Info("searchJson=" + string(searchJson))
  178. result := makeRepoResult(res, Key, OnlyReturnNum)
  179. ctx.JSON(200, result)
  180. } else {
  181. log.Info("query es error," + err.Error())
  182. ctx.JSON(200, "")
  183. }
  184. } else {
  185. log.Info("query all content.")
  186. //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}}
  187. res, err := client.Search(TableName).SortBy(elastic.NewFieldSort(SortBy).Order(ascending)).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
  188. if err == nil {
  189. searchJson, _ := json.Marshal(res)
  190. log.Info("searchJson=" + string(searchJson))
  191. result := makeRepoResult(res, "", OnlyReturnNum)
  192. ctx.JSON(200, result)
  193. } else {
  194. log.Info("query es error," + err.Error())
  195. ctx.JSON(200, "")
  196. }
  197. }
  198. }
  199. func makePrivateRepo(repos models.RepositoryList, res *SearchRes, keyword string) {
  200. for _, repo := range repos {
  201. record := make(map[string]interface{})
  202. record["id"] = repo.ID
  203. record["name"] = makeHighLight(keyword, repo.Name)
  204. record["real_name"] = repo.Name
  205. record["owner_name"] = repo.OwnerName
  206. record["description"] = truncLongText(makeHighLight(keyword, repo.Name), true, keyword)
  207. hightTopics := make([]string, 0)
  208. if len(repo.Topics) > 0 {
  209. for _, t := range repo.Topics {
  210. hightTopics = append(hightTopics, makeHighLight(keyword, t))
  211. }
  212. }
  213. record["hightTopics"] = hightTopics
  214. record["num_watches"] = repo.NumWatches
  215. record["num_stars"] = repo.NumStars
  216. record["num_forks"] = repo.NumForks
  217. record["alias"] = repo.Alias
  218. record["lower_alias"] = repo.LowerAlias
  219. record["topics"] = repo.Topics
  220. record["avatar"] = repo.RelAvatarLink()
  221. record["updated_unix"] = repo.UpdatedUnix
  222. record["lang"] = ""
  223. res.Result = append(res.Result, record)
  224. }
  225. }
  226. func makeHighLight(keyword string, dest string) string {
  227. textRune := []rune(dest)
  228. index := chineseIndex(textRune, []rune(keyword))
  229. if index > 0 {
  230. dest = strings.ReplaceAll(dest, keyword, "\u003cfont color='red'\u003e"+keyword+"\u003c/font\u003e")
  231. }
  232. return dest
  233. }
  234. func makeRepoResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool) *SearchRes {
  235. total := sRes.Hits.TotalHits.Value
  236. result := make([]map[string]interface{}, 0)
  237. if !OnlyReturnNum {
  238. for i, hit := range sRes.Hits.Hits {
  239. log.Info("this is repo query " + fmt.Sprint(i) + " result.")
  240. recordSource := make(map[string]interface{})
  241. source, err := hit.Source.MarshalJSON()
  242. if err == nil {
  243. err = json.Unmarshal(source, &recordSource)
  244. if err == nil {
  245. record := make(map[string]interface{})
  246. record["id"] = hit.Id
  247. record["alias"] = getLabelValue("alias", recordSource, hit.Highlight)
  248. record["real_name"] = recordSource["name"]
  249. record["owner_name"] = recordSource["owner_name"]
  250. if recordSource["description"] != nil {
  251. desc := getLabelValue("description", recordSource, hit.Highlight)
  252. record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
  253. } else {
  254. record["description"] = ""
  255. }
  256. record["hightTopics"] = jsonStrToArray(getLabelValue("topics", recordSource, hit.Highlight))
  257. record["num_watches"] = recordSource["num_watches"]
  258. record["num_stars"] = recordSource["num_stars"]
  259. record["num_forks"] = recordSource["num_forks"]
  260. record["lower_alias"] = recordSource["lower_alias"]
  261. if recordSource["topics"] != nil {
  262. topicsStr := recordSource["topics"].(string)
  263. log.Info("topicsStr=" + topicsStr)
  264. if topicsStr != "null" {
  265. record["topics"] = jsonStrToArray(topicsStr)
  266. }
  267. }
  268. if recordSource["avatar"] != nil {
  269. avatarstr := recordSource["avatar"].(string)
  270. if len(avatarstr) == 0 {
  271. record["avatar"] = setting.RepositoryAvatarFallbackImage
  272. } else {
  273. record["avatar"] = filepath.Join(setting.RepositoryAvatarUploadPath, avatarstr)
  274. }
  275. }
  276. record["updated_unix"] = recordSource["updated_unix"]
  277. record["lang"] = recordSource["lang"]
  278. result = append(result, record)
  279. } else {
  280. log.Info("deal repo source error," + err.Error())
  281. }
  282. } else {
  283. log.Info("deal repo source error," + err.Error())
  284. }
  285. }
  286. }
  287. returnObj := &SearchRes{
  288. Total: total,
  289. Result: result,
  290. }
  291. return returnObj
  292. }
  293. func jsonStrToArray(str string) []string {
  294. b := []byte(str)
  295. strs := make([]string, 0)
  296. err := json.Unmarshal(b, &strs)
  297. if err != nil {
  298. log.Info("convert str arrar error, str=" + str)
  299. }
  300. return strs
  301. }
  302. func dealLongText(text string, Key string, MatchedQueries []string) string {
  303. var isNeedToDealText bool
  304. isNeedToDealText = false
  305. if len(MatchedQueries) > 0 && Key != "" {
  306. if MatchedQueries[0] == "f_second" || MatchedQueries[0] == "f_third" {
  307. isNeedToDealText = true
  308. }
  309. }
  310. return truncLongText(text, isNeedToDealText, "color=")
  311. }
  312. func truncLongText(text string, isNeedToDealText bool, keyword string) string {
  313. textRune := []rune(text)
  314. stringlen := len(textRune)
  315. if isNeedToDealText && stringlen > 200 {
  316. index := chineseIndex(textRune, []rune(keyword))
  317. if index > 0 {
  318. start := index - 50
  319. if start < 0 {
  320. start = 0
  321. }
  322. end := index + 150
  323. if end >= stringlen {
  324. end = stringlen
  325. }
  326. return string(textRune[start:end])
  327. } else {
  328. return string(textRune[0:200])
  329. }
  330. } else {
  331. if stringlen > 200 {
  332. return string(textRune[0:200])
  333. } else {
  334. return text
  335. }
  336. }
  337. }
  338. func chineseIndex(text []rune, childText []rune) int {
  339. for i, v := range text {
  340. if v == childText[0] {
  341. re := true
  342. for j, k := range childText {
  343. if k != text[i+j] {
  344. re = false
  345. break
  346. }
  347. }
  348. if re {
  349. return i
  350. }
  351. }
  352. }
  353. return -1
  354. }
  355. func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page int, PageSize int, IsQueryUser bool, OnlyReturnNum bool) {
  356. /*
  357. 用户或者组织 ES名称: user-es-index
  358. 搜索:
  359. name , 名称
  360. full_name 全名
  361. description 描述或者简介
  362. 排序:
  363. created_unix
  364. 名称字母序
  365. */
  366. SortBy := ctx.Query("SortBy")
  367. if SortBy == "" {
  368. SortBy = "created_unix.keyword"
  369. }
  370. ascending := ctx.QueryBool("Ascending")
  371. boolQ := elastic.NewBoolQuery()
  372. typeValue := 1
  373. if IsQueryUser {
  374. typeValue = 0
  375. }
  376. UserOrOrgQuery := elastic.NewTermQuery("type", typeValue)
  377. if Key != "" {
  378. boolKeyQ := elastic.NewBoolQuery()
  379. log.Info("user or org Key=" + Key)
  380. nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
  381. full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("f_second")
  382. descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("f_third")
  383. boolKeyQ.Should(nameQuery, full_nameQuery, descriptionQuery)
  384. boolQ.Must(UserOrOrgQuery, boolKeyQ)
  385. } else {
  386. boolQ.Must(UserOrOrgQuery)
  387. }
  388. res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context())
  389. if err == nil {
  390. searchJson, _ := json.Marshal(res)
  391. log.Info("searchJson=" + string(searchJson))
  392. result := makeUserOrOrgResult(res, Key, ctx, OnlyReturnNum)
  393. ctx.JSON(200, result)
  394. } else {
  395. log.Info("query es error," + err.Error())
  396. ctx.JSON(200, "")
  397. }
  398. }
  399. func getLabelValue(key string, recordSource map[string]interface{}, searchHighliht elastic.SearchHitHighlight) string {
  400. if value, ok := searchHighliht[key]; !ok {
  401. if recordSource[key] != nil {
  402. return recordSource[key].(string)
  403. } else {
  404. return ""
  405. }
  406. } else {
  407. return value[0]
  408. }
  409. }
  410. func makeUserOrOrgResult(sRes *elastic.SearchResult, Key string, ctx *context.Context, OnlyReturnNum bool) *SearchRes {
  411. total := sRes.Hits.TotalHits.Value
  412. result := make([]map[string]interface{}, 0)
  413. if !OnlyReturnNum {
  414. for i, hit := range sRes.Hits.Hits {
  415. log.Info("this is user query " + fmt.Sprint(i) + " result.")
  416. recordSource := make(map[string]interface{})
  417. source, err := hit.Source.MarshalJSON()
  418. if err == nil {
  419. err = json.Unmarshal(source, &recordSource)
  420. if err == nil {
  421. record := make(map[string]interface{})
  422. record["id"] = hit.Id
  423. record["name"] = getLabelValue("name", recordSource, hit.Highlight)
  424. record["real_name"] = recordSource["name"]
  425. record["full_name"] = getLabelValue("full_name", recordSource, hit.Highlight)
  426. if recordSource["description"] != nil {
  427. desc := getLabelValue("description", recordSource, hit.Highlight)
  428. record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
  429. } else {
  430. record["description"] = ""
  431. }
  432. if ctx.User != nil {
  433. record["email"] = recordSource["email"]
  434. } else {
  435. record["email"] = ""
  436. }
  437. record["location"] = recordSource["location"]
  438. record["website"] = recordSource["website"]
  439. record["num_repos"] = recordSource["num_repos"]
  440. record["num_teams"] = recordSource["num_teams"]
  441. record["num_members"] = recordSource["num_members"]
  442. record["avatar"] = strings.TrimRight(setting.AppSubURL, "/") + "/user/avatar/" + recordSource["name"].(string) + "/" + strconv.Itoa(-1)
  443. record["updated_unix"] = recordSource["updated_unix"]
  444. record["created_unix"] = recordSource["created_unix"]
  445. result = append(result, record)
  446. } else {
  447. log.Info("deal user source error," + err.Error())
  448. }
  449. } else {
  450. log.Info("deal user source error," + err.Error())
  451. }
  452. }
  453. }
  454. returnObj := &SearchRes{
  455. Total: total,
  456. Result: result,
  457. }
  458. return returnObj
  459. }
  460. func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) {
  461. /*
  462. 数据集,ES名称:dataset-es-index
  463. 搜索:
  464. title , 名称
  465. description 描述
  466. category 标签
  467. file_name 数据集文件名称
  468. 排序:
  469. download_times
  470. */
  471. SortBy := ctx.Query("SortBy")
  472. if SortBy == "" {
  473. SortBy = "download_times.keyword"
  474. }
  475. ascending := ctx.QueryBool("Ascending")
  476. log.Info("query searchRepo start")
  477. boolQ := elastic.NewBoolQuery()
  478. if Key != "" {
  479. nameQuery := elastic.NewMatchQuery("title", Key).Boost(2).QueryName("f_first")
  480. descQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second")
  481. fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third")
  482. categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth")
  483. boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery)
  484. res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context())
  485. if err == nil {
  486. searchJson, _ := json.Marshal(res)
  487. log.Info("searchJson=" + string(searchJson))
  488. result := makeDatasetResult(res, Key, OnlyReturnNum)
  489. ctx.JSON(200, result)
  490. } else {
  491. log.Info("query es error," + err.Error())
  492. }
  493. } else {
  494. log.Info("query all content.")
  495. //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}}
  496. res, err := client.Search(TableName).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
  497. if err == nil {
  498. searchJson, _ := json.Marshal(res)
  499. log.Info("searchJson=" + string(searchJson))
  500. result := makeDatasetResult(res, "", OnlyReturnNum)
  501. ctx.JSON(200, result)
  502. } else {
  503. log.Info("query es error," + err.Error())
  504. ctx.JSON(200, "")
  505. }
  506. }
  507. }
  508. func makeDatasetResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool) *SearchRes {
  509. total := sRes.Hits.TotalHits.Value
  510. result := make([]map[string]interface{}, 0)
  511. if !OnlyReturnNum {
  512. for i, hit := range sRes.Hits.Hits {
  513. log.Info("this is dataset query " + fmt.Sprint(i) + " result.")
  514. recordSource := make(map[string]interface{})
  515. source, err := hit.Source.MarshalJSON()
  516. if err == nil {
  517. err = json.Unmarshal(source, &recordSource)
  518. if err == nil {
  519. record := make(map[string]interface{})
  520. record["id"] = hit.Id
  521. userIdStr := recordSource["user_id"].(string)
  522. userId, cerr := strconv.ParseInt(userIdStr, 10, 64)
  523. if cerr == nil {
  524. user, errUser := models.GetUserByID(userId)
  525. if errUser == nil {
  526. record["owerName"] = user.GetDisplayName()
  527. record["avatar"] = user.RelAvatarLink()
  528. }
  529. }
  530. setRepoInfo(recordSource, record)
  531. record["title"] = getLabelValue("title", recordSource, hit.Highlight)
  532. record["category"] = getLabelValue("category", recordSource, hit.Highlight)
  533. if recordSource["description"] != nil {
  534. desc := getLabelValue("description", recordSource, hit.Highlight)
  535. record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
  536. } else {
  537. record["description"] = ""
  538. }
  539. record["file_name"] = getLabelValue("file_name", recordSource, hit.Highlight)
  540. record["task"] = recordSource["task"]
  541. record["download_times"] = recordSource["download_times"]
  542. record["created_unix"] = recordSource["created_unix"]
  543. result = append(result, record)
  544. } else {
  545. log.Info("deal dataset source error," + err.Error())
  546. }
  547. } else {
  548. log.Info("deal dataset source error," + err.Error())
  549. }
  550. }
  551. }
  552. returnObj := &SearchRes{
  553. Total: total,
  554. Result: result,
  555. }
  556. return returnObj
  557. }
  558. func searchIssue(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) {
  559. /*
  560. 任务,合并请求 ES名称:issue-es-index
  561. 搜索:
  562. name character varying(255) , 标题
  563. content text, 内容
  564. comment text, 评论
  565. 排序:
  566. updated_unix
  567. */
  568. SortBy := ctx.Query("SortBy")
  569. if SortBy == "" {
  570. SortBy = "updated_unix.keyword"
  571. }
  572. ascending := ctx.QueryBool("Ascending")
  573. boolQ := elastic.NewBoolQuery()
  574. isIssueQuery := elastic.NewTermQuery("is_pull", "f")
  575. if Key != "" {
  576. boolKeyQ := elastic.NewBoolQuery()
  577. log.Info("issue Key=" + Key)
  578. nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
  579. contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second")
  580. commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third")
  581. boolKeyQ.Should(nameQuery, contentQuery, commentQuery)
  582. boolQ.Must(isIssueQuery, boolKeyQ)
  583. } else {
  584. boolQ.Must(isIssueQuery)
  585. }
  586. res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context())
  587. if err == nil {
  588. searchJson, _ := json.Marshal(res)
  589. log.Info("searchJson=" + string(searchJson))
  590. result := makeIssueResult(res, Key, OnlyReturnNum)
  591. ctx.JSON(200, result)
  592. } else {
  593. log.Info("query es error," + err.Error())
  594. }
  595. }
  596. func queryHighlight(names ...string) *elastic.Highlight {
  597. re := elastic.NewHighlight()
  598. for i := 0; i < len(names); i++ {
  599. field := &elastic.HighlighterField{
  600. Name: names[i],
  601. }
  602. re.Fields(field)
  603. }
  604. re.PreTags("<font color='red'>")
  605. re.PostTags("</font>")
  606. return re
  607. }
  608. func setRepoInfo(recordSource map[string]interface{}, record map[string]interface{}) {
  609. repoIdstr := recordSource["repo_id"].(string)
  610. repoId, cerr := strconv.ParseInt(repoIdstr, 10, 64)
  611. if cerr == nil {
  612. repo, errRepo := models.GetRepositoryByID(repoId)
  613. if errRepo == nil {
  614. log.Info("repo_url=" + repo.FullName())
  615. record["repoUrl"] = repo.FullName()
  616. record["avatar"] = repo.RelAvatarLink()
  617. } else {
  618. log.Info("repo err=" + errRepo.Error())
  619. }
  620. } else {
  621. log.Info("parse int err=" + cerr.Error())
  622. }
  623. }
  624. func makeIssueResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool) *SearchRes {
  625. total := sRes.Hits.TotalHits.Value
  626. result := make([]map[string]interface{}, 0)
  627. if !OnlyReturnNum {
  628. for i, hit := range sRes.Hits.Hits {
  629. log.Info("this is issue query " + fmt.Sprint(i) + " result.")
  630. recordSource := make(map[string]interface{})
  631. source, err := hit.Source.MarshalJSON()
  632. if err == nil {
  633. err = json.Unmarshal(source, &recordSource)
  634. if err == nil {
  635. record := make(map[string]interface{})
  636. record["id"] = hit.Id
  637. record["repo_id"] = recordSource["repo_id"]
  638. log.Info("recordSource[\"repo_id\"]=" + fmt.Sprint(recordSource["repo_id"]))
  639. setRepoInfo(recordSource, record)
  640. record["name"] = getLabelValue("name", recordSource, hit.Highlight)
  641. if recordSource["content"] != nil {
  642. desc := getLabelValue("content", recordSource, hit.Highlight)
  643. record["content"] = dealLongText(desc, Key, hit.MatchedQueries)
  644. if _, ok := hit.Highlight["content"]; !ok {
  645. if _, ok_comment := hit.Highlight["comment"]; ok_comment {
  646. desc := getLabelValue("comment", recordSource, hit.Highlight)
  647. record["content"] = dealLongText(desc, Key, hit.MatchedQueries)
  648. }
  649. }
  650. } else {
  651. if recordSource["comment"] != nil {
  652. desc := getLabelValue("comment", recordSource, hit.Highlight)
  653. record["content"] = dealLongText(desc, Key, hit.MatchedQueries)
  654. }
  655. }
  656. if recordSource["pr_id"] != nil {
  657. record["pr_id"] = recordSource["pr_id"]
  658. }
  659. log.Info("index=" + recordSource["index"].(string))
  660. record["index"] = recordSource["index"]
  661. record["num_comments"] = recordSource["num_comments"]
  662. record["is_closed"] = recordSource["is_closed"]
  663. record["updated_unix"] = recordSource["updated_unix"]
  664. result = append(result, record)
  665. } else {
  666. log.Info("deal issue source error," + err.Error())
  667. }
  668. } else {
  669. log.Info("deal issue source error," + err.Error())
  670. }
  671. }
  672. }
  673. returnObj := &SearchRes{
  674. Total: total,
  675. Result: result,
  676. }
  677. return returnObj
  678. }
  679. func searchPR(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) {
  680. /*
  681. 任务,合并请求 ES名称:issue-es-index
  682. 搜索:
  683. name character varying(255) , 标题
  684. content text, 内容
  685. comment text, 评论
  686. 排序:
  687. updated_unix
  688. */
  689. SortBy := ctx.Query("SortBy")
  690. if SortBy == "" {
  691. SortBy = "updated_unix.keyword"
  692. }
  693. ascending := ctx.QueryBool("Ascending")
  694. boolQ := elastic.NewBoolQuery()
  695. isPRQuery := elastic.NewTermQuery("is_pull", "t")
  696. if Key != "" {
  697. boolKeyQ := elastic.NewBoolQuery()
  698. log.Info("issue Key=" + Key)
  699. nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
  700. contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second")
  701. commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third")
  702. boolKeyQ.Should(nameQuery, contentQuery, commentQuery)
  703. boolQ.Must(isPRQuery, boolKeyQ)
  704. } else {
  705. boolQ.Must(isPRQuery)
  706. }
  707. res, err := client.Search(TableName).Query(boolQ).Sort(SortBy, ascending).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context())
  708. if err == nil {
  709. searchJson, _ := json.Marshal(res)
  710. log.Info("searchJson=" + string(searchJson))
  711. result := makeIssueResult(res, Key, OnlyReturnNum)
  712. ctx.JSON(200, result)
  713. } else {
  714. log.Info("query es error," + err.Error())
  715. }
  716. }