aiforge/routers/search.go

package routers

import (
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/timeutil"
	"github.com/olivere/elastic/v7"
)

type SearchRes struct {
	Total        int64
	Result       []map[string]interface{}
	PrivateTotal int64
}

var client *elastic.Client

func InitESClient() {
	ESSearchUrl := setting.ESSearchURL
	var err error
	client, err = elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(ESSearchUrl))
	if err != nil {
		log.Info("es init error.")
		//panic(err)
	}
}

func EmptySearch(ctx *context.Context) {
	log.Info("search template.")
	ctx.Data["Keyword"] = ""
	ctx.HTML(200, "explore/search_new")
}

func Search(ctx *context.Context) {
	log.Info("search template.")
	keyword := strings.Trim(ctx.Query("q"), " ")
	ctx.Data["Keyword"] = keyword
	ctx.Data["SortType"] = "newest"
	ctx.HTML(200, "explore/search_new")
}

func SearchApi(ctx *context.Context) {
	TableName := ctx.Query("TableName")
	Key := ctx.Query("Key")
	Page := ctx.QueryInt("Page")
	PageSize := ctx.QueryInt("PageSize")
	OnlyReturnNum := ctx.QueryBool("OnlyReturnNum")
	OnlySearchLabel := ctx.QueryBool("OnlySearchLabel")

	if Page <= 0 {
		Page = 1
	}
	if PageSize <= 0 || PageSize > 200 {
		PageSize = setting.UI.IssuePagingNum
	}
	if Key != "" && !OnlyReturnNum {
		go models.SaveSearchKeywordToDb(Key)
	}
	if TableName == "repository" {
		if OnlySearchLabel {
			searchRepoByLabel(ctx, Key, Page, PageSize)
		} else {
			searchRepo(ctx, "repository-es-index"+setting.INDEXPOSTFIX, Key, Page, PageSize, OnlyReturnNum)
		}
		return
	} else if TableName == "issue" {
		searchIssueOrPr(ctx, "issue-es-index"+setting.INDEXPOSTFIX, Key, Page, PageSize, OnlyReturnNum, "f")
		return
	} else if TableName == "user" {
		searchUserOrOrg(ctx, "user-es-index"+setting.INDEXPOSTFIX, Key, Page, PageSize, true, OnlyReturnNum)
		return
	} else if TableName == "org" {
		searchUserOrOrg(ctx, "user-es-index"+setting.INDEXPOSTFIX, Key, Page, PageSize, false, OnlyReturnNum)
		return
	} else if TableName == "dataset" {
		searchDataSet(ctx, "dataset-es-index"+setting.INDEXPOSTFIX, Key, Page, PageSize, OnlyReturnNum)
		return
	} else if TableName == "pr" {
		searchIssueOrPr(ctx, "issue-es-index"+setting.INDEXPOSTFIX, Key, Page, PageSize, OnlyReturnNum, "t")
		//searchPR(ctx, "issue-es-index", Key, Page, PageSize, OnlyReturnNum)
		return
	}
}

func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) {
	/*
			   项目， ES名称： repository-es-index
			   搜索:
			   name character varying(255) ,  项目名称
			   description text,  项目描述
			   topics json,  标签
			   排序:
			   updated_unix
			   num_watches,
		       num_stars,
		       num_forks,
	*/
	SortBy := ctx.Query("SortBy")
	PrivateTotal := ctx.QueryInt("PrivateTotal")
	WebTotal := ctx.QueryInt("WebTotal")
	ascending := ctx.QueryBool("Ascending")
	language := ctx.Query("language")
	if language == "" {
		language = "zh-CN"
	}
	from := (Page - 1) * PageSize
	resultObj := &SearchRes{}
	log.Info("WebTotal=" + fmt.Sprint(WebTotal))
	log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal))
	resultObj.Result = make([]map[string]interface{}, 0)
	if from == 0 {
		WebTotal = 0
	}
	if ctx.User != nil && (from < PrivateTotal || from == 0) {
		orderBy := models.SearchOrderByRecentUpdated
		switch SortBy {
		case "updated_unix.keyword":
			orderBy = models.SearchOrderByRecentUpdated
		case "num_stars":
			orderBy = models.SearchOrderByStarsReverse
		case "num_forks":
			orderBy = models.SearchOrderByForksReverse
		case "num_watches":
			orderBy = models.SearchOrderByWatches
		}
		log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil))
		repos, count, err := models.SearchRepository(&models.SearchRepoOptions{
			ListOptions: models.ListOptions{
				Page:     Page,
				PageSize: PageSize,
			},
			Actor:              ctx.User,
			OrderBy:            orderBy,
			Private:            true,
			OnlyPrivate:        true,
			TopicOnly:          true,
			TopicName:          Key,
			IncludeDescription: setting.UI.SearchRepoDescription,
		})
		if err != nil {
			ctx.JSON(200, "")
			return
		}
		resultObj.PrivateTotal = count
		if repos.Len() > 0 {
			log.Info("Query private repo number is:" + fmt.Sprint(repos.Len()))
			makePrivateRepo(repos, resultObj, Key, language)
		} else {
			log.Info("not found private repo,keyword=" + Key)
		}
		if repos.Len() >= PageSize {
			if WebTotal > 0 {
				resultObj.Total = int64(WebTotal)
				ctx.JSON(200, resultObj)
				return
			}
		}
	} else {
		if ctx.User == nil {
			resultObj.PrivateTotal = 0
		} else {
			resultObj.PrivateTotal = int64(PrivateTotal)
		}
	}

	from = from - PrivateTotal
	if from < 0 {
		from = 0
	}
	Size := PageSize - len(resultObj.Result)

	log.Info("query searchRepoByLabel start")
	if Key != "" {
		boolQ := elastic.NewBoolQuery()
		topicsQuery := elastic.NewMatchQuery("topics", Key)
		boolQ.Should(topicsQuery)

		res, err := client.Search("repository-es-index").Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("topics")).Do(ctx.Req.Context())
		if err == nil {
			searchJson, _ := json.Marshal(res)
			log.Info("searchJson=" + string(searchJson))
			esresult := makeRepoResult(res, "", false, language)
			resultObj.Total = resultObj.PrivateTotal + esresult.Total
			resultObj.Result = append(resultObj.Result, esresult.Result...)
			ctx.JSON(200, resultObj)
		} else {
			log.Info("query es error," + err.Error())
			ctx.JSON(200, "")
		}
	} else {
		ctx.JSON(200, "")
	}
}

func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter {
	sort := make([]elastic.Sorter, 0)
	if SortBy == "default" || SortBy == "" {
		sort = append(sort, elastic.NewScoreSort())
		if secondSortBy != "" {
			log.Info("SortBy=" + SortBy + " secondSortBy=" + secondSortBy)
			sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending))
		}
	} else {
		sort = append(sort, elastic.NewFieldSort(SortBy).Order(ascending))
	}
	log.Info("sort size=" + fmt.Sprint(len(sort)))
	return sort
}

func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) {
	/*
			   项目， ES名称： repository-es-index
			   搜索:
			   name character varying(255) ,  项目名称
			   description text,  项目描述
			   topics json,  标签
			   排序:
			   updated_unix
			   num_watches,
		       num_stars,
		       num_forks,
	*/

	SortBy := ctx.Query("SortBy")
	PrivateTotal := ctx.QueryInt("PrivateTotal")
	WebTotal := ctx.QueryInt("WebTotal")
	ascending := ctx.QueryBool("Ascending")
	from := (Page - 1) * PageSize
	resultObj := &SearchRes{}
	log.Info("WebTotal=" + fmt.Sprint(WebTotal))
	log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal))
	resultObj.Result = make([]map[string]interface{}, 0)
	if from == 0 {
		WebTotal = 0
	}
	language := ctx.Query("language")
	if language == "" {
		language = "zh-CN"
	}
	if ctx.User != nil && (from < PrivateTotal || from == 0) {
		orderBy := models.SearchOrderByRecentUpdated
		switch SortBy {
		case "updated_unix.keyword":
			orderBy = models.SearchOrderByRecentUpdated
		case "num_stars":
			orderBy = models.SearchOrderByStarsReverse
		case "num_forks":
			orderBy = models.SearchOrderByForksReverse
		case "num_watches":
			orderBy = models.SearchOrderByWatches
		}
		log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil))
		repos, count, err := models.SearchRepository(&models.SearchRepoOptions{
			ListOptions: models.ListOptions{
				Page:     Page,
				PageSize: PageSize,
			},
			Actor:              ctx.User,
			OrderBy:            orderBy,
			Private:            true,
			OnlyPrivate:        true,
			Keyword:            Key,
			IncludeDescription: setting.UI.SearchRepoDescription,
			OnlySearchPrivate:  true,
		})
		if err != nil {
			ctx.JSON(200, "")
			return
		}
		resultObj.PrivateTotal = count
		if repos.Len() > 0 {
			log.Info("Query private repo number is:" + fmt.Sprint(repos.Len()))
			makePrivateRepo(repos, resultObj, Key, language)
		} else {
			log.Info("not found private repo,keyword=" + Key)
		}
		if repos.Len() >= PageSize {
			if WebTotal > 0 {
				resultObj.Total = int64(WebTotal)
				ctx.JSON(200, resultObj)
				return
			}
		}
	} else {
		if ctx.User == nil {
			resultObj.PrivateTotal = 0
		} else {
			resultObj.PrivateTotal = int64(PrivateTotal)
		}
	}

	from = from - PrivateTotal
	if from < 0 {
		from = 0
	}
	Size := PageSize - len(resultObj.Result)

	log.Info("query searchRepo start")
	if Key != "" {
		boolQ := elastic.NewBoolQuery()
		nameQuery := elastic.NewMatchQuery("alias", Key).Boost(1024).QueryName("f_first")
		descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second")
		topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third")
		boolQ.Should(nameQuery, descriptionQuery, topicsQuery)

		res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "num_stars", false)...).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context())
		if err == nil {
			esresult := makeRepoResult(res, Key, OnlyReturnNum, language)
			setForkRepoOrder(esresult, SortBy)
			resultObj.Total = resultObj.PrivateTotal + esresult.Total
			isNeedSort := false
			if len(resultObj.Result) > 0 {
				isNeedSort = true
			}
			resultObj.Result = append(resultObj.Result, esresult.Result...)
			if isNeedSort {
				sortRepo(resultObj.Result, SortBy, ascending)
			}
			ctx.JSON(200, resultObj)
		} else {
			log.Info("query es error," + err.Error())
			ctx.JSON(200, "")
		}
	} else {
		log.Info("query all content.")
		//搜索的属性要指定{"timestamp":{"unmapped_type":"date"}}
		res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context())
		if err == nil {
			searchJson, _ := json.Marshal(res)
			log.Info("searchJson=" + string(searchJson))
			esresult := makeRepoResult(res, "", OnlyReturnNum, language)
			resultObj.Total = resultObj.PrivateTotal + esresult.Total
			resultObj.Result = append(resultObj.Result, esresult.Result...)
			ctx.JSON(200, resultObj)
		} else {
			log.Info("query es error," + err.Error())
			ctx.JSON(200, "")
		}
	}
}

func setForkRepoOrder(esresult *SearchRes, SortBy string) {
	if SortBy == "default" || SortBy == "" {
		return
	}
	forkidMap := make(map[string]int, 0)
	for index, re := range esresult.Result {
		if re["fork_id"] != nil {
			fork_id := re["fork_id"].(string)
			if _, ok := forkidMap[fork_id]; !ok {
				forkidMap[fork_id] = index
			}
		}
	}
	for key, value := range forkidMap {
		for index, re := range esresult.Result {
			if re["id"].(string) == key {
				if value < index { //swap
					tmp := esresult.Result[index]
					esresult.Result[index] = esresult.Result[value]
					esresult.Result[value] = tmp
					break
				}
			}
		}
	}
}

func sortRepo(Result []map[string]interface{}, SortBy string, ascending bool) {
	orderBy := ""
	switch SortBy {
	case "updated_unix.keyword":
		orderBy = "updated_unix"
	case "num_stars":
		orderBy = "num_stars"
	case "num_forks":
		orderBy = "num_forks"
	case "num_watches":
		orderBy = "num_watches"
	}
	sort.Slice(Result, func(i, j int) bool {
		return getInt(Result[i][orderBy], orderBy) > getInt(Result[j][orderBy], orderBy)
	})
}

func getInt(tmp interface{}, orderBy string) int64 {
	timeInt, err := strconv.ParseInt(fmt.Sprint(tmp), 10, 64)
	if err == nil {
		return timeInt
	} else {
		log.Info("convert " + orderBy + " error type=" + fmt.Sprint(tmp))
	}
	return -1
}

func makePrivateRepo(repos models.RepositoryList, res *SearchRes, keyword string, language string) {

	for _, repo := range repos {
		record := make(map[string]interface{})
		record["id"] = repo.ID
		record["name"] = makeHighLight(keyword, repo.Name)
		record["real_name"] = repo.Name
		record["owner_name"] = repo.OwnerName
		record["description"] = truncLongText(makeHighLight(keyword, repo.Description), true)

		hightTopics := make([]string, 0)
		if len(repo.Topics) > 0 {
			for _, t := range repo.Topics {
				hightTopics = append(hightTopics, makeHighLight(keyword, t))
			}
		}
		record["hightTopics"] = hightTopics

		record["num_watches"] = repo.NumWatches
		record["num_stars"] = repo.NumStars
		record["num_forks"] = repo.NumForks
		record["alias"] = truncLongText(makeHighLight(keyword, repo.Alias), true)
		record["lower_alias"] = repo.LowerAlias
		record["topics"] = repo.Topics
		record["avatar"] = repo.RelAvatarLink()
		if len(repo.RelAvatarLink()) == 0 {
			record["avatar"] = setting.RepositoryAvatarFallbackImage
		}
		record["updated_unix"] = repo.UpdatedUnix
		record["updated_html"] = timeutil.TimeSinceUnix(repo.UpdatedUnix, language)
		lang, err := repo.GetTopLanguageStats(1)
		if err == nil && len(lang) > 0 {
			record["lang"] = lang[0].Language
		} else {
			record["lang"] = ""
		}
		record["is_private"] = true
		res.Result = append(res.Result, record)
	}
}

func makeHighLight(keyword string, dest string) string {

	dest = replaceIngoreUpperOrLower(dest, strings.ToLower(dest), strings.ToLower(keyword))

	return dest
}

func replaceIngoreUpperOrLower(dest string, destLower string, keywordLower string) string {
	re := ""
	last := 0
	lenDestLower := len(destLower)
	lenkeywordLower := len(keywordLower)
	for i := 0; i < lenDestLower; i++ {
		if destLower[i] == keywordLower[0] {
			isFind := true
			for j := 1; j < lenkeywordLower; j++ {
				if (i+j) < lenDestLower && keywordLower[j] != destLower[i+j] {
					isFind = false
					break
				}
			}
			if isFind && (i+lenkeywordLower) <= lenDestLower {
				re += dest[last:i] + "\u003cfont color='red'\u003e" + dest[i:(i+lenkeywordLower)] + "\u003c/font\u003e"
				i = i + lenkeywordLower
				last = i
			}
		}
	}
	if last < lenDestLower {
		re += dest[last:lenDestLower]
	}
	return re
}

func makeRepoResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, language string) *SearchRes {
	total := sRes.Hits.TotalHits.Value
	result := make([]map[string]interface{}, 0)
	if !OnlyReturnNum {
		for i, hit := range sRes.Hits.Hits {
			log.Info("this is repo query " + fmt.Sprint(i) + " result.")
			recordSource := make(map[string]interface{})
			source, err := hit.Source.MarshalJSON()

			if err == nil {
				err = json.Unmarshal(source, &recordSource)
				if err == nil {
					record := make(map[string]interface{})
					record["id"] = hit.Id
					record["alias"] = getLabelValue("alias", recordSource, hit.Highlight)
					record["real_name"] = recordSource["name"]
					record["owner_name"] = recordSource["owner_name"]
					if recordSource["description"] != nil {
						desc := getLabelValue("description", recordSource, hit.Highlight)
						record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
					} else {
						record["description"] = ""
					}

					record["hightTopics"] = jsonStrToArray(getLabelValue("topics", recordSource, hit.Highlight))
					record["num_watches"] = recordSource["num_watches"]
					record["num_stars"] = recordSource["num_stars"]
					record["num_forks"] = recordSource["num_forks"]
					record["lower_alias"] = recordSource["lower_alias"]
					record["fork_id"] = recordSource["fork_id"]
					if recordSource["topics"] != nil {
						topicsStr := recordSource["topics"].(string)
						log.Info("topicsStr=" + topicsStr)
						if topicsStr != "null" {
							record["topics"] = jsonStrToArray(topicsStr)
						}
					}
					if recordSource["avatar"] != nil {
						avatarstr := recordSource["avatar"].(string)
						if len(avatarstr) == 0 {
							// record["avatar"] = setting.RepositoryAvatarFallbackImage
						} else {
							record["avatar"] = setting.AppSubURL + "/repo-avatars/" + avatarstr
						}
					}
					record["updated_unix"] = recordSource["updated_unix"]
					setUpdateHtml(record, recordSource["updated_unix"].(string), language)

					record["lang"] = recordSource["lang"]
					record["is_private"] = false
					result = append(result, record)
				} else {
					log.Info("deal repo source error," + err.Error())
				}
			} else {
				log.Info("deal repo source error," + err.Error())
			}
		}
	}
	returnObj := &SearchRes{
		Total:  total,
		Result: result,
	}

	return returnObj
}

func setUpdateHtml(record map[string]interface{}, updated_unix string, language string) {
	timeInt, err := strconv.ParseInt(updated_unix, 10, 64)
	if err == nil {
		record["updated_html"] = timeutil.TimeSinceUnix(timeutil.TimeStamp(timeInt), language)
	}
}

func jsonStrToArray(str string) []string {
	b := []byte(str)
	strs := make([]string, 0)
	err := json.Unmarshal(b, &strs)
	if err != nil {
		log.Info("convert str arrar error, str=" + str)
	}
	return strs
}

func dealLongText(text string, Key string, MatchedQueries []string) string {
	var isNeedToDealText bool
	isNeedToDealText = false
	if len(MatchedQueries) > 0 && Key != "" {
		if MatchedQueries[0] == "f_second" || MatchedQueries[0] == "f_third" {
			isNeedToDealText = true
		}
	}
	return truncLongText(text, isNeedToDealText)
}

func truncLongText(text string, isNeedToDealText bool) string {
	startStr := "color="
	textRune := []rune(text)
	stringlen := len(textRune)
	if isNeedToDealText && stringlen > 200 {
		index := findFont(textRune, []rune(startStr))
		if index > 0 {
			start := index - 50
			if start < 0 {
				start = 0
			}
			end := index + 150
			if end >= stringlen {
				end = stringlen
			}
			return trimFontHtml(textRune[start:end]) + "..."
		} else {
			return trimFontHtml(textRune[0:200]) + "..."
		}
	} else {
		if stringlen > 200 {
			return trimFontHtml(textRune[0:200]) + "..."
		} else {
			return text
		}
	}
}

func trimFontHtml(text []rune) string {
	startRune := rune('<')
	endRune := rune('>')
	count := 0
	i := 0
	for ; i < len(text); i++ {
		if text[i] == startRune { //start <
			re := false
			j := i + 1
			for ; j < len(text); j++ {
				if text[j] == endRune {
					re = true
					break
				}
			}
			if re { //found >
				i = j
				count++
			} else {
				if count%2 == 1 {
					return string(text[0:i]) + "</font>"
				} else {
					return string(text[0:i])
				}
			}
		}
	}
	if count%2 == 1 {
		return string(text[0:i]) + "</font>"
	} else {
		return string(text[0:i])
	}
}

func trimHrefHtml(result string) string {
	result = strings.Replace(result, "</a>", "", -1)
	result = strings.Replace(result, "\n", "", -1)
	var index int
	for {
		index = findSubstr(result, 0, "<a")
		if index != -1 {
			sIndex := findSubstr(result, index+2, ">")
			if sIndex != -1 {
				result = result[0:index] + result[sIndex+1:]
			} else {
				result = result[0:index] + result[index+2:]
			}
		} else {
			break
		}
	}
	return result
}

func findFont(text []rune, childText []rune) int {
	for i := 0; i < len(text); i++ {
		if text[i] == childText[0] {
			re := true
			for j, k := range childText {
				if k != text[i+j] {
					re = false
					break
				}
			}
			if re {
				return i
			}
		}
	}
	return -1
}

func findSubstr(text string, startindex int, childText string) int {
	for i := startindex; i < len(text); i++ {
		if text[i] == childText[0] {
			re := true
			for k := range childText {
				if childText[k] != text[i+k] {
					re = false
					break
				}
			}
			if re {
				return i
			}
		}
	}
	return -1
}

func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page int, PageSize int, IsQueryUser bool, OnlyReturnNum bool) {
	/*
	   用户或者组织 ES名称： user-es-index
	   搜索:
	   name  ,  名称
	   full_name   全名
	   description  描述或者简介
	   排序:
	   created_unix
	   名称字母序
	*/
	SortBy := ctx.Query("SortBy")
	ascending := ctx.QueryBool("Ascending")
	boolQ := elastic.NewBoolQuery()

	typeValue := 1
	if IsQueryUser {
		typeValue = 0
	}
	UserOrOrgQuery := elastic.NewTermQuery("type", typeValue)
	if Key != "" {
		boolKeyQ := elastic.NewBoolQuery()
		log.Info("user or org Key=" + Key)
		nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
		full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("f_second")
		descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("f_third")
		boolKeyQ.Should(nameQuery, full_nameQuery, descriptionQuery)
		boolQ.Must(UserOrOrgQuery, boolKeyQ)
	} else {
		boolQ.Must(UserOrOrgQuery)
	}

	res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context())
	if err == nil {
		searchJson, _ := json.Marshal(res)
		log.Info("searchJson=" + string(searchJson))
		result := makeUserOrOrgResult(res, Key, ctx, OnlyReturnNum)
		ctx.JSON(200, result)
	} else {
		log.Info("query es error," + err.Error())
		ctx.JSON(200, "")
	}
}

func getLabelValue(key string, recordSource map[string]interface{}, searchHighliht elastic.SearchHitHighlight) string {
	if value, ok := searchHighliht[key]; !ok {
		if recordSource[key] != nil {
			return recordSource[key].(string)
		} else {
			return ""
		}
	} else {
		return value[0]
	}
}

func makeUserOrOrgResult(sRes *elastic.SearchResult, Key string, ctx *context.Context, OnlyReturnNum bool) *SearchRes {
	total := sRes.Hits.TotalHits.Value
	result := make([]map[string]interface{}, 0)
	if !OnlyReturnNum {
		for i, hit := range sRes.Hits.Hits {
			log.Info("this is user query " + fmt.Sprint(i) + " result.")
			recordSource := make(map[string]interface{})
			source, err := hit.Source.MarshalJSON()

			if err == nil {
				err = json.Unmarshal(source, &recordSource)
				if err == nil {
					record := make(map[string]interface{})
					record["id"] = hit.Id
					record["name"] = getLabelValue("name", recordSource, hit.Highlight)
					record["real_name"] = recordSource["name"]
					record["full_name"] = getLabelValue("full_name", recordSource, hit.Highlight)
					if recordSource["description"] != nil {
						desc := getLabelValue("description", recordSource, hit.Highlight)
						record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
					} else {
						record["description"] = ""
					}
					if ctx.User != nil {
						record["email"] = recordSource["email"]
					} else {
						record["email"] = ""
					}

					record["location"] = recordSource["location"]
					record["website"] = recordSource["website"]
					record["num_repos"] = recordSource["num_repos"]
					record["num_teams"] = recordSource["num_teams"]
					record["num_members"] = recordSource["num_members"]

					record["avatar"] = strings.TrimRight(setting.AppSubURL, "/") + "/user/avatar/" + recordSource["name"].(string) + "/" + strconv.Itoa(-1)
					record["updated_unix"] = recordSource["updated_unix"]
					record["created_unix"] = recordSource["created_unix"]
					record["add_time"] = getAddTime(recordSource["created_unix"].(string))
					result = append(result, record)
				} else {
					log.Info("deal user source error," + err.Error())
				}
			} else {
				log.Info("deal user source error," + err.Error())
			}
		}
	}
	returnObj := &SearchRes{
		Total:  total,
		Result: result,
	}
	return returnObj
}

func getAddTime(time string) string {
	timeInt, err := strconv.ParseInt(time, 10, 64)
	if err == nil {
		t := timeutil.TimeStamp(timeInt)
		return t.FormatShort()
	}
	return ""
}

func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) {
	/*
	   数据集，ES名称：dataset-es-index
	   搜索:
	   title  ,  名称
	   description   描述
	   category  标签
	   file_name  数据集文件名称
	   排序:
	   download_times

	*/
	log.Info("query searchdataset start")
	SortBy := ctx.Query("SortBy")
	ascending := ctx.QueryBool("Ascending")
	PrivateTotal := ctx.QueryInt("PrivateTotal")
	WebTotal := ctx.QueryInt("WebTotal")
	language := ctx.Query("language")
	if language == "" {
		language = "zh-CN"
	}
	from := (Page - 1) * PageSize
	if from == 0 {
		WebTotal = 0
	}
	resultObj := &SearchRes{}
	log.Info("WebTotal=" + fmt.Sprint(WebTotal))
	log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal))
	resultObj.Result = make([]map[string]interface{}, 0)

	if ctx.User != nil && (from < PrivateTotal || from == 0) {

		log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil))
		datasets, count, err := models.SearchDatasetBySQL(Page, PageSize, Key, ctx.User.ID)
		if err != nil {
			ctx.JSON(200, "")
			return
		}
		resultObj.PrivateTotal = count
		datasetSize := len(datasets)
		if datasetSize > 0 {
			log.Info("Query private dataset number is:" + fmt.Sprint(datasetSize) + " count=" + fmt.Sprint(count))
			makePrivateDataSet(datasets, resultObj, Key, language)
		} else {
			log.Info("not found private dataset, keyword=" + Key)
		}
		if datasetSize >= PageSize {
			if WebTotal > 0 { //next page， not first query.
				resultObj.Total = int64(WebTotal)
				ctx.JSON(200, resultObj)
				return
			}
		}
	} else {
		resultObj.PrivateTotal = int64(PrivateTotal)
	}

	from = from - PrivateTotal
	if from < 0 {
		from = 0
	}
	Size := PageSize - len(resultObj.Result)

	boolQ := elastic.NewBoolQuery()
	if Key != "" {
		nameQuery := elastic.NewMatchQuery("title", Key).Boost(2).QueryName("f_first")
		descQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second")
		fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third")
		categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth")
		boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery)
		res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context())
		if err == nil {
			searchJson, _ := json.Marshal(res)
			log.Info("searchJson=" + string(searchJson))
			esresult := makeDatasetResult(res, Key, OnlyReturnNum, language)
			resultObj.Total = resultObj.PrivateTotal + esresult.Total
			log.Info("query dataset es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total))
			resultObj.Result = append(resultObj.Result, esresult.Result...)
			ctx.JSON(200, resultObj)
		} else {
			log.Info("query es error," + err.Error())
		}
	} else {
		log.Info("query all datasets.")
		//搜索的属性要指定{"timestamp":{"unmapped_type":"date"}}
		res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context())
		if err == nil {
			searchJson, _ := json.Marshal(res)
			log.Info("searchJson=" + string(searchJson))
			esresult := makeDatasetResult(res, "", OnlyReturnNum, language)
			resultObj.Total = resultObj.PrivateTotal + esresult.Total
			log.Info("query dataset es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total))
			resultObj.Result = append(resultObj.Result, esresult.Result...)
			ctx.JSON(200, resultObj)
		} else {
			log.Info("query es error," + err.Error())
			ctx.JSON(200, "")
		}
	}

}

func makePrivateDataSet(datasets []*models.Dataset, res *SearchRes, Key string, language string) {
	for _, dataset := range datasets {
		record := make(map[string]interface{})

		record["id"] = dataset.ID
		userId := dataset.UserID

		user, errUser := models.GetUserByID(userId)
		if errUser == nil {
			record["owerName"] = user.GetDisplayName()
			record["avatar"] = user.RelAvatarLink()
		}

		repo, errRepo := models.GetRepositoryByID(dataset.RepoID)
		if errRepo == nil {
			log.Info("repo_url=" + repo.FullName())
			record["repoUrl"] = repo.FullName()
			record["avatar"] = repo.RelAvatarLink()
		} else {
			log.Info("repo err=" + errRepo.Error())
		}

		record["title"] = makeHighLight(Key, dataset.Title)
		record["description"] = truncLongText(makeHighLight(Key, dataset.Description), true)

		record["category"] = dataset.Category
		record["task"] = dataset.Task
		record["download_times"] = dataset.DownloadTimes
		record["created_unix"] = dataset.CreatedUnix
		record["updated_unix"] = repo.UpdatedUnix
		record["updated_html"] = timeutil.TimeSinceUnix(repo.UpdatedUnix, language)

		res.Result = append(res.Result, record)
	}
}

func makeDatasetResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, language string) *SearchRes {
	total := sRes.Hits.TotalHits.Value
	result := make([]map[string]interface{}, 0)
	if !OnlyReturnNum {
		for i, hit := range sRes.Hits.Hits {
			log.Info("this is dataset query " + fmt.Sprint(i) + " result.")
			recordSource := make(map[string]interface{})
			source, err := hit.Source.MarshalJSON()

			if err == nil {
				err = json.Unmarshal(source, &recordSource)
				if err == nil {
					record := make(map[string]interface{})
					record["id"] = hit.Id
					userIdStr := recordSource["user_id"].(string)
					userId, cerr := strconv.ParseInt(userIdStr, 10, 64)
					if cerr == nil {
						user, errUser := models.GetUserByID(userId)
						if errUser == nil {
							record["owerName"] = user.GetDisplayName()
							record["avatar"] = user.RelAvatarLink()
						}
					}
					setRepoInfo(recordSource, record)
					record["title"] = getLabelValue("title", recordSource, hit.Highlight)
					record["category"] = getLabelValue("category", recordSource, hit.Highlight)
					if recordSource["description"] != nil {
						desc := getLabelValue("description", recordSource, hit.Highlight)
						record["description"] = dealLongText(desc, Key, hit.MatchedQueries)
					} else {
						record["description"] = ""
					}
					record["file_name"] = getDatasetFileName(getLabelValue("file_name", recordSource, hit.Highlight))
					record["task"] = recordSource["task"]
					record["download_times"] = recordSource["download_times"]
					record["created_unix"] = recordSource["created_unix"]
					setUpdateHtml(record, recordSource["updated_unix"].(string), language)
					result = append(result, record)
				} else {
					log.Info("deal dataset source error," + err.Error())
				}
			} else {
				log.Info("deal dataset source error," + err.Error())
			}
		}
	}
	returnObj := &SearchRes{
		Total:  total,
		Result: result,
	}

	return returnObj
}

func getDatasetFileName(fileName string) string {
	slices := strings.Split(fileName, "-#,#-")
	fileName = strings.Join(slices, ",  ")
	return fileName
}

func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool, issueOrPr string) {

	/*
	   任务，合并请求  ES名称：issue-es-index
	   搜索:
	   name character varying(255) ,  标题
	   content text,  内容
	   comment text,  评论
	   排序:
	   updated_unix
	*/
	SortBy := ctx.Query("SortBy")
	ascending := ctx.QueryBool("Ascending")
	PrivateTotal := ctx.QueryInt("PrivateTotal")
	WebTotal := ctx.QueryInt("WebTotal")
	language := ctx.Query("language")
	if language == "" {
		language = "zh-CN"
	}
	from := (Page - 1) * PageSize
	if from == 0 {
		WebTotal = 0
	}
	resultObj := &SearchRes{}
	log.Info("WebTotal=" + fmt.Sprint(WebTotal))
	log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal))
	resultObj.Result = make([]map[string]interface{}, 0)
	isPull := false
	if issueOrPr == "t" {
		isPull = true
	}

	if ctx.User != nil && (from < PrivateTotal || from == 0) {

		log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil))
		issues, count, err := models.SearchPrivateIssueOrPr(Page, PageSize, Key, isPull, ctx.User.ID)
		if err != nil {
			ctx.JSON(200, "")
			return
		}
		resultObj.PrivateTotal = count
		issuesSize := len(issues)
		if issuesSize > 0 {
			log.Info("Query private repo issue number is:" + fmt.Sprint(issuesSize) + " count=" + fmt.Sprint(count))
			makePrivateIssueOrPr(issues, resultObj, Key, language)
		} else {
			log.Info("not found private repo issue,keyword=" + Key)
		}
		if issuesSize >= PageSize {
			if WebTotal > 0 { //next page， not first query.
				resultObj.Total = int64(WebTotal)
				ctx.JSON(200, resultObj)
				return
			}
		}
	} else {
		resultObj.PrivateTotal = int64(PrivateTotal)
	}

	from = from - PrivateTotal
	if from < 0 {
		from = 0
	}
	Size := PageSize - len(resultObj.Result)

	boolQ := elastic.NewBoolQuery()
	isIssueQuery := elastic.NewTermQuery("is_pull", issueOrPr)

	if Key != "" {
		boolKeyQ := elastic.NewBoolQuery()
		log.Info("issue Key=" + Key)
		nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first")
		contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second")
		commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third")
		boolKeyQ.Should(nameQuery, contentQuery, commentQuery)
		boolQ.Must(isIssueQuery, boolKeyQ)
	} else {
		boolQ.Must(isIssueQuery)
	}

	res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context())
	if err == nil {
		searchJson, _ := json.Marshal(res)
		log.Info("searchJson=" + string(searchJson))
		esresult := makeIssueResult(res, Key, OnlyReturnNum, language)

		resultObj.Total = resultObj.PrivateTotal + esresult.Total
		log.Info("query issue es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total))
		resultObj.Result = append(resultObj.Result, esresult.Result...)
		ctx.JSON(200, resultObj)
	} else {
		log.Info("query es error," + err.Error())
	}
}

func queryHighlight(names ...string) *elastic.Highlight {
	re := elastic.NewHighlight()
	for i := 0; i < len(names); i++ {
		field := &elastic.HighlighterField{
			Name: names[i],
		}
		re.Fields(field)
	}
	re.PreTags("<font color='red'>")
	re.PostTags("</font>")
	return re
}

func setRepoInfo(recordSource map[string]interface{}, record map[string]interface{}) {
	repoIdstr := recordSource["repo_id"].(string)
	repoId, cerr := strconv.ParseInt(repoIdstr, 10, 64)
	if cerr == nil {
		repo, errRepo := models.GetRepositoryByID(repoId)
		if errRepo == nil {
			log.Info("repo_url=" + repo.FullName())
			record["repoUrl"] = repo.FullName()
			record["avatar"] = repo.RelAvatarLink()
		} else {
			log.Info("repo err=" + errRepo.Error())
		}
	} else {
		log.Info("parse int err=" + cerr.Error())
	}
}

func makePrivateIssueOrPr(issues []*models.Issue, res *SearchRes, Key string, language string) {
	for _, issue := range issues {
		record := make(map[string]interface{})
		record["id"] = issue.ID
		record["repo_id"] = issue.RepoID

		repo, errRepo := models.GetRepositoryByID(issue.RepoID)
		if errRepo == nil {
			log.Info("repo_url=" + repo.FullName())
			record["repoUrl"] = repo.FullName()
			record["avatar"] = repo.RelAvatarLink()
		} else {
			log.Info("repo err=" + errRepo.Error())
		}
		record["name"] = makeHighLight(Key, issue.Title)
		record["content"] = truncLongText(makeHighLight(Key, issue.Content), true)

		if issue.IsPull {
			pr, err1 := issue.GetPullRequest()
			if err1 == nil && pr != nil {
				record["pr_id"] = pr.ID
			}
		}
		record["index"] = issue.Index
		record["num_comments"] = issue.NumComments
		record["is_closed"] = issue.IsClosed
		record["updated_unix"] = issue.UpdatedUnix
		record["updated_html"] = timeutil.TimeSinceUnix(issue.UpdatedUnix, language)
		res.Result = append(res.Result, record)
	}
}

func makeIssueResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, language string) *SearchRes {
	total := sRes.Hits.TotalHits.Value
	result := make([]map[string]interface{}, 0)
	if !OnlyReturnNum {
		for i, hit := range sRes.Hits.Hits {
			log.Info("this is issue query " + fmt.Sprint(i) + " result.")
			recordSource := make(map[string]interface{})
			source, err := hit.Source.MarshalJSON()

			if err == nil {
				err = json.Unmarshal(source, &recordSource)
				if err == nil {
					record := make(map[string]interface{})
					record["id"] = hit.Id
					record["repo_id"] = recordSource["repo_id"]
					log.Info("recordSource[\"repo_id\"]=" + fmt.Sprint(recordSource["repo_id"]))
					setRepoInfo(recordSource, record)
					record["name"] = getLabelValue("name", recordSource, hit.Highlight)
					if recordSource["content"] != nil {
						desc := getLabelValue("content", recordSource, hit.Highlight)
						record["content"] = dealLongText(desc, Key, hit.MatchedQueries)
						if _, ok := hit.Highlight["content"]; !ok {
							if _, ok_comment := hit.Highlight["comment"]; ok_comment {
								desc := getLabelValue("comment", recordSource, hit.Highlight)
								record["content"] = trimHrefHtml(dealLongText(desc, Key, hit.MatchedQueries))
							}
						}
					} else {
						if recordSource["comment"] != nil {
							desc := getLabelValue("comment", recordSource, hit.Highlight)
							record["content"] = dealLongText(desc, Key, hit.MatchedQueries)
						}
					}
					if recordSource["pr_id"] != nil {
						record["pr_id"] = recordSource["pr_id"]
					}
					log.Info("index=" + recordSource["index"].(string))
					record["index"] = recordSource["index"]
					record["num_comments"] = recordSource["num_comments"]
					record["is_closed"] = recordSource["is_closed"]
					record["updated_unix"] = recordSource["updated_unix"]
					setUpdateHtml(record, recordSource["updated_unix"].(string), language)
					result = append(result, record)
				} else {
					log.Info("deal issue source error," + err.Error())
				}
			} else {
				log.Info("deal issue source error," + err.Error())
			}
		}
	}
	returnObj := &SearchRes{
		Total:  total,
		Result: result,
	}

	return returnObj
}