From 616aba813cee662f0abae1b1c42ee36bee4b70a8 Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 12 Oct 2021 16:55:30 +0800 Subject: [PATCH 1/8] el --- models/elk_pagedata.go | 180 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 models/elk_pagedata.go diff --git a/models/elk_pagedata.go b/models/elk_pagedata.go new file mode 100644 index 000000000..2f5c241a8 --- /dev/null +++ b/models/elk_pagedata.go @@ -0,0 +1,180 @@ +package models + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +//输出的json结构begin +type Hits struct { + Total int `json:"total"` +} +type RawResponse struct { + Hits Hits `json:"hits"` +} +type Result struct { + RawResponse RawResponse `json:"rawResponse"` + Loaded int `json:"loaded"` +} +type ResultInfo struct { + Id int `json:"id"` + Result Result `json:"result"` +} + +//输出的json结构end + +// //输入的json结构end +type InputInfo struct { + Batch []Batch `json:"batch"` +} +type Fields struct { + Field string `json:"field"` + Format string `json:"format"` +} +type MatchPhrase struct { + Message string `json:"message"` +} +type Should struct { + MatchPhrase MatchPhrase `json:"match_phrase"` +} +type Bool struct { + Should []Should `json:"should"` + MinimumShouldMatch int `json:"minimum_should_match"` +} +type Timestamptest struct { + // Gte time.Time `json:"gte"` + Gte string `json:"gte"` + Lte string `json:"lte"` + Format string `json:"format"` +} +type Range struct { + Timestamptest Timestamptest `json:"@timestamptest"` +} + +type FilterMatchPhrase struct { + UserName string `json:"userName.keyword,omitempty"` + ProjectName string `json:"projectName.keyword,omitempty"` +} + +type Filter struct { + Bool *Bool `json:"bool,omitempty"` + Range *Range `json:"range,omitempty"` + FilterMatchPhrase *FilterMatchPhrase `json:"match_phrase,omitempty"` +} +type MatchPhraseIn struct { + ProjectName string `json:"projectName"` +} +type MustNot struct { + MatchPhraseIn MatchPhraseIn `json:"match_phrase"` +} +type BoolIn struct { + Filter []Filter `json:"filter"` + MustNot []MustNot `json:"must_not"` +} +type Query struct { + BoolIn BoolIn `json:"bool"` +} +type Body struct { + Size int `json:"size"` + Fields []Fields `json:"fields"` + Query Query `json:"query"` +} +type Params struct { + Index string `json:"index"` + Body Body `json:"body"` +} +type Request struct { + Params Params `json:"params"` +} +type Batch struct { + Request Request `json:"request"` +} + +func SendData(jsonStr []byte) (content string) { + url := "http://192.168.207.35:5601/internal/bsearch" + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-version", "7.13.2") + req.Header.Set("Authorization", "Basic cWl6aGk6UGNsMjAyMA==") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + return string(body) +} + +func getdata(resultinfo ResultInfo, jobResult string) (value1 int, value2 int) { + var resultTest ResultInfo + errs := json.Unmarshal([]byte(jobResult), &resultTest) + fmt.Println(errs) + return resultTest.Result.Loaded, resultTest.Result.RawResponse.Hits.Total +} + +func inputStructInit(User string, Project string, Gte string, Lte string) (inputStructInit InputInfo) { + var inputStruct InputInfo + inputStruct.Batch = make([]Batch, 1) + inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Body.Size = 0 + inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3) + + var timeRange Range + timeRange.Timestamptest.Gte = Gte + timeRange.Timestamptest.Lte = Lte + timeRange.Timestamptest.Format = "strict_date_optional_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Range = &timeRange + + var userName FilterMatchPhrase + userName.UserName = User + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &userName + + var projectName FilterMatchPhrase + projectName.ProjectName = Project + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].FilterMatchPhrase = &projectName + + return inputStruct +} + +// @title ProjectViewData +// @description 获取指定用户和项目的访问量 +// @param User string "用户名" +// @param Project string "项目名" +// @param Gte string "起始时间" 如 time.Now().AddDate(0, 0, -1).Format(time.RFC3339) +// @param Lte string "结束时间" 如 time.Now().Format(time.RFC3339) +// @return value int "访问量" +func ProjectViewData(User string, Project string, Gte string, Lte string) (value int) { + InitInfo := inputStructInit(User, Project, Gte, Lte) + jsons, errs := json.Marshal(InitInfo) + if errs != nil { + fmt.Println("errs:", errs.Error()) + } + var jsonStr = []byte(jsons) + var resultTest ResultInfo + loaded, total := getdata(resultTest, SendData(jsonStr)) + time := 0 + for { + if loaded == 0 { + value1, total := getdata(resultTest, SendData(jsonStr)) + time++ + if value1 != 0 && time < 100 { + fmt.Println("total:", total) + return total + } + if time > 100 { + break + } + } else { + break + } + } + fmt.Println("loaded:", loaded) + return total +} From 400b1bd1a620c165acfd86dc81aac4be4d8a2c06 Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 12 Oct 2021 17:10:04 +0800 Subject: [PATCH 2/8] elk --- models/elk_pagedata.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/models/elk_pagedata.go b/models/elk_pagedata.go index 2f5c241a8..a0f7f3ca0 100644 --- a/models/elk_pagedata.go +++ b/models/elk_pagedata.go @@ -93,7 +93,7 @@ type Batch struct { Request Request `json:"request"` } -func SendData(jsonStr []byte) (content string) { +func sendData(jsonStr []byte) (content string) { url := "http://192.168.207.35:5601/internal/bsearch" req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) req.Header.Set("Content-Type", "application/json") @@ -109,7 +109,7 @@ func SendData(jsonStr []byte) (content string) { return string(body) } -func getdata(resultinfo ResultInfo, jobResult string) (value1 int, value2 int) { +func getData(resultinfo ResultInfo, jobResult string) (value1 int, value2 int) { var resultTest ResultInfo errs := json.Unmarshal([]byte(jobResult), &resultTest) fmt.Println(errs) @@ -143,14 +143,14 @@ func inputStructInit(User string, Project string, Gte string, Lte string) (input return inputStruct } -// @title ProjectViewData +// @title projectViewData // @description 获取指定用户和项目的访问量 // @param User string "用户名" // @param Project string "项目名" // @param Gte string "起始时间" 如 time.Now().AddDate(0, 0, -1).Format(time.RFC3339) // @param Lte string "结束时间" 如 time.Now().Format(time.RFC3339) // @return value int "访问量" -func ProjectViewData(User string, Project string, Gte string, Lte string) (value int) { +func projectViewData(User string, Project string, Gte string, Lte string) (value int) { InitInfo := inputStructInit(User, Project, Gte, Lte) jsons, errs := json.Marshal(InitInfo) if errs != nil { @@ -158,11 +158,11 @@ func ProjectViewData(User string, Project string, Gte string, Lte string) (value } var jsonStr = []byte(jsons) var resultTest ResultInfo - loaded, total := getdata(resultTest, SendData(jsonStr)) + loaded, total := getData(resultTest, sendData(jsonStr)) time := 0 for { if loaded == 0 { - value1, total := getdata(resultTest, SendData(jsonStr)) + value1, total := getData(resultTest, sendData(jsonStr)) time++ if value1 != 0 && time < 100 { fmt.Println("total:", total) From 895739e48c0710e41ab7413bc01cf3c615f89ce8 Mon Sep 17 00:00:00 2001 From: Gitea Date: Wed, 13 Oct 2021 16:43:23 +0800 Subject: [PATCH 3/8] elk_projectView --- models/elk_pagedata.go | 180 --------------------- modules/repository/elk_pagedata.go | 309 +++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 180 deletions(-) delete mode 100644 models/elk_pagedata.go create mode 100644 modules/repository/elk_pagedata.go diff --git a/models/elk_pagedata.go b/models/elk_pagedata.go deleted file mode 100644 index a0f7f3ca0..000000000 --- a/models/elk_pagedata.go +++ /dev/null @@ -1,180 +0,0 @@ -package models - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" -) - -//输出的json结构begin -type Hits struct { - Total int `json:"total"` -} -type RawResponse struct { - Hits Hits `json:"hits"` -} -type Result struct { - RawResponse RawResponse `json:"rawResponse"` - Loaded int `json:"loaded"` -} -type ResultInfo struct { - Id int `json:"id"` - Result Result `json:"result"` -} - -//输出的json结构end - -// //输入的json结构end -type InputInfo struct { - Batch []Batch `json:"batch"` -} -type Fields struct { - Field string `json:"field"` - Format string `json:"format"` -} -type MatchPhrase struct { - Message string `json:"message"` -} -type Should struct { - MatchPhrase MatchPhrase `json:"match_phrase"` -} -type Bool struct { - Should []Should `json:"should"` - MinimumShouldMatch int `json:"minimum_should_match"` -} -type Timestamptest struct { - // Gte time.Time `json:"gte"` - Gte string `json:"gte"` - Lte string `json:"lte"` - Format string `json:"format"` -} -type Range struct { - Timestamptest Timestamptest `json:"@timestamptest"` -} - -type FilterMatchPhrase struct { - UserName string `json:"userName.keyword,omitempty"` - ProjectName string `json:"projectName.keyword,omitempty"` -} - -type Filter struct { - Bool *Bool `json:"bool,omitempty"` - Range *Range `json:"range,omitempty"` - FilterMatchPhrase *FilterMatchPhrase `json:"match_phrase,omitempty"` -} -type MatchPhraseIn struct { - ProjectName string `json:"projectName"` -} -type MustNot struct { - MatchPhraseIn MatchPhraseIn `json:"match_phrase"` -} -type BoolIn struct { - Filter []Filter `json:"filter"` - MustNot []MustNot `json:"must_not"` -} -type Query struct { - BoolIn BoolIn `json:"bool"` -} -type Body struct { - Size int `json:"size"` - Fields []Fields `json:"fields"` - Query Query `json:"query"` -} -type Params struct { - Index string `json:"index"` - Body Body `json:"body"` -} -type Request struct { - Params Params `json:"params"` -} -type Batch struct { - Request Request `json:"request"` -} - -func sendData(jsonStr []byte) (content string) { - url := "http://192.168.207.35:5601/internal/bsearch" - req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("kbn-version", "7.13.2") - req.Header.Set("Authorization", "Basic cWl6aGk6UGNsMjAyMA==") - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - panic(err) - } - defer resp.Body.Close() - body, _ := ioutil.ReadAll(resp.Body) - return string(body) -} - -func getData(resultinfo ResultInfo, jobResult string) (value1 int, value2 int) { - var resultTest ResultInfo - errs := json.Unmarshal([]byte(jobResult), &resultTest) - fmt.Println(errs) - return resultTest.Result.Loaded, resultTest.Result.RawResponse.Hits.Total -} - -func inputStructInit(User string, Project string, Gte string, Lte string) (inputStructInit InputInfo) { - var inputStruct InputInfo - inputStruct.Batch = make([]Batch, 1) - inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" - inputStruct.Batch[0].Request.Params.Body.Size = 0 - inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) - inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" - inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" - inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3) - - var timeRange Range - timeRange.Timestamptest.Gte = Gte - timeRange.Timestamptest.Lte = Lte - timeRange.Timestamptest.Format = "strict_date_optional_time" - inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Range = &timeRange - - var userName FilterMatchPhrase - userName.UserName = User - inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &userName - - var projectName FilterMatchPhrase - projectName.ProjectName = Project - inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].FilterMatchPhrase = &projectName - - return inputStruct -} - -// @title projectViewData -// @description 获取指定用户和项目的访问量 -// @param User string "用户名" -// @param Project string "项目名" -// @param Gte string "起始时间" 如 time.Now().AddDate(0, 0, -1).Format(time.RFC3339) -// @param Lte string "结束时间" 如 time.Now().Format(time.RFC3339) -// @return value int "访问量" -func projectViewData(User string, Project string, Gte string, Lte string) (value int) { - InitInfo := inputStructInit(User, Project, Gte, Lte) - jsons, errs := json.Marshal(InitInfo) - if errs != nil { - fmt.Println("errs:", errs.Error()) - } - var jsonStr = []byte(jsons) - var resultTest ResultInfo - loaded, total := getData(resultTest, sendData(jsonStr)) - time := 0 - for { - if loaded == 0 { - value1, total := getData(resultTest, sendData(jsonStr)) - time++ - if value1 != 0 && time < 100 { - fmt.Println("total:", total) - return total - } - if time > 100 { - break - } - } else { - break - } - } - fmt.Println("loaded:", loaded) - return total -} diff --git a/modules/repository/elk_pagedata.go b/modules/repository/elk_pagedata.go new file mode 100644 index 000000000..fa4b61beb --- /dev/null +++ b/modules/repository/elk_pagedata.go @@ -0,0 +1,309 @@ +package repository + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +//输入elk的json结构begin +type InputInfo struct { + Batch []Batch `json:"batch"` +} +type Fields struct { + Field string `json:"field"` + Format string `json:"format"` +} +type MatchPhrase struct { + Message string `json:"message"` +} +type Should struct { + MatchPhrase MatchPhrase `json:"match_phrase"` +} +type Bool struct { + Should []Should `json:"should"` + MinimumShouldMatch int `json:"minimum_should_match"` +} +type Timestamptest struct { + // Gte time.Time `json:"gte"` + Gte string `json:"gte"` + Lte string `json:"lte"` + Format string `json:"format"` +} +type Range struct { + Timestamptest Timestamptest `json:"@timestamptest"` +} + +type FilterMatchPhrase struct { + UserName string `json:"userName.keyword,omitempty"` + ProjectName string `json:"projectName.keyword,omitempty"` + TagName string `json:"tagName.keyword,omitempty"` +} + +type Filter struct { + Bool *Bool `json:"bool,omitempty"` + Range *Range `json:"range,omitempty"` + FilterMatchPhrase *FilterMatchPhrase `json:"match_phrase,omitempty"` +} +type MustNotMatchPhrase struct { + ProjectName string `json:"projectName"` +} +type MustNot struct { + MustNotMatchPhrase MustNotMatchPhrase `json:"match_phrase"` +} +type BoolIn struct { + Filter []Filter `json:"filter"` + MustNot []MustNot `json:"must_not"` +} +type Query struct { + BoolIn BoolIn `json:"bool"` +} +type Body struct { + Size int `json:"size"` + Fields []Fields `json:"fields"` + Query Query `json:"query"` +} +type Params struct { + Index string `json:"index"` + Body Body `json:"body"` +} +type Request struct { + Params Params `json:"params"` +} +type Batch struct { + Request Request `json:"request"` +} + +//输入elk的json结构end + +//elk输出的json结构begin +type Hits struct { + Total int `json:"total"` +} +type RawResponse struct { + Hits Hits `json:"hits"` +} +type Result struct { + RawResponse RawResponse `json:"rawResponse"` + Loaded int `json:"loaded"` +} +type ResultInfo struct { + Id int `json:"id"` + Result Result `json:"result"` +} + +//elk输出的json结构end + +//发送post请求到elk +func SendReqToElk(jsonStr []byte) (content string) { + url := "http://192.168.207.35:5601/internal/bsearch" + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-version", "7.13.2") + req.Header.Set("Authorization", "Basic cWl6aGk6UGNsMjAyMA==") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + return string(body) +} + +//处理返回的elk数据，只保留totalView，即访问量；loaded是分片载入次数，用来判断返回的数据是否准确 +func GetResultFromElk(resultinfo ResultInfo, jobResult string) (loaded int, totalView int) { + var resultTest ResultInfo + errs := json.Unmarshal([]byte(jobResult), &resultTest) + fmt.Println(errs) + return resultTest.Result.Loaded, resultTest.Result.RawResponse.Hits.Total +} + +//初始化传给elk的数据结构，给定用户名和项目名，查询的起止时间，返回初始化后的结构 +func ProjectViewInit(User string, Project string, Gte string, Lte string) (projectViewInit InputInfo) { + var inputStruct InputInfo + inputStruct.Batch = make([]Batch, 1) + inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Body.Size = 0 + inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3) + //限定查询时间 + var timeRange Range + timeRange.Timestamptest.Gte = Gte + timeRange.Timestamptest.Lte = Lte + timeRange.Timestamptest.Format = "strict_date_optional_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Range = &timeRange + //限定用户 + var userName FilterMatchPhrase + userName.UserName = User + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &userName + //限定项目 + var projectName FilterMatchPhrase + projectName.ProjectName = Project + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].FilterMatchPhrase = &projectName + return inputStruct +} + +//初始化传给elk的数据结构，给定查询信息和非项目名，查询的起止时间，返回初始化后的结构 +func AllProjectViewInit(MessageInfo string, NotProject string, Gte string, Lte string) (allProjectViewInit InputInfo) { + var inputStruct InputInfo + inputStruct.Batch = make([]Batch, 1) + inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Body.Size = 0 + inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 2) + //限定message + var bool Bool + bool.Should = make([]Should, 1) + bool.Should[0].MatchPhrase.Message = MessageInfo + bool.MinimumShouldMatch = 1 + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Bool = &bool + //限定查询时间 + var timeRange Range + timeRange.Timestamptest.Gte = Gte + timeRange.Timestamptest.Lte = Lte + timeRange.Timestamptest.Format = "strict_date_optional_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].Range = &timeRange + //限定非项目 + // var boolIn BoolIn + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.MustNot = make([]MustNot, 1) + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.MustNot[0].MustNotMatchPhrase.ProjectName = NotProject + return inputStruct +} + +//初始化传给elk的数据结构，给定查询信息和tagName，查询的起止时间，返回初始化后的结构 +func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (projectViewInit InputInfo) { + var inputStruct InputInfo + inputStruct.Batch = make([]Batch, 1) + inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Body.Size = 0 + inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3) + //限定message + var bool Bool + bool.Should = make([]Should, 1) + bool.Should[0].MatchPhrase.Message = MessageInfo + bool.MinimumShouldMatch = 1 + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Bool = &bool + //限定tagName + var tagName FilterMatchPhrase + tagName.TagName = Tagname + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &tagName + //限定查询时间 + var timeRange Range + timeRange.Timestamptest.Gte = Gte + timeRange.Timestamptest.Lte = Lte + timeRange.Timestamptest.Format = "strict_date_optional_time" + inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].Range = &timeRange + return inputStruct +} + +//向elk发送请求，将获取的结果只保留访问量，输入是初始化后的数据结构，返回访问量 +func ViewInfo(viewInfo InputInfo) (totalView int) { + jsons, errs := json.Marshal(viewInfo) + if errs != nil { + fmt.Println("errs:", errs.Error()) + } + // fmt.Println("viewInfoInit:",string(jsons)) + var jsonStr = []byte(jsons) + var resultInfo ResultInfo + loaded, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr)) + time := 0 + for { + if loaded == 0 { + loaded_next, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr)) + time++ + if loaded_next != 0 && time < 100 { + fmt.Println("totalView:", totalView) + return totalView + } + if time > 100 { + break + } + } else { + break + } + } + fmt.Println("loaded:", loaded) + return totalView +} + +// @title ProjectView +// @description 获取指定用户和项目的访问量 +// @param User string "用户名" +// @param Project string "项目名" +// @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) +// @param Lte string "结束时间" +// @return totalView int "访问量" +func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int) { + InitInfo := ProjectViewInit(User, Project, Gte, Lte) + return ViewInfo(InitInfo) +} + +//统计项目相关页面的访问量 +type ProjectInfo struct { + /* 统计所有项目中该页面的浏览情况，不需要区分项目。以aiforge项目为例 */ + //地址：https://git.openi.org.cn/OpenI/aiforge/datasets?type=0 + Project_dataset_type_0 int + //地址：https://git.openi.org.cn/OpenI/aiforge/datasets?type=1 + Project_dataset_type_1 int + //地址：https://git.openi.org.cn/OpenI/aiforge/issues + Project_issues int + //地址：https://git.openi.org.cn/OpenI/aiforge/labels + Project_labels int + //地址：https://git.openi.org.cn/OpenI/aiforge/milestones + Project_milestones int + //地址：https://git.openi.org.cn/OpenI/aiforge/pulls + Project_pulls int + //地址：https://git.openi.org.cn/OpenI/aiforge/release + Project_release int + //地址：https://git.openi.org.cn/OpenI/aiforge/wiki + Project_wiki int + //地址：https://git.openi.org.cn/OpenI/aiforge/activity + Project_activity int + //地址：https://git.openi.org.cn/OpenI/aiforge/cloudbrain + Project_cloudbrain int + //地址：https://git.openi.org.cn/OpenI/aiforge/modelarts + Project_modelarts int + //地址：https://git.openi.org.cn/OpenI/aiforge/blockchain + Project_blockchain int + //地址：https://git.openi.org.cn/OpenI/aiforge/watchers + Project_watchers int + //地址：https://git.openi.org.cn/OpenI/aiforge/stars + Project_stars int + //地址：https://git.openi.org.cn/OpenI/aiforge/forks + Project_forks int +} + +// @title AllProjectView +// @description 获取指定用户和项目的访问量 +// @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) +// @param Lte string "结束时间" +// @return projectInfo ProjectInfo "统计所有项目中页面的浏览情况，不需要区分项目" +func AllProjectView(Gte string, Lte string) (projectInfo ProjectInfo) { + projectInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte)) + projectInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte)) + return projectInfo +} From fae6e7523ffef75e2f1e4b71de205b685bbe530d Mon Sep 17 00:00:00 2001 From: Gitea Date: Wed, 13 Oct 2021 19:34:11 +0800 Subject: [PATCH 4/8] Add configuration information --- modules/repository/elk_pagedata.go | 27 +++++++++++++-------------- modules/setting/setting.go | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/modules/repository/elk_pagedata.go b/modules/repository/elk_pagedata.go index fa4b61beb..f74324664 100644 --- a/modules/repository/elk_pagedata.go +++ b/modules/repository/elk_pagedata.go @@ -6,6 +6,8 @@ import ( "fmt" "io/ioutil" "net/http" + + "code.gitea.io/gitea/modules/setting" ) //输入elk的json结构begin @@ -98,7 +100,7 @@ type ResultInfo struct { //发送post请求到elk func SendReqToElk(jsonStr []byte) (content string) { - url := "http://192.168.207.35:5601/internal/bsearch" + url := setting.ElkUrl req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) req.Header.Set("Content-Type", "application/json") req.Header.Set("kbn-version", "7.13.2") @@ -126,17 +128,16 @@ func GetResultFromElk(resultinfo ResultInfo, jobResult string) (loaded int, tota func ProjectViewInit(User string, Project string, Gte string, Lte string) (projectViewInit InputInfo) { var inputStruct InputInfo inputStruct.Batch = make([]Batch, 1) - inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Index = setting.Index inputStruct.Batch[0].Request.Params.Body.Size = 0 inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) - inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" - inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3) //限定查询时间 var timeRange Range timeRange.Timestamptest.Gte = Gte timeRange.Timestamptest.Lte = Lte - timeRange.Timestamptest.Format = "strict_date_optional_time" inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Range = &timeRange //限定用户 var userName FilterMatchPhrase @@ -153,11 +154,11 @@ func ProjectViewInit(User string, Project string, Gte string, Lte string) (proje func AllProjectViewInit(MessageInfo string, NotProject string, Gte string, Lte string) (allProjectViewInit InputInfo) { var inputStruct InputInfo inputStruct.Batch = make([]Batch, 1) - inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Index = setting.Index inputStruct.Batch[0].Request.Params.Body.Size = 0 inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) - inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" - inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 2) //限定message var bool Bool @@ -169,7 +170,6 @@ func AllProjectViewInit(MessageInfo string, NotProject string, Gte string, Lte s var timeRange Range timeRange.Timestamptest.Gte = Gte timeRange.Timestamptest.Lte = Lte - timeRange.Timestamptest.Format = "strict_date_optional_time" inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].Range = &timeRange //限定非项目 // var boolIn BoolIn @@ -182,11 +182,11 @@ func AllProjectViewInit(MessageInfo string, NotProject string, Gte string, Lte s func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (projectViewInit InputInfo) { var inputStruct InputInfo inputStruct.Batch = make([]Batch, 1) - inputStruct.Batch[0].Request.Params.Index = "filebeat-7.3.2*" + inputStruct.Batch[0].Request.Params.Index = setting.Index inputStruct.Batch[0].Request.Params.Body.Size = 0 inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1) - inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = "@timestamptest" - inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = "date_time" + inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField + inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3) //限定message var bool Bool @@ -202,7 +202,6 @@ func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (pr var timeRange Range timeRange.Timestamptest.Gte = Gte timeRange.Timestamptest.Lte = Lte - timeRange.Timestamptest.Format = "strict_date_optional_time" inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].Range = &timeRange return inputStruct } @@ -242,7 +241,7 @@ func ViewInfo(viewInfo InputInfo) (totalView int) { // @param User string "用户名" // @param Project string "项目名" // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) -// @param Lte string "结束时间" +// @param Lte string "结束时间" 如time.Now().Format(time.RFC3339) // @return totalView int "访问量" func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int) { InitInfo := ProjectViewInit(User, Project, Gte, Lte) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0fbc9f909..1830deff4 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -482,6 +482,14 @@ var ( PoolInfos string Flavor string FlavorInfos string + + //obs config + ElkUrl string + ElkUser string + ElkPassword string + Index string + TimeField string + ElkTimeFormat string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1201,6 +1209,14 @@ func NewContext() { PoolInfos = sec.Key("POOL_INFOS").MustString("") Flavor = sec.Key("FLAVOR").MustString("") FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("") + + sec = Cfg.Section("elk") + ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch") + ElkUser = sec.Key("ELKUSER").MustString("Qizhi") + ElkPassword = sec.Key("ELKPASSWORD").MustString("Pcl2020") + Index = sec.Key("INDEX").MustString("filebeat-7.3.2*") + TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest") + ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") } func loadInternalToken(sec *ini.Section) string { From 1f3c45e527baca223b2abb48b278d3904ae5c91f Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 14 Oct 2021 10:38:36 +0800 Subject: [PATCH 5/8] elk --- modules/setting/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1830deff4..e1e7b7902 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -483,7 +483,7 @@ var ( Flavor string FlavorInfos string - //obs config + //elk config ElkUrl string ElkUser string ElkPassword string From 10d7e9f8e814783265c003319266dff37f35ee5d Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 14 Oct 2021 10:49:35 +0800 Subject: [PATCH 6/8] base64 --- modules/repository/elk_pagedata.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/repository/elk_pagedata.go b/modules/repository/elk_pagedata.go index f74324664..bb027726d 100644 --- a/modules/repository/elk_pagedata.go +++ b/modules/repository/elk_pagedata.go @@ -2,6 +2,7 @@ package repository import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "io/ioutil" @@ -100,11 +101,14 @@ type ResultInfo struct { //发送post请求到elk func SendReqToElk(jsonStr []byte) (content string) { + ElkBase64Init := setting.ElkUser + ":" + setting.ElkPassword + ElkBase64 := base64.StdEncoding.EncodeToString([]byte(ElkBase64Init)) + BasicElkBase64 := "Basic" + " " + ElkBase64 url := setting.ElkUrl req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) req.Header.Set("Content-Type", "application/json") req.Header.Set("kbn-version", "7.13.2") - req.Header.Set("Authorization", "Basic cWl6aGk6UGNsMjAyMA==") + req.Header.Set("Authorization", BasicElkBase64) client := &http.Client{} resp, err := client.Do(req) From 4dca969c691d66665bc41510dac3b4dc88286be8 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 14 Oct 2021 11:55:42 +0800 Subject: [PATCH 7/8] fix-464 --- go.mod | 1 + models/cloudbrain.go | 25 ++++++++++++++++++++++++- routers/repo/cloudbrain.go | 3 ++- templates/repo/cloudbrain/index.tmpl | 4 ++-- 4 files changed, 29 insertions(+), 4 deletions(-) mode change 100644 => 100755 go.mod diff --git a/go.mod b/go.mod old mode 100644 new mode 100755 index c663ab2ff..9f93281c3 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28 github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 github.com/golang/protobuf v1.4.1 // indirect + github.com/gomodule/redigo v2.0.0+incompatible github.com/google/go-github/v24 v24.0.1 github.com/gorilla/context v1.1.1 github.com/hashicorp/go-retryablehttp v0.6.6 // indirect diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 4f615ed00..b2c5eb874 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,6 +1,7 @@ package models import ( + "code.gitea.io/gitea/modules/log" "encoding/json" "fmt" "strings" @@ -59,6 +60,7 @@ type Cloudbrain struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` DeletedAt time.Time `xorm:"deleted"` CanDebug bool `xorm:"-"` + CanDel bool `xorm:"-"` Type int `xorm:"INDEX DEFAULT 0"` User *User `xorm:"-"` @@ -67,7 +69,7 @@ type Cloudbrain struct { type CloudbrainInfo struct { Cloudbrain `xorm:"extends"` - User `xorm:"extends"` + User `xorm:"extends"` } type CloudBrainLoginResult struct { @@ -669,3 +671,24 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobName: jobName} return getRepoCloudBrain(cb) } + +func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { + if !isSigned || job.Status != string(JobStopped) { + return false + } + repo, err := GetRepositoryByID(job.RepoID) + if err != nil { + log.Error("GetRepositoryByID failed:%v", err.Error()) + return false + } + permission, _ := GetUserRepoPermission(repo, user) + if err != nil { + log.Error("GetUserRepoPermission failed:%v", err.Error()) + return false + } + + if user.ID == job.UserID || user.IsAdmin || permission.AccessMode >= AccessModeAdmin { + return true + } + return false +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index ccd51a89f..c0295bd43 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -69,12 +69,13 @@ func CloudBrainIndex(ctx *context.Context) { timestamp := time.Now().Unix() for i, task := range ciTasks { - log.Info("", task.User.Name) if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) { ciTasks[i].CanDebug = true } else { ciTasks[i].CanDebug = false } + + ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index b1afa7027..200f5e133 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -353,9 +353,9 @@ -
+ {{$.CsrfTokenHtml}} - + 删除
From efd3cae25f475728b1899724ef71d99084b92d68 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 14 Oct 2021 14:17:25 +0800 Subject: [PATCH 8/8] format --- models/cloudbrain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index b2c5eb874..1ff4801f1 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,16 +1,16 @@ package models import ( - "code.gitea.io/gitea/modules/log" "encoding/json" "fmt" "strings" "time" + "xorm.io/builder" "xorm.io/xorm" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "xorm.io/builder" ) type CloudbrainStatus string