Merge branch 'V20211018' into fix-455

go.mod

@@ -52,6 +52,7 @@ require (
 	github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28
 	github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14
 	github.com/golang/protobuf v1.4.1 // indirect
+	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/google/go-github/v24 v24.0.1
 	github.com/gorilla/context v1.1.1
 	github.com/hashicorp/go-retryablehttp v0.6.6 // indirect

models/cloudbrain.go

@@ -5,11 +5,12 @@ import (
 	"fmt"
 	"strings"
 	"time"
+	"xorm.io/builder"
 	"xorm.io/xorm"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
-	"xorm.io/builder"
 )
 
 type CloudbrainStatus string
@@ -59,6 +60,7 @@ type Cloudbrain struct {
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
 	DeletedAt   time.Time          `xorm:"deleted"`
 	CanDebug    bool               `xorm:"-"`
+	CanDel      bool               `xorm:"-"`
 	Type        int                `xorm:"INDEX DEFAULT 0"`
 
 	User *User       `xorm:"-"`
@@ -67,7 +69,7 @@ type Cloudbrain struct {
 
 type CloudbrainInfo struct {
 	Cloudbrain `xorm:"extends"`
-	User `xorm:"extends"`
+	User       `xorm:"extends"`
 }
 
 type CloudBrainLoginResult struct {
@@ -669,3 +671,24 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
 	cb := &Cloudbrain{JobName: jobName}
 	return getRepoCloudBrain(cb)
 }
+
+func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
+	if !isSigned || job.Status != string(JobStopped) {
+		return false
+	}
+	repo, err := GetRepositoryByID(job.RepoID)
+	if err != nil {
+		log.Error("GetRepositoryByID failed:%v", err.Error())
+		return false
+	}
+	permission, _ := GetUserRepoPermission(repo, user)
+	if err != nil {
+		log.Error("GetUserRepoPermission failed:%v", err.Error())
+		return false
+	}
+
+	if user.ID == job.UserID || user.IsAdmin || permission.AccessMode >= AccessModeAdmin {
+		return true
+	}
+	return false
+}

modules/repository/elk_pagedata.go

@@ -0,0 +1,312 @@
+package repository
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+//输入elk的json结构begin
+type InputInfo struct {
+	Batch []Batch `json:"batch"`
+}
+type Fields struct {
+	Field  string `json:"field"`
+	Format string `json:"format"`
+}
+type MatchPhrase struct {
+	Message string `json:"message"`
+}
+type Should struct {
+	MatchPhrase MatchPhrase `json:"match_phrase"`
+}
+type Bool struct {
+	Should             []Should `json:"should"`
+	MinimumShouldMatch int      `json:"minimum_should_match"`
+}
+type Timestamptest struct {
+	// Gte time.Time `json:"gte"`
+	Gte    string `json:"gte"`
+	Lte    string `json:"lte"`
+	Format string `json:"format"`
+}
+type Range struct {
+	Timestamptest Timestamptest `json:"@timestamptest"`
+}
+
+type FilterMatchPhrase struct {
+	UserName    string `json:"userName.keyword,omitempty"`
+	ProjectName string `json:"projectName.keyword,omitempty"`
+	TagName     string `json:"tagName.keyword,omitempty"`
+}
+
+type Filter struct {
+	Bool              *Bool              `json:"bool,omitempty"`
+	Range             *Range             `json:"range,omitempty"`
+	FilterMatchPhrase *FilterMatchPhrase `json:"match_phrase,omitempty"`
+}
+type MustNotMatchPhrase struct {
+	ProjectName string `json:"projectName"`
+}
+type MustNot struct {
+	MustNotMatchPhrase MustNotMatchPhrase `json:"match_phrase"`
+}
+type BoolIn struct {
+	Filter  []Filter  `json:"filter"`
+	MustNot []MustNot `json:"must_not"`
+}
+type Query struct {
+	BoolIn BoolIn `json:"bool"`
+}
+type Body struct {
+	Size   int      `json:"size"`
+	Fields []Fields `json:"fields"`
+	Query  Query    `json:"query"`
+}
+type Params struct {
+	Index string `json:"index"`
+	Body  Body   `json:"body"`
+}
+type Request struct {
+	Params Params `json:"params"`
+}
+type Batch struct {
+	Request Request `json:"request"`
+}
+
+//输入elk的json结构end
+
+//elk输出的json结构begin
+type Hits struct {
+	Total int `json:"total"`
+}
+type RawResponse struct {
+	Hits Hits `json:"hits"`
+}
+type Result struct {
+	RawResponse RawResponse `json:"rawResponse"`
+	Loaded      int         `json:"loaded"`
+}
+type ResultInfo struct {
+	Id     int    `json:"id"`
+	Result Result `json:"result"`
+}
+
+//elk输出的json结构end
+
+//发送post请求到elk
+func SendReqToElk(jsonStr []byte) (content string) {
+	ElkBase64Init := setting.ElkUser + ":" + setting.ElkPassword
+	ElkBase64 := base64.StdEncoding.EncodeToString([]byte(ElkBase64Init))
+	BasicElkBase64 := "Basic" + " " + ElkBase64
+	url := setting.ElkUrl
+	req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("kbn-version", "7.13.2")
+	req.Header.Set("Authorization", BasicElkBase64)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	body, _ := ioutil.ReadAll(resp.Body)
+	return string(body)
+}
+
+//处理返回的elk数据，只保留totalView，即访问量；loaded是分片载入次数，用来判断返回的数据是否准确
+func GetResultFromElk(resultinfo ResultInfo, jobResult string) (loaded int, totalView int) {
+	var resultTest ResultInfo
+	errs := json.Unmarshal([]byte(jobResult), &resultTest)
+	fmt.Println(errs)
+	return resultTest.Result.Loaded, resultTest.Result.RawResponse.Hits.Total
+}
+
+//初始化传给elk的数据结构，给定用户名和项目名，查询的起止时间，返回初始化后的结构
+func ProjectViewInit(User string, Project string, Gte string, Lte string) (projectViewInit InputInfo) {
+	var inputStruct InputInfo
+	inputStruct.Batch = make([]Batch, 1)
+	inputStruct.Batch[0].Request.Params.Index = setting.Index
+	inputStruct.Batch[0].Request.Params.Body.Size = 0
+	inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1)
+	inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField
+	inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3)
+	//限定查询时间
+	var timeRange Range
+	timeRange.Timestamptest.Gte = Gte
+	timeRange.Timestamptest.Lte = Lte
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Range = &timeRange
+	//限定用户
+	var userName FilterMatchPhrase
+	userName.UserName = User
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &userName
+	//限定项目
+	var projectName FilterMatchPhrase
+	projectName.ProjectName = Project
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].FilterMatchPhrase = &projectName
+	return inputStruct
+}
+
+//初始化传给elk的数据结构，给定查询信息和非项目名，查询的起止时间，返回初始化后的结构
+func AllProjectViewInit(MessageInfo string, NotProject string, Gte string, Lte string) (allProjectViewInit InputInfo) {
+	var inputStruct InputInfo
+	inputStruct.Batch = make([]Batch, 1)
+	inputStruct.Batch[0].Request.Params.Index = setting.Index
+	inputStruct.Batch[0].Request.Params.Body.Size = 0
+	inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1)
+	inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField
+	inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 2)
+	//限定message
+	var bool Bool
+	bool.Should = make([]Should, 1)
+	bool.Should[0].MatchPhrase.Message = MessageInfo
+	bool.MinimumShouldMatch = 1
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Bool = &bool
+	//限定查询时间
+	var timeRange Range
+	timeRange.Timestamptest.Gte = Gte
+	timeRange.Timestamptest.Lte = Lte
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].Range = &timeRange
+	//限定非项目
+	// var boolIn BoolIn
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.MustNot = make([]MustNot, 1)
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.MustNot[0].MustNotMatchPhrase.ProjectName = NotProject
+	return inputStruct
+}
+
+//初始化传给elk的数据结构，给定查询信息和tagName，查询的起止时间，返回初始化后的结构
+func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (projectViewInit InputInfo) {
+	var inputStruct InputInfo
+	inputStruct.Batch = make([]Batch, 1)
+	inputStruct.Batch[0].Request.Params.Index = setting.Index
+	inputStruct.Batch[0].Request.Params.Body.Size = 0
+	inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1)
+	inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField
+	inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3)
+	//限定message
+	var bool Bool
+	bool.Should = make([]Should, 1)
+	bool.Should[0].MatchPhrase.Message = MessageInfo
+	bool.MinimumShouldMatch = 1
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Bool = &bool
+	//限定tagName
+	var tagName FilterMatchPhrase
+	tagName.TagName = Tagname
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &tagName
+	//限定查询时间
+	var timeRange Range
+	timeRange.Timestamptest.Gte = Gte
+	timeRange.Timestamptest.Lte = Lte
+	inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].Range = &timeRange
+	return inputStruct
+}
+
+//向elk发送请求，将获取的结果只保留访问量，输入是初始化后的数据结构，返回访问量
+func ViewInfo(viewInfo InputInfo) (totalView int) {
+	jsons, errs := json.Marshal(viewInfo)
+	if errs != nil {
+		fmt.Println("errs:", errs.Error())
+	}
+	// fmt.Println("viewInfoInit:",string(jsons))
+	var jsonStr = []byte(jsons)
+	var resultInfo ResultInfo
+	loaded, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr))
+	time := 0
+	for {
+		if loaded == 0 {
+			loaded_next, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr))
+			time++
+			if loaded_next != 0 && time < 100 {
+				fmt.Println("totalView:", totalView)
+				return totalView
+			}
+			if time > 100 {
+				break
+			}
+		} else {
+			break
+		}
+	}
+	fmt.Println("loaded:", loaded)
+	return totalView
+}
+
+// @title    ProjectView
+// @description   获取指定用户和项目的访问量
+// @param     User       string      "用户名"
+// @param     Project    string      "项目名"
+// @param     Gte        string      "起始时间"  如time.Now().AddDate(0, 0, -1).Format(time.RFC3339)
+// @param     Lte        string      "结束时间"  如time.Now().Format(time.RFC3339)
+// @return    totalView      int         "访问量"
+func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int) {
+	InitInfo := ProjectViewInit(User, Project, Gte, Lte)
+	return ViewInfo(InitInfo)
+}
+
+//统计项目相关页面的访问量
+type ProjectInfo struct {
+	/* 统计所有项目中该页面的浏览情况，不需要区分项目。以aiforge项目为例 */
+	//地址：https://git.openi.org.cn/OpenI/aiforge/datasets?type=0
+	Project_dataset_type_0 int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/datasets?type=1
+	Project_dataset_type_1 int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/issues
+	Project_issues int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/labels
+	Project_labels int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/milestones
+	Project_milestones int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/pulls
+	Project_pulls int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/release
+	Project_release int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/wiki
+	Project_wiki int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/activity
+	Project_activity int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/cloudbrain
+	Project_cloudbrain int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/modelarts
+	Project_modelarts int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/blockchain
+	Project_blockchain int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/watchers
+	Project_watchers int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/stars
+	Project_stars int
+	//地址：https://git.openi.org.cn/OpenI/aiforge/forks
+	Project_forks int
+}
+
+// @title    AllProjectView
+// @description   获取指定用户和项目的访问量
+// @param     Gte        string      "起始时间"  如time.Now().AddDate(0, 0, -1).Format(time.RFC3339)
+// @param     Lte        string      "结束时间"
+// @return    projectInfo      ProjectInfo         "统计所有项目中页面的浏览情况，不需要区分项目"
+func AllProjectView(Gte string, Lte string) (projectInfo ProjectInfo) {
+	projectInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte))
+	projectInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	projectInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte))
+	return projectInfo
+}

modules/setting/setting.go

@@ -482,6 +482,14 @@ var (
 	PoolInfos         string
 	Flavor            string
 	FlavorInfos       string
+
+	//elk config
+	ElkUrl        string
+	ElkUser       string
+	ElkPassword   string
+	Index         string
+	TimeField     string
+	ElkTimeFormat string
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -1201,6 +1209,14 @@ func NewContext() {
 	PoolInfos = sec.Key("POOL_INFOS").MustString("")
 	Flavor = sec.Key("FLAVOR").MustString("")
 	FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("")
+
+	sec = Cfg.Section("elk")
+	ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch")
+	ElkUser = sec.Key("ELKUSER").MustString("Qizhi")
+	ElkPassword = sec.Key("ELKPASSWORD").MustString("Pcl2020")
+	Index = sec.Key("INDEX").MustString("filebeat-7.3.2*")
+	TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest")
+	ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time")
 }
 
 func loadInternalToken(sec *ini.Section) string {

routers/repo/cloudbrain.go

@@ -69,12 +69,13 @@ func CloudBrainIndex(ctx *context.Context) {
 
 	timestamp := time.Now().Unix()
 	for i, task := range ciTasks {
-		log.Info("", task.User.Name)
 		if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) {
 			ciTasks[i].CanDebug = true
 		} else {
 			ciTasks[i].CanDebug = false
 		}
+
+		ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task)
 	}
 
 	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)

templates/repo/cloudbrain/index.tmpl

@@ -353,9 +353,9 @@
                                     </div>
                                     
                                     <!-- 删除镜像 -->
-                                    <form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if ne .Status "STOPPED"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
+                                    <form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if not .CanDel}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
                                         {{$.CsrfTokenHtml}}
-                                        <a class="ui compact {{if ne .Status "STOPPED"}}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">
+                                        <a class="ui compact {{if not .CanDel}}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">
                                             删除
                                         </a>
                                     </form>