Merge branch 'V20211115' into liuzx_trainjob

models/models.go

@@ -139,6 +139,7 @@ func init() {
 		new(RepoStatistic),
 		new(SummaryStatistic),
 		new(UserBusinessAnalysis),
+		new(UserBusinessAnalysisAll),
 		new(UserLoginLog),
 	)
 

models/user_business_analysis.go

@@ -4,13 +4,80 @@ import (
 	"encoding/json"
 	"fmt"
 	"sort"
+	"strconv"
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/timeutil"
 	"xorm.io/builder"
+	"xorm.io/xorm"
 )
 
+type UserBusinessAnalysisAll struct {
+	ID int64 `xorm:"pk"`
+
+	CountDate int64 `xorm:"pk"`
+
+	//action :ActionMergePullRequest                         // 11
+	CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//action :ActionCommitRepo                               // 5
+	CommitCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//action :ActionCreateIssue                             // 10
+	IssueCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//comment table current date
+	CommentCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//watch table current date
+	FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//star table current date
+	StarRepoCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//follow table
+	WatchedCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	// user table
+	GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"`
+
+	//
+	CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"`
+
+	//attachement table
+	CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"`
+
+	//0
+	CommitModelCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//issue, issueassignees
+	SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//baike
+	EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//user
+	RegistDate timeutil.TimeStamp `xorm:"NOT NULL"`
+
+	//repo
+	CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//login count, from elk
+	LoginCount int `xorm:"NOT NULL DEFAULT 0"`
+
+	//openi index
+	OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"`
+
+	//user
+	Email string `xorm:"NOT NULL"`
+
+	//user
+	Name string `xorm:"NOT NULL"`
+
+	DataDate string `xorm:"NULL"`
+}
+
 type UserBusinessAnalysis struct {
 	ID int64 `xorm:"pk"`
 
@@ -93,46 +160,12 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool {
 	return ulist[i].ID > ulist[j].ID
 }
 
-func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis {
-	log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime))
-	statictisSess := xStatistic.NewSession()
-	defer statictisSess.Close()
-
-	statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc")
-
-	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
-	statictisSess.Find(&userBusinessAnalysisList)
-
-	resultMap := make(map[int64]*UserBusinessAnalysis)
-	log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
-	for _, userRecord := range userBusinessAnalysisList {
-		if _, ok := resultMap[userRecord.ID]; !ok {
-			resultMap[userRecord.ID] = userRecord
-		} else {
-			resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
-			resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
-			resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
-			resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
-			resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
-			resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
-			resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
-			resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
-			resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
-			resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
-			resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
-			resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
-			resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
-			resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
-		}
-	}
+type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll
 
-	userBusinessAnalysisReturnList := UserBusinessAnalysisList{}
-	for _, v := range resultMap {
-		userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v)
-	}
-	sort.Sort(userBusinessAnalysisReturnList)
-	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
-	return userBusinessAnalysisReturnList
+func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] }
+func (ulist UserBusinessAnalysisAllList) Len() int      { return len(ulist) }
+func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool {
+	return ulist[i].ID > ulist[j].ID
 }
 
 func getLastCountDate() int64 {
@@ -153,71 +186,36 @@ func getLastCountDate() int64 {
 	return pageStartTime.Unix()
 }
 
-func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) {
+func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) {
 	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll))
 
 	statictisSess := xStatistic.NewSession()
 	defer statictisSess.Close()
 
-	resultMap := make(map[int64]*UserBusinessAnalysis)
-
-	var newAndCond = builder.NewCond()
-	if len(opts.UserName) > 0 {
-		newAndCond = newAndCond.And(
-			builder.Like{"name", opts.UserName},
-		)
-	}
-	if !opts.IsAll {
-		newAndCond = newAndCond.And(
-			builder.Gte{"count_date": opts.StartTime},
-		)
-		newAndCond = newAndCond.And(
-			builder.Lte{"count_date": opts.EndTime},
-		)
-	}
-
-	allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis))
+	allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll))
 	if err != nil {
 		log.Info("query error." + err.Error())
 		return nil, 0
 	}
 	log.Info("query return total:" + fmt.Sprint(allCount))
-	pageSize := 200
+	if allCount == 0 {
+		RefreshUserStaticAllTabel()
+	}
+	pageSize := 1000
 	totalPage := int(allCount) / pageSize
-
+	userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{}
 	for i := 0; i <= int(totalPage); i++ {
-		userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
-		if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize).
-			Find(&userBusinessAnalysisList); err != nil {
+		userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0)
+		if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize).
+			Find(&userBusinessAnalysisAllList); err != nil {
 			return nil, 0
 		}
-		log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
-		for _, userRecord := range userBusinessAnalysisList {
-			if _, ok := resultMap[userRecord.ID]; !ok {
-				resultMap[userRecord.ID] = userRecord
-			} else {
-				resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
-				resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
-				resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
-				resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
-				resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
-				resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
-				resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
-				resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
-				resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
-				resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
-				resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
-				resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
-				resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
-				resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
-			}
+		log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList)))
+		for _, userRecord := range userBusinessAnalysisAllList {
+			userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord)
 		}
 	}
 
-	userBusinessAnalysisReturnList := UserBusinessAnalysisList{}
-	for _, v := range resultMap {
-		userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v)
-	}
 	sort.Sort(userBusinessAnalysisReturnList)
 	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
 	return userBusinessAnalysisReturnList, allCount
@@ -338,6 +336,50 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
 	return userBusinessAnalysisReturnList, count
 }
 
+func RefreshUserStaticAllTabel() {
+
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+	log.Info("delete all data from table: user_business_analysis_all")
+	statictisSess.Exec("delete from user_business_analysis_all")
+
+	currentTimeNow := time.Now()
+	pageStartTime := getLastCountDate()
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix()
+
+	var cond = builder.NewCond()
+	cond = cond.And(
+		builder.Gte{"count_date": pageStartTime},
+	)
+	cond = cond.And(
+		builder.Lte{"count_date": pageEndTime},
+	)
+	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
+	if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc").
+		Find(&userBusinessAnalysisList); err != nil {
+		return
+	}
+	log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList)))
+	for _, userRecord := range userBusinessAnalysisList {
+		log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID))
+		allData := getAllData(userRecord.ID, statictisSess)
+		allData.ID = userRecord.ID
+		allData.CountDate = 0
+		allData.DataDate = userRecord.DataDate
+		allData.Email = userRecord.Email
+		allData.OpenIIndex = userRecord.OpenIIndex
+		allData.GiteaAgeMonth = userRecord.GiteaAgeMonth
+		allData.Name = userRecord.Name
+		allData.RegistDate = userRecord.RegistDate
+
+		_, err := statictisSess.Insert(&allData)
+		if err != nil {
+			log.Info("insert all data failed." + err.Error())
+		}
+	}
+	log.Info("refresh all data finished.")
+}
+
 func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error {
 
 	log.Info("start to count other user info data")
@@ -491,10 +533,99 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 			log.Info("insert daterecord failed." + err.Error())
 			return err
 		}
+
+		if isExistUserInAllTable(dateRecord.ID, statictisSess) {
+			updateCurrentData(dateRecord.ID, statictisSess, dateRecord)
+		} else {
+			log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID))
+			allData := getAllData(dateRecord.ID, statictisSess)
+			allData.ID = dateRecord.ID
+			allData.CountDate = 0
+			allData.DataDate = dateRecord.DataDate
+			allData.Email = dateRecord.Email
+			allData.OpenIIndex = dateRecord.OpenIIndex
+			allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth
+			allData.Name = dateRecord.Name
+			allData.RegistDate = dateRecord.RegistDate
+
+			_, err = statictisSess.Insert(&allData)
+			if err != nil {
+				log.Info("insert all data failed." + err.Error())
+				return err
+			}
+		}
+
 	}
 	return nil
 }
 
+func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) {
+
+	_, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) +
+		",commit_count+=" + fmt.Sprint(currentData.CommitCount) +
+		",issue_count+=" + fmt.Sprint(currentData.IssueCount) +
+		",comment_count+=" + fmt.Sprint(currentData.CommentCount) +
+		",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) +
+		",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) +
+		",watched_count+=" + fmt.Sprint(currentData.WatchedCount) +
+		",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) +
+		",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) +
+		",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) +
+		",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) +
+		",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) +
+		",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) +
+		",login_count+=" + fmt.Sprint(currentData.LoginCount) +
+		" where id=" + fmt.Sprint(userId))
+
+	if err != nil {
+		log.Info("update table failed." + err.Error())
+	}
+
+}
+
+func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool {
+
+	allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll))
+	if err != nil {
+		return false
+	}
+	return allCount > 0
+}
+
+func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll {
+	var dateRecord UserBusinessAnalysisAll
+
+	rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id")
+	if err == nil {
+		for i, row := range rows {
+			log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"]))
+			dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"]))
+			dateRecord.CommitCount = getInt(string(row["commit_count"]))
+			dateRecord.IssueCount = getInt(string(row["issue_count"]))
+			dateRecord.CommentCount = getInt(string(row["comment_count"]))
+			dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"]))
+			dateRecord.StarRepoCount = getInt(string(row["star_repo_count"]))
+			dateRecord.WatchedCount = getInt(string(row["watched_count"]))
+			dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"]))
+			dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"]))
+			dateRecord.CommitModelCount = getInt(string(row["commit_model_count"]))
+			dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"]))
+			dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"]))
+			dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"]))
+			dateRecord.LoginCount = getInt(string(row["login_count"]))
+		}
+	}
+	return dateRecord
+}
+
+func getInt(str string) int {
+	re, err := strconv.ParseInt(str, 10, 32)
+	if err != nil {
+		return 0
+	}
+	return int(re)
+}
+
 func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
 	CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false)
 }

routers/repo/user_data_analysis.go

@@ -16,17 +16,6 @@ import (
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
 )
 
-func QueryUserStaticData(ctx *context.Context) {
-	startDate := ctx.Query("startDate")
-	endDate := ctx.Query("endDate")
-	log.Info("startDate=" + startDate + "  endDate=" + endDate)
-	startTime, _ := time.Parse("2006-01-02", startDate)
-	endTime, _ := time.Parse("2006-01-02", endDate)
-	endTime = endTime.AddDate(0, 0, 1)
-	log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
-	ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix()))
-}
-
 func QueryUserStaticDataPage(ctx *context.Context) {
 	startDate := ctx.Query("startDate")
 	endDate := ctx.Query("endDate")
@@ -131,8 +120,8 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 			formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
 			xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3])
 
-			formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05")
-			xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3])
+			formatTime = userRecord.DataDate
+			xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01")
 		}
 
 		//设置默认打开的表单
@@ -171,6 +160,12 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 }
 
 func TimingCountDataByDateAndReCount(date string, isReCount bool) {
+
+	if date == "refreshAll" {
+		models.RefreshUserStaticAllTabel()
+		return
+	}
+
 	t, _ := time.Parse("2006-01-02", date)
 	startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
 
@@ -223,6 +218,10 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) {
 		log.Error("count user info error." + err.Error())
 		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
 	}
+
+	if isReCount {
+		models.RefreshUserStaticAllTabel()
+	}
 }
 
 func TimingCountDataByDate(date string) {

routers/routes/routes.go

@@ -792,7 +792,6 @@ func RegisterRoutes(m *macaron.Macaron) {
 	}, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef())
 
 	m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action)
-	m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData)
 	m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage)
 	// Grouping for those endpoints not requiring authentication
 	m.Group("/:username/:reponame", func() {