diff --git a/models/models.go b/models/models.go index 4d39e2f14..809f9b0ca 100755 --- a/models/models.go +++ b/models/models.go @@ -139,6 +139,7 @@ func init() { new(RepoStatistic), new(SummaryStatistic), new(UserBusinessAnalysis), + new(UserBusinessAnalysisAll), new(UserLoginLog), ) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 08be81241..f81683242 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -4,13 +4,80 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) +type UserBusinessAnalysisAll struct { + ID int64 `xorm:"pk"` + + CountDate int64 `xorm:"pk"` + + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCommitRepo // 5 + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCreateIssue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + + //user + Email string `xorm:"NOT NULL"` + + //user + Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NULL"` +} + type UserBusinessAnalysis struct { ID int64 `xorm:"pk"` @@ -93,46 +160,12 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool { return ulist[i].ID > ulist[j].ID } -func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { - log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") - - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - statictisSess.Find(&userBusinessAnalysisList) - - resultMap := make(map[int64]*UserBusinessAnalysis) - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } - } +type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList +func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisAllList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool { + return ulist[i].ID > ulist[j].ID } func getLastCountDate() int64 { @@ -153,71 +186,36 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } -func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { +func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) statictisSess := xStatistic.NewSession() defer statictisSess.Close() - resultMap := make(map[int64]*UserBusinessAnalysis) - - var newAndCond = builder.NewCond() - if len(opts.UserName) > 0 { - newAndCond = newAndCond.And( - builder.Like{"name", opts.UserName}, - ) - } - if !opts.IsAll { - newAndCond = newAndCond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - newAndCond = newAndCond.And( - builder.Lte{"count_date": opts.EndTime}, - ) - } - - allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll)) if err != nil { log.Info("query error." + err.Error()) return nil, 0 } log.Info("query return total:" + fmt.Sprint(allCount)) - pageSize := 200 + if allCount == 0 { + RefreshUserStaticAllTabel() + } + pageSize := 1000 totalPage := int(allCount) / pageSize - + userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} for i := 0; i <= int(totalPage); i++ { - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). - Find(&userBusinessAnalysisList); err != nil { + userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) + if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisAllList); err != nil { return nil, 0 } - log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } + log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList))) + for _, userRecord := range userBusinessAnalysisAllList { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord) } } - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, allCount @@ -338,6 +336,50 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } +func RefreshUserStaticAllTabel() { + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + log.Info("delete all data from table: user_business_analysis_all") + statictisSess.Exec("delete from user_business_analysis_all") + + currentTimeNow := time.Now() + pageStartTime := getLastCountDate() + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() + + var cond = builder.NewCond() + cond = cond.And( + builder.Gte{"count_date": pageStartTime}, + ) + cond = cond.And( + builder.Lte{"count_date": pageEndTime}, + ) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). + Find(&userBusinessAnalysisList); err != nil { + return + } + log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) + allData := getAllData(userRecord.ID, statictisSess) + allData.ID = userRecord.ID + allData.CountDate = 0 + allData.DataDate = userRecord.DataDate + allData.Email = userRecord.Email + allData.OpenIIndex = userRecord.OpenIIndex + allData.GiteaAgeMonth = userRecord.GiteaAgeMonth + allData.Name = userRecord.Name + allData.RegistDate = userRecord.RegistDate + + _, err := statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + log.Info("refresh all data finished.") +} + func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { log.Info("start to count other user info data") @@ -491,10 +533,99 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("insert daterecord failed." + err.Error()) return err } + + if isExistUserInAllTable(dateRecord.ID, statictisSess) { + updateCurrentData(dateRecord.ID, statictisSess, dateRecord) + } else { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) + allData := getAllData(dateRecord.ID, statictisSess) + allData.ID = dateRecord.ID + allData.CountDate = 0 + allData.DataDate = dateRecord.DataDate + allData.Email = dateRecord.Email + allData.OpenIIndex = dateRecord.OpenIIndex + allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth + allData.Name = dateRecord.Name + allData.RegistDate = dateRecord.RegistDate + + _, err = statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + return err + } + } + } return nil } +func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { + + _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + + ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + + ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + + ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + + ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + + ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + + ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + + ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + + ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + + ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + + ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + + ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + + ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + + ",login_count+=" + fmt.Sprint(currentData.LoginCount) + + " where id=" + fmt.Sprint(userId)) + + if err != nil { + log.Info("update table failed." + err.Error()) + } + +} + +func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { + + allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) + if err != nil { + return false + } + return allCount > 0 +} + +func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { + var dateRecord UserBusinessAnalysisAll + + rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") + if err == nil { + for i, row := range rows { + log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) + dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) + dateRecord.CommitCount = getInt(string(row["commit_count"])) + dateRecord.IssueCount = getInt(string(row["issue_count"])) + dateRecord.CommentCount = getInt(string(row["comment_count"])) + dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) + dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) + dateRecord.WatchedCount = getInt(string(row["watched_count"])) + dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) + dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) + dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) + dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) + dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) + dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) + dateRecord.LoginCount = getInt(string(row["login_count"])) + } + } + return dateRecord +} + +func getInt(str string) int { + re, err := strconv.ParseInt(str, 10, 32) + if err != nil { + return 0 + } + return int(re) +} + func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 5a6b3d724..a141df086 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -16,17 +16,6 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) -func QueryUserStaticData(ctx *context.Context) { - startDate := ctx.Query("startDate") - endDate := ctx.Query("endDate") - log.Info("startDate=" + startDate + " endDate=" + endDate) - startTime, _ := time.Parse("2006-01-02", startDate) - endTime, _ := time.Parse("2006-01-02", endDate) - endTime = endTime.AddDate(0, 0, 1) - log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) - ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) -} - func QueryUserStaticDataPage(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") @@ -131,8 +120,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) - formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") - xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) + formatTime = userRecord.DataDate + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01") } //设置默认打开的表单 @@ -171,6 +160,12 @@ func QueryUserStaticDataPage(ctx *context.Context) { } func TimingCountDataByDateAndReCount(date string, isReCount bool) { + + if date == "refreshAll" { + models.RefreshUserStaticAllTabel() + return + } + t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -223,6 +218,10 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } + + if isReCount { + models.RefreshUserStaticAllTabel() + } } func TimingCountDataByDate(date string) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index fe2588b25..d3212690c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,7 +792,6 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData) m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() {