diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index cf0065282..9b9d5e0ad 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -7,10 +7,14 @@ import ( "strconv" "time" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" - "xorm.io/xorm" +) + +const ( + Page_SIZE = 2000 ) type UserBusinessAnalysisAll struct { @@ -199,7 +203,13 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } log.Info("query return total:" + fmt.Sprint(allCount)) if allCount == 0 { - RefreshUserStaticAllTabel() + CommitCodeSizeMap, err := GetAllUserKPIStats() + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + RefreshUserStaticAllTabel(make(map[string]int), CommitCodeSizeMap) } pageSize := 1000 totalPage := int(allCount) / pageSize @@ -336,47 +346,164 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func RefreshUserStaticAllTabel() { +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { + + sess := x.NewSession() + defer sess.Close() statictisSess := xStatistic.NewSession() defer statictisSess.Close() - log.Info("delete all data from table: user_business_analysis_all") - statictisSess.Exec("delete from user_business_analysis_all") + log.Info("truncate all data from table: user_business_analysis_all") + statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all") currentTimeNow := time.Now() - pageStartTime := getLastCountDate() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() - var cond = builder.NewCond() - cond = cond.And( - builder.Gte{"count_date": pageStartTime}, - ) - cond = cond.And( - builder.Lte{"count_date": pageEndTime}, - ) - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). - Find(&userBusinessAnalysisList); err != nil { + startTime := currentTimeNow.AddDate(0, 0, -1) + + pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) + log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) + + start_unix := pageStartTime.Unix() + end_unix := pageEndTime.Unix() + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryAction(start_unix, end_unix, 6) + + CommentCountMap := queryComment(start_unix, end_unix) + FocusRepoCountMap := queryWatch(start_unix, end_unix) + StarRepoCountMap := queryStar(start_unix, end_unix) + WatchedCountMap := queryFollow(start_unix, end_unix) + + CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + + OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) + + DataDate := currentTimeNow.Format("2006-01-02") + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") return } - log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) - allData := getAllData(userRecord.ID, statictisSess) - allData.ID = userRecord.ID - allData.CountDate = 0 - allData.DataDate = userRecord.DataDate - allData.Email = userRecord.Email - allData.OpenIIndex = userRecord.OpenIIndex - allData.GiteaAgeMonth = userRecord.GiteaAgeMonth - allData.Name = userRecord.Name - allData.RegistDate = userRecord.RegistDate - - _, err := statictisSess.Insert(&allData) - if err != nil { - log.Info("insert all data failed." + err.Error()) + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + for i, userRecord := range userList { + log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + var dateRecordAll UserBusinessAnalysisAll + dateRecordAll.ID = userRecord.ID + dateRecordAll.Email = userRecord.Email + dateRecordAll.RegistDate = userRecord.CreatedUnix + dateRecordAll.Name = userRecord.Name + dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecordAll.DataDate = DataDate + + if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CodeMergeCount = 0 + } else { + dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitCount = 0 + } else { + dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID] + } + + if _, ok := IssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.IssueCount = 0 + } else { + dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID] + } + + if _, ok := CommentCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommentCount = 0 + } else { + dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID] + } + + if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.FocusRepoCount = 0 + } else { + dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID] + } + + if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.StarRepoCount = 0 + } else { + dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID] + } + + if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.WatchedCount = 0 + } else { + dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + + if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitDatasetSize = 0 + } else { + dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID] + } + + if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.SolveIssueCount = 0 + } else { + dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID] + } + + if _, ok := wikiCountMap[dateRecordAll.Name]; !ok { + dateRecordAll.EncyclopediasCount = 0 + } else { + dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name] + } + + if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CreateRepoCount = 0 + } else { + dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID] + } + + if _, ok := LoginCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.LoginCount = 0 + } else { + dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID] + } + + if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok { + dateRecordAll.OpenIIndex = 0 + } else { + dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID] + } + + dateRecordAll.CommitModelCount = 0 + _, err = statictisSess.Insert(&dateRecordAll) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + log.Info("refresh all data finished.") } @@ -385,19 +512,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") - userList := make([]*User, 0) - sess.Find(&userList) currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - //yesterday := currentTimeNow.AddDate(0, 0, -1) - //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) if isReCount { @@ -429,193 +550,134 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, statictisSess := xStatistic.NewSession() defer statictisSess.Close() - for i, userRecord := range userList { - var dateRecord UserBusinessAnalysis - dateRecord.ID = userRecord.ID - log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) - dateRecord.CountDate = CountDate.Unix() - - statictisSess.Delete(&dateRecord) - - dateRecord.Email = userRecord.Email - dateRecord.RegistDate = userRecord.CreatedUnix - dateRecord.Name = userRecord.Name - dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) - dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") + return err + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysis + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate + if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { + dateRecord.CodeMergeCount = 0 + } else { + dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] + } - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } + if _, ok := CommitCountMap[dateRecord.ID]; !ok { + dateRecord.CommitCount = 0 + } else { + dateRecord.CommitCount = CommitCountMap[dateRecord.ID] + } - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } + if _, ok := IssueCountMap[dateRecord.ID]; !ok { + dateRecord.IssueCount = 0 + } else { + dateRecord.IssueCount = IssueCountMap[dateRecord.ID] + } - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } + if _, ok := CommentCountMap[dateRecord.ID]; !ok { + dateRecord.CommentCount = 0 + } else { + dateRecord.CommentCount = CommentCountMap[dateRecord.ID] + } - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } + if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { + dateRecord.FocusRepoCount = 0 + } else { + dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] + } - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } + if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { + dateRecord.StarRepoCount = 0 + } else { + dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] + } - if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { - dateRecord.CommitCodeSize = 0 - } else { - dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) - } + if _, ok := WatchedCountMap[dateRecord.ID]; !ok { + dateRecord.WatchedCount = 0 + } else { + dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] + } - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) + } - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } + if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { + dateRecord.CommitDatasetSize = 0 + } else { + dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] + } - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } + if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { + dateRecord.SolveIssueCount = 0 + } else { + dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] + } - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } + if _, ok := wikiCountMap[dateRecord.Name]; !ok { + dateRecord.EncyclopediasCount = 0 + } else { + dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] + } - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } + if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { + dateRecord.CreateRepoCount = 0 + } else { + dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] + } - if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { - dateRecord.OpenIIndex = 0 - } else { - dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] - } + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } - dateRecord.CommitModelCount = 0 + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] + } - _, err = statictisSess.Insert(&dateRecord) - if err != nil { - log.Info("insert daterecord failed." + err.Error()) - return err - } + dateRecord.CommitModelCount = 0 - if isExistUserInAllTable(dateRecord.ID, statictisSess) { - updateCurrentData(dateRecord.ID, statictisSess, dateRecord) - } else { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) - allData := getAllData(dateRecord.ID, statictisSess) - allData.ID = dateRecord.ID - allData.CountDate = 0 - allData.DataDate = dateRecord.DataDate - allData.Email = dateRecord.Email - allData.OpenIIndex = dateRecord.OpenIIndex - allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth - allData.Name = dateRecord.Name - allData.RegistDate = dateRecord.RegistDate - - _, err = statictisSess.Insert(&allData) + _, err = statictisSess.Insert(&dateRecord) if err != nil { - log.Info("insert all data failed." + err.Error()) + log.Info("insert daterecord failed." + err.Error()) return err } } - } - return nil -} - -func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { - - _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + - ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + - ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + - ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + - ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + - ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + - ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + - ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + - ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + - ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + - ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + - ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + - ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + - ",login_count+=" + fmt.Sprint(currentData.LoginCount) + - " where id=" + fmt.Sprint(userId)) - - if err != nil { - log.Info("update table failed." + err.Error()) + indexTotal += Page_SIZE + if indexTotal >= count { + break + } } -} - -func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { - - allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) - if err != nil { - return false - } - return allCount > 0 -} + RefreshUserStaticAllTabel(wikiCountMap, CommitCodeSizeMap) -func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { - var dateRecord UserBusinessAnalysisAll - - rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") - if err == nil { - for i, row := range rows { - log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) - dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) - dateRecord.CommitCount = getInt(string(row["commit_count"])) - dateRecord.IssueCount = getInt(string(row["issue_count"])) - dateRecord.CommentCount = getInt(string(row["comment_count"])) - dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) - dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) - dateRecord.WatchedCount = getInt(string(row["watched_count"])) - dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) - dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) - dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) - dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) - dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) - dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) - dateRecord.LoginCount = getInt(string(row["login_count"])) - } - } - return dateRecord + return nil } func getInt(str string) int { @@ -631,43 +693,70 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { - //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() defer sess.Close() - sess.Select("issue_assignees.*").Table("issue_assignees"). - Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) - issueAssigneesList := make([]*IssueAssignees, 0) - sess.Find(&issueAssigneesList) resultMap := make(map[int64]int) - log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) - for _, issueAssigneesRecord := range issueAssigneesList { - if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { - resultMap[issueAssigneesRecord.AssigneeID] = 1 - } else { - resultMap[issueAssigneesRecord.AssigneeID] += 1 + cond := "issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Table("issue_assignees").Join("inner", "issue", "issue.id=issue_assignees.issue_id").Where(cond).Count(new(IssueAssignees)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueAssigneesList := make([]*IssueAssignees, 0) + sess.Select("issue_assignees.*").Table("issue_assignees"). + Join("inner", "issue", "issue.id=issue_assignees.issue_id"). + Where(cond).Limit(Page_SIZE, int(indexTotal)) + + sess.Find(&issueAssigneesList) + + log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) + for _, issueAssigneesRecord := range issueAssigneesList { + if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { + resultMap[issueAssigneesRecord.AssigneeID] = 1 + } else { + resultMap[issueAssigneesRecord.AssigneeID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - - sess.Select("issue.*").Table("issue"). - Join("inner", "pull_request", "issue.id=pull_request.issue_id"). - Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) - - issueList := make([]*Issue, 0) - sess.Find(&issueList) resultMap := make(map[int64]int) - log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) - for _, issueRecord := range issueList { - if _, ok := resultMap[issueRecord.PosterID]; !ok { - resultMap[issueRecord.PosterID] = 1 - } else { - resultMap[issueRecord.PosterID] += 1 + cond := "pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Count(new(Issue)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueList := make([]*Issue, 0) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&issueList) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -676,34 +765,68 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + + cond := "user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + cond := "op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query Action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -713,16 +836,30 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - commentList := make([]*Comment, 0) - sess.Find(&commentList) + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Comment size=" + fmt.Sprint(len(commentList))) - for _, commentRecord := range commentList { - if _, ok := resultMap[commentRecord.PosterID]; !ok { - resultMap[commentRecord.PosterID] = 1 - } else { - resultMap[commentRecord.PosterID] += 1 + count, err := sess.Where(cond).Count(new(Comment)) + if err != nil { + log.Info("query Comment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,type,poster_id").Table("comment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + commentList := make([]*Comment, 0) + sess.Find(&commentList) + log.Info("query Comment size=" + fmt.Sprint(len(commentList))) + for _, commentRecord := range commentList { + if _, ok := resultMap[commentRecord.PosterID]; !ok { + resultMap[commentRecord.PosterID] = 1 + } else { + resultMap[commentRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -732,18 +869,37 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - watchList := make([]*Watch, 0) - sess.Find(&watchList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + resultMap := make(map[int64]int) - log.Info("query Watch size=" + fmt.Sprint(len(watchList))) - for _, watchRecord := range watchList { - if _, ok := resultMap[watchRecord.UserID]; !ok { - resultMap[watchRecord.UserID] = 1 - } else { - resultMap[watchRecord.UserID] += 1 + count, err := sess.Where(cond).Count(new(Watch)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + watchList := make([]*Watch, 0) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&watchList) + + log.Info("query Watch size=" + fmt.Sprint(len(watchList))) + for _, watchRecord := range watchList { + if _, ok := resultMap[watchRecord.UserID]; !ok { + resultMap[watchRecord.UserID] = 1 + } else { + resultMap[watchRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } @@ -752,82 +908,151 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - starList := make([]*Star, 0) - sess.Find(&starList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Star size=" + fmt.Sprint(len(starList))) - for _, starRecord := range starList { - if _, ok := resultMap[starRecord.UID]; !ok { - resultMap[starRecord.UID] = 1 - } else { - resultMap[starRecord.UID] += 1 + + count, err := sess.Where(cond).Count(new(Star)) + if err != nil { + log.Info("query star error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uid,repo_id").Table("star").Where(cond).Limit(Page_SIZE, int(indexTotal)) + starList := make([]*Star, 0) + sess.Find(&starList) + + log.Info("query Star size=" + fmt.Sprint(len(starList))) + for _, starRecord := range starList { + if _, ok := resultMap[starRecord.UID]; !ok { + resultMap[starRecord.UID] = 1 + } else { + resultMap[starRecord.UID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap - } func queryFollow(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - followList := make([]*Follow, 0) - sess.Find(&followList) resultMap := make(map[int64]int) - log.Info("query Follow size=" + fmt.Sprint(len(followList))) - for _, followRecord := range followList { - if _, ok := resultMap[followRecord.FollowID]; !ok { - resultMap[followRecord.FollowID] = 1 - } else { - resultMap[followRecord.FollowID] += 1 + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Follow)) + if err != nil { + log.Info("query follow error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).Limit(Page_SIZE, int(indexTotal)) + followList := make([]*Follow, 0) + sess.Find(&followList) + + log.Info("query Follow size=" + fmt.Sprint(len(followList))) + for _, followRecord := range followList { + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 + } else { + resultMap[followRecord.FollowID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - attachmentList := make([]*Attachment, 0) - sess.Find(&attachmentList) resultMap := make(map[int64]int) - log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) - for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB - } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Attachment)) + if err != nil { + log.Info("query attachment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + attachmentList := make([]*Attachment, 0) + sess.Find(&attachmentList) + + log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) + for _, attachRecord := range attachmentList { + if _, ok := resultMap[attachRecord.UploaderID]; !ok { + resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + } else { + resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } - return resultMap + return resultMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - repoList := make([]*Repository, 0) - sess.Find(&repoList) resultMap := make(map[int64]int) - log.Info("query Repository size=" + fmt.Sprint(len(repoList))) - for _, repoRecord := range repoList { - if _, ok := resultMap[repoRecord.OwnerID]; !ok { - resultMap[repoRecord.OwnerID] = 1 - } else { - resultMap[repoRecord.OwnerID] += 1 + + cond := "is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Repository)) + if err != nil { + log.Info("query Repository error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,owner_id,name").Table("repository").Where(cond).Limit(Page_SIZE, int(indexTotal)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := resultMap[repoRecord.OwnerID]; !ok { + resultMap[repoRecord.OwnerID] = 1 + } else { + resultMap[repoRecord.OwnerID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + + statictisSess.Select("id,repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)).OrderBy("id desc") repoStatisticList := make([]*RepoStatistic, 0) statictisSess.Find(&repoStatisticList) repoOpenIIndexMap := make(map[int64]float64) @@ -883,18 +1108,34 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - userLoginLogList := make([]*UserLoginLog, 0) - statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) - log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) - for _, loginRecord := range userLoginLogList { - if _, ok := resultMap[loginRecord.UId]; !ok { - resultMap[loginRecord.UId] = 1 - } else { - resultMap[loginRecord.UId] += 1 + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := statictisSess.Where(cond).Count(new(UserLoginLog)) + if err != nil { + log.Info("query UserLoginLog error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2f37dc15e..42189c57f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -148,11 +148,6 @@ func QueryUserStaticDataPage(ctx *context.Context) { func TimingCountDataByDateAndReCount(date string, isReCount bool) { - if date == "refreshAll" { - models.RefreshUserStaticAllTabel() - return - } - t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -205,10 +200,9 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } - - if isReCount { - models.RefreshUserStaticAllTabel() - } + log.Info("start to count all user info data") + //models.RefreshUserStaticAllTabel(wikiMap) + log.Info("end to count all user info data") } func TimingCountDataByDate(date string) {