package models import ( "fmt" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" ) type UserBusinessAnalysis struct { ID int64 `xorm:"pk"` CountDate int64 `xorm:"pk"` //action :ActionMergePullRequest // 11 CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` //action :ActionCommitRepo // 5 CommitCount int `xorm:"NOT NULL DEFAULT 0"` //action :ActionCommentIssue // 10 IssueCount int `xorm:"NOT NULL DEFAULT 0"` //comment table current date CommentCount int `xorm:"NOT NULL DEFAULT 0"` //watch table current date FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` //star table current date StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` //follow table WatchedCount int `xorm:"NOT NULL DEFAULT 0"` // user table GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` // CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` //attachement table CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` //0 CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` //issue, issueassignees SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` //baike EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` //user RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` //repo CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` //login count, from elk LoginCount int `xorm:"NOT NULL DEFAULT 0"` //openi index OpenIIndex int `xorm:"NOT NULL DEFAULT 0"` //user Email string `xorm:"NOT NULL"` //user Name string `xorm:"NOT NULL"` } func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) statictisSess := xStatistic.NewSession() defer statictisSess.Close() statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) statictisSess.Find(&userBusinessAnalysisList) resultMap := make(map[int64]*UserBusinessAnalysis) log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) for _, userRecord := range userBusinessAnalysisList { if _, ok := resultMap[userRecord.ID]; !ok { resultMap[userRecord.ID] = userRecord } else { resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount resultMap[userRecord.ID].CommitCount += userRecord.CommitCount resultMap[userRecord.ID].IssueCount += userRecord.IssueCount resultMap[userRecord.ID].CommentCount += userRecord.CommentCount resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount resultMap[userRecord.ID].LoginCount += userRecord.LoginCount } } userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) index := 0 for _, v := range resultMap { userBusinessAnalysisReturnList[index] = v index += 1 } log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList } func CountData(wikiCountMap map[string]int) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() sess.Select("`user`.*").Table("user") userList := make([]*User, 0) sess.Find(&userList) currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) yesterday := currentTimeNow.AddDate(0, 0, -1) startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryAction(start_unix, end_unix, 5) IssueCountMap := queryAction(start_unix, end_unix, 10) CommentCountMap := queryComment(start_unix, end_unix) FocusRepoCountMap := queryWatch(start_unix, end_unix) StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap := queryFollow(start_unix, end_unix) CommitCodeSizeMap, err := GetAllUserKPIStats() if err != nil { log.Info("query commit code errr.") } else { log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) } CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) for i, userRecord := range userList { var dateRecord UserBusinessAnalysis dateRecord.ID = userRecord.ID log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) dateRecord.CountDate = CountDate.Unix() dateRecord.Email = userRecord.Email dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { dateRecord.CodeMergeCount = 0 } else { dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] } if _, ok := CommitCountMap[dateRecord.ID]; !ok { dateRecord.CommitCount = 0 } else { dateRecord.CommitCount = CommitCountMap[dateRecord.ID] } if _, ok := IssueCountMap[dateRecord.ID]; !ok { dateRecord.IssueCount = 0 } else { dateRecord.IssueCount = IssueCountMap[dateRecord.ID] } if _, ok := CommentCountMap[dateRecord.ID]; !ok { dateRecord.CommentCount = 0 } else { dateRecord.CommentCount = CommentCountMap[dateRecord.ID] } if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { dateRecord.FocusRepoCount = 0 } else { dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] } if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { dateRecord.StarRepoCount = 0 } else { dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] } if _, ok := WatchedCountMap[dateRecord.ID]; !ok { dateRecord.WatchedCount = 0 } else { dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] } if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { dateRecord.CommitCodeSize = 0 } else { dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) } if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { dateRecord.CommitDatasetSize = 0 } else { dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] } if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { dateRecord.SolveIssueCount = 0 } else { dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] } if _, ok := wikiCountMap[dateRecord.Name]; !ok { dateRecord.EncyclopediasCount = 0 } else { dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] } if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { dateRecord.CreateRepoCount = 0 } else { dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] } if _, ok := LoginCountMap[dateRecord.ID]; !ok { dateRecord.LoginCount = 0 } else { dateRecord.LoginCount = LoginCountMap[dateRecord.ID] } dateRecord.CommitModelCount = 0 statictisSess := xStatistic.NewSession() defer statictisSess.Close() statictisSess.Insert(&dateRecord) } } func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() defer sess.Close() sess.Select("issue_assignees.*").Table("issue_assignees"). Join("inner", "issue", "issue.id=issue_assignees.issue_id"). Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) issueAssigneesList := make([]*IssueAssignees, 0) sess.Find(&issueAssigneesList) resultMap := make(map[int64]int) log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) for _, issueAssigneesRecord := range issueAssigneesList { if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { resultMap[issueAssigneesRecord.AssigneeID] = 1 } else { resultMap[issueAssigneesRecord.AssigneeID] += 1 } } return resultMap } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("issue.*").Table("issue"). Join("inner", "pull_request", "issue.id=pull_request.issue_id"). Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) issueList := make([]*Issue, 0) sess.Find(&issueList) resultMap := make(map[int64]int) log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) for _, issueRecord := range issueList { if _, ok := resultMap[issueRecord.PosterID]; !ok { resultMap[issueRecord.PosterID] = 1 } else { resultMap[issueRecord.PosterID] += 1 } } return resultMap } func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) actionList := make([]*Action, 0) sess.Find(&actionList) resultMap := make(map[int64]int) log.Info("query action size=" + fmt.Sprint(len(actionList))) for _, actionRecord := range actionList { if _, ok := resultMap[actionRecord.UserID]; !ok { resultMap[actionRecord.UserID] = 1 } else { resultMap[actionRecord.UserID] += 1 } } return resultMap } func queryComment(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) commentList := make([]*Comment, 0) sess.Find(&commentList) resultMap := make(map[int64]int) log.Info("query Comment size=" + fmt.Sprint(len(commentList))) for _, commentRecord := range commentList { if _, ok := resultMap[commentRecord.PosterID]; !ok { resultMap[commentRecord.PosterID] = 1 } else { resultMap[commentRecord.PosterID] += 1 } } return resultMap } func queryWatch(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) watchList := make([]*Watch, 0) sess.Find(&watchList) resultMap := make(map[int64]int) log.Info("query Watch size=" + fmt.Sprint(len(watchList))) for _, watchRecord := range watchList { if _, ok := resultMap[watchRecord.UserID]; !ok { resultMap[watchRecord.UserID] = 1 } else { resultMap[watchRecord.UserID] += 1 } } return resultMap } func queryStar(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) starList := make([]*Star, 0) sess.Find(&starList) resultMap := make(map[int64]int) log.Info("query Star size=" + fmt.Sprint(len(starList))) for _, starRecord := range starList { if _, ok := resultMap[starRecord.UID]; !ok { resultMap[starRecord.UID] = 1 } else { resultMap[starRecord.UID] += 1 } } return resultMap } func queryFollow(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) followList := make([]*Follow, 0) sess.Find(&followList) resultMap := make(map[int64]int) log.Info("query Follow size=" + fmt.Sprint(len(followList))) for _, followRecord := range followList { if _, ok := resultMap[followRecord.UserID]; !ok { resultMap[followRecord.UserID] = 1 } else { resultMap[followRecord.UserID] += 1 } } return resultMap } func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) attachmentList := make([]*Attachment, 0) sess.Find(&attachmentList) resultMap := make(map[int64]int) log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) for _, attachRecord := range attachmentList { if _, ok := resultMap[attachRecord.UploaderID]; !ok { resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB } else { resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB } } return resultMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) repoList := make([]*Repository, 0) sess.Find(&repoList) resultMap := make(map[int64]int) log.Info("query Repository size=" + fmt.Sprint(len(repoList))) for _, repoRecord := range repoList { if _, ok := resultMap[repoRecord.OwnerID]; !ok { resultMap[repoRecord.OwnerID] = 1 } else { resultMap[repoRecord.OwnerID] += 1 } } return resultMap } func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) userLoginLogList := make([]*UserLoginLog, 0) statictisSess.Find(&userLoginLogList) resultMap := make(map[int64]int) log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) for _, loginRecord := range userLoginLogList { if _, ok := resultMap[loginRecord.UId]; !ok { resultMap[loginRecord.UId] = 1 } else { resultMap[loginRecord.UId] += 1 } } return resultMap } func subMonth(t1, t2 time.Time) (month int) { y1 := t1.Year() y2 := t2.Year() m1 := int(t1.Month()) m2 := int(t2.Month()) d1 := t1.Day() d2 := t2.Day() yearInterval := y1 - y2 // 如果 d1的 月-日 小于 d2的 月-日 那么 yearInterval-- 这样就得到了相差的年数 if m1 < m2 || m1 == m2 && d1 < d2 { yearInterval-- } // 获取月数差值 monthInterval := (m1 + 12) - m2 if d1 < d2 { monthInterval-- } monthInterval %= 12 month = yearInterval*12 + monthInterval return month }