From 84e7032ab5fede47db5e46f0eddc5461ad6c31c7 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 11:54:53 +0800 Subject: [PATCH 01/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 696 ------------------------------------- routers/repo/user_data_analysis.go | 12 +- 2 files changed, 8 insertions(+), 700 deletions(-) delete mode 100644 models/user_business_analysis.go diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go deleted file mode 100644 index 502cec806..000000000 --- a/models/user_business_analysis.go +++ /dev/null @@ -1,696 +0,0 @@ -package models - -import ( - "encoding/json" - "fmt" - "sort" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/timeutil" - "xorm.io/builder" -) - -type UserBusinessAnalysis struct { - ID int64 `xorm:"pk"` - - CountDate int64 `xorm:"pk"` - - //action :ActionMergePullRequest // 11 - CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` - - //action :ActionCommitRepo // 5 - CommitCount int `xorm:"NOT NULL DEFAULT 0"` - - //action :ActionCreateIssue // 10 - IssueCount int `xorm:"NOT NULL DEFAULT 0"` - - //comment table current date - CommentCount int `xorm:"NOT NULL DEFAULT 0"` - - //watch table current date - FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` - - //star table current date - StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` - - //follow table - WatchedCount int `xorm:"NOT NULL DEFAULT 0"` - - // user table - GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` - - // - CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` - - //attachement table - CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` - - //0 - CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` - - //issue, issueassignees - SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` - - //baike - EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` - - //user - RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` - - //repo - CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` - - //login count, from elk - LoginCount int `xorm:"NOT NULL DEFAULT 0"` - - //openi index - OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` - - //user - Email string `xorm:"NOT NULL"` - - //user - Name string `xorm:"NOT NULL"` - - DataDate string `xorm:"NULL"` -} - -type UserBusinessAnalysisQueryOptions struct { - ListOptions - UserName string - SortType string - StartTime int64 - EndTime int64 - IsAll bool -} - -type UserBusinessAnalysisList []*UserBusinessAnalysis - -func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } -func (ulist UserBusinessAnalysisList) Len() int { return len(ulist) } -func (ulist UserBusinessAnalysisList) Less(i, j int) bool { - return ulist[i].ID > ulist[j].ID -} - -func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { - log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") - - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - statictisSess.Find(&userBusinessAnalysisList) - - resultMap := make(map[int64]*UserBusinessAnalysis) - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } - } - - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList -} - -func getLastCountDate() int64 { - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - statictisSess.Limit(1, 0) - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").OrderBy("count_date desc").Limit(1, 0). - Find(&userBusinessAnalysisList); err == nil { - for _, userRecord := range userBusinessAnalysisList { - return userRecord.CountDate - 10000 - } - } else { - log.Info("query error." + err.Error()) - } - currentTimeNow := time.Now() - pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) - return pageStartTime.Unix() -} - -func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { - - log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - currentTimeNow := time.Now() - pageStartTime := getLastCountDate() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() - - var cond = builder.NewCond() - if len(opts.UserName) > 0 { - cond = cond.And( - builder.Like{"name", opts.UserName}, - ) - } - cond = cond.And( - builder.Gte{"count_date": pageStartTime}, - ) - cond = cond.And( - builder.Lte{"count_date": pageEndTime}, - ) - - count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) - if err != nil { - log.Info("query error." + err.Error()) - return nil, 0 - } - - if opts.Page >= 0 && opts.PageSize > 0 { - var start int - if opts.Page == 0 { - start = 0 - } else { - start = (opts.Page - 1) * opts.PageSize - } - statictisSess.Limit(opts.PageSize, start) - } - - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). - Find(&userBusinessAnalysisList); err != nil { - return nil, 0 - } - - resultMap := make(map[int64]*UserBusinessAnalysis) - - if len(userBusinessAnalysisList) > 0 { - var newAndCond = builder.NewCond() - var newOrCond = builder.NewCond() - for _, userRecord := range userBusinessAnalysisList { - newOrCond = newOrCond.Or( - builder.Eq{"id": userRecord.ID}, - ) - } - newAndCond = newAndCond.And( - newOrCond, - ) - if !opts.IsAll { - newAndCond = newAndCond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - newAndCond = newAndCond.And( - builder.Lte{"count_date": opts.EndTime}, - ) - } - userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc"). - Find(&userBusinessAnalysisList); err != nil { - return nil, 0 - } - } - - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } - } - - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList, count -} - -func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { - - log.Info("start to count other user info data") - sess := x.NewSession() - defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") - userList := make([]*User, 0) - sess.Find(&userList) - - currentTimeNow := time.Now() - log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - - //yesterday := currentTimeNow.AddDate(0, 0, -1) - //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) - start_unix := startTime.Unix() - log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - - //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) - end_unix := endTime.Unix() - CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) - if isReCount { - CountDate = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 1, 0, 0, currentTimeNow.Location()) - } - - DataDate := startTime.Format("2006-01-02") - CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap := queryAction(start_unix, end_unix, 5) - IssueCountMap := queryAction(start_unix, end_unix, 6) - - CommentCountMap := queryComment(start_unix, end_unix) - FocusRepoCountMap := queryWatch(start_unix, end_unix) - StarRepoCountMap := queryStar(start_unix, end_unix) - WatchedCountMap := queryFollow(start_unix, end_unix) - - CommitCodeSizeMap, err := GetAllUserKPIStats() - if err != nil { - log.Info("query commit code errr.") - } else { - log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) - } - CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) - SolveIssueCountMap := querySolveIssue(start_unix, end_unix) - CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) - LoginCountMap := queryLoginCount(start_unix, end_unix) - OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) - - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - for i, userRecord := range userList { - var dateRecord UserBusinessAnalysis - dateRecord.ID = userRecord.ID - log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) - dateRecord.CountDate = CountDate.Unix() - - statictisSess.Delete(&dateRecord) - - dateRecord.Email = userRecord.Email - dateRecord.RegistDate = userRecord.CreatedUnix - dateRecord.Name = userRecord.Name - dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) - dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } - - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } - - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } - - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } - - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } - - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } - - if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { - dateRecord.CommitCodeSize = 0 - } else { - dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) - } - - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } - - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } - - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } - - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } - - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } - - if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { - dateRecord.OpenIIndex = 0 - } else { - dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] - } - - dateRecord.CommitModelCount = 0 - - _, err = statictisSess.Insert(&dateRecord) - if err != nil { - log.Info("insert daterecord failed." + err.Error()) - return err - } - } - return nil -} - -func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { - CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) -} - -func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { - //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id - sess := x.NewSession() - defer sess.Close() - sess.Select("issue_assignees.*").Table("issue_assignees"). - Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) - issueAssigneesList := make([]*IssueAssignees, 0) - sess.Find(&issueAssigneesList) - resultMap := make(map[int64]int) - log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) - for _, issueAssigneesRecord := range issueAssigneesList { - if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { - resultMap[issueAssigneesRecord.AssigneeID] = 1 - } else { - resultMap[issueAssigneesRecord.AssigneeID] += 1 - } - } - return resultMap - -} - -func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { - sess := x.NewSession() - defer sess.Close() - - sess.Select("issue.*").Table("issue"). - Join("inner", "pull_request", "issue.id=pull_request.issue_id"). - Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) - - issueList := make([]*Issue, 0) - sess.Find(&issueList) - resultMap := make(map[int64]int) - log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) - for _, issueRecord := range issueList { - if _, ok := resultMap[issueRecord.PosterID]; !ok { - resultMap[issueRecord.PosterID] = 1 - } else { - resultMap[issueRecord.PosterID] += 1 - } - } - return resultMap -} - -func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { - sess := x.NewSession() - defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) - resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 - } - } - return resultMap -} - -func queryComment(start_unix int64, end_unix int64) map[int64]int { - - sess := x.NewSession() - defer sess.Close() - sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - commentList := make([]*Comment, 0) - sess.Find(&commentList) - resultMap := make(map[int64]int) - log.Info("query Comment size=" + fmt.Sprint(len(commentList))) - for _, commentRecord := range commentList { - if _, ok := resultMap[commentRecord.PosterID]; !ok { - resultMap[commentRecord.PosterID] = 1 - } else { - resultMap[commentRecord.PosterID] += 1 - } - } - return resultMap -} - -func queryWatch(start_unix int64, end_unix int64) map[int64]int { - - sess := x.NewSession() - defer sess.Close() - sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - watchList := make([]*Watch, 0) - sess.Find(&watchList) - resultMap := make(map[int64]int) - log.Info("query Watch size=" + fmt.Sprint(len(watchList))) - for _, watchRecord := range watchList { - if _, ok := resultMap[watchRecord.UserID]; !ok { - resultMap[watchRecord.UserID] = 1 - } else { - resultMap[watchRecord.UserID] += 1 - } - } - return resultMap - -} - -func queryStar(start_unix int64, end_unix int64) map[int64]int { - - sess := x.NewSession() - defer sess.Close() - sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - starList := make([]*Star, 0) - sess.Find(&starList) - resultMap := make(map[int64]int) - log.Info("query Star size=" + fmt.Sprint(len(starList))) - for _, starRecord := range starList { - if _, ok := resultMap[starRecord.UID]; !ok { - resultMap[starRecord.UID] = 1 - } else { - resultMap[starRecord.UID] += 1 - } - } - return resultMap - -} - -func queryFollow(start_unix int64, end_unix int64) map[int64]int { - - sess := x.NewSession() - defer sess.Close() - sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - followList := make([]*Follow, 0) - sess.Find(&followList) - resultMap := make(map[int64]int) - log.Info("query Follow size=" + fmt.Sprint(len(followList))) - for _, followRecord := range followList { - if _, ok := resultMap[followRecord.FollowID]; !ok { - resultMap[followRecord.FollowID] = 1 - } else { - resultMap[followRecord.FollowID] += 1 - } - } - return resultMap -} - -func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { - sess := x.NewSession() - defer sess.Close() - sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - attachmentList := make([]*Attachment, 0) - sess.Find(&attachmentList) - resultMap := make(map[int64]int) - log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) - for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB - } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB - } - } - return resultMap - -} - -func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { - sess := x.NewSession() - defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - repoList := make([]*Repository, 0) - sess.Find(&repoList) - resultMap := make(map[int64]int) - log.Info("query Repository size=" + fmt.Sprint(len(repoList))) - for _, repoRecord := range repoList { - if _, ok := resultMap[repoRecord.OwnerID]; !ok { - resultMap[repoRecord.OwnerID] = 1 - } else { - resultMap[repoRecord.OwnerID] += 1 - } - } - return resultMap -} - -func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - repoStatisticList := make([]*RepoStatistic, 0) - statictisSess.Find(&repoStatisticList) - repoOpenIIndexMap := make(map[int64]float64) - log.Info("query repo_statistic size=" + fmt.Sprint(len(repoStatisticList))) - for _, repoRecord := range repoStatisticList { - if _, ok := repoOpenIIndexMap[repoRecord.RepoID]; !ok { - repoOpenIIndexMap[repoRecord.RepoID] = repoRecord.RadarTotal - } - } - - sess := x.NewSession() - defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false") - repoList := make([]*Repository, 0) - sess.Find(&repoList) - - userMap := make(map[int64]float64) - - log.Info("query Repository size=" + fmt.Sprint(len(repoList))) - for _, repoRecord := range repoList { - if _, ok := userMap[repoRecord.OwnerID]; !ok { - if _, ok := repoOpenIIndexMap[repoRecord.ID]; ok { - userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID] - } - } - } - - //query collaboration - sess.Select("repo_id,user_id,mode").Table("collaboration") - collaborationList := make([]*Collaboration, 0) - sess.Find(&collaborationList) - - log.Info("query collaborationList size=" + fmt.Sprint(len(collaborationList))) - - for _, collaborationRecord := range collaborationList { - if _, ok := userMap[collaborationRecord.UserID]; !ok { - if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { - userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID] - } - } else { - if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { - userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID] - } - } - } - - userMapJson, _ := json.Marshal(userMap) - log.Info("userMapJson=" + string(userMapJson)) - - return userMap -} - -func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - userLoginLogList := make([]*UserLoginLog, 0) - statictisSess.Find(&userLoginLogList) - resultMap := make(map[int64]int) - log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) - for _, loginRecord := range userLoginLogList { - if _, ok := resultMap[loginRecord.UId]; !ok { - resultMap[loginRecord.UId] = 1 - } else { - resultMap[loginRecord.UId] += 1 - } - } - return resultMap -} - -func subMonth(t1, t2 time.Time) (month int) { - y1 := t1.Year() - y2 := t2.Year() - m1 := int(t1.Month()) - m2 := int(t2.Month()) - d1 := t1.Day() - d2 := t2.Day() - - yearInterval := y1 - y2 - // 如果 d1的 月-日 小于 d2的 月-日 那么 yearInterval-- 这样就得到了相差的年数 - if m1 < m2 || m1 == m2 && d1 < d2 { - yearInterval-- - } - // 获取月数差值 - monthInterval := (m1 + 12) - m2 - if d1 < d2 { - monthInterval-- - } - monthInterval %= 12 - month = yearInterval*12 + monthInterval - if month == 0 { - month = 1 - } - return month -} diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 5c701eadb..26ca3ae8f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -49,13 +49,13 @@ func QueryUserStaticDataPage(ctx *context.Context) { startTime = time.Now() endTime = time.Now() } else { - startTime, _ = time.Parse("2006-01-02", startDate) + startTime, _ = time.ParseInLocation("2006-01-02", startDate, time.Local) settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime) if startTime.Unix() < settingStartTime.Unix() { startTime = settingStartTime startDate = settingStartTime.Format("2006-01-02") } - endTime, _ = time.Parse("2006-01-02", endDate) + endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local) endTime = endTime.AddDate(0, 0, 1) isAll = false log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) @@ -129,8 +129,12 @@ func QueryUserStaticDataPage(ctx *context.Context) { xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) - xlsx.SetCellValue(sheetName, "P"+rows, userRecord.RegistDate.Format("2006-01-02")) - xlsx.SetCellValue(sheetName, "Q"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02")) + + formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) + + formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) } //设置默认打开的表单 From f14325be504e53f8a43338e141028ed5e736b4c3 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 14:16:28 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=B8=8E=E7=95=8C=E9=9D=A2=E6=97=B6=E9=97=B4=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E7=BB=9F=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 696 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 696 insertions(+) create mode 100644 models/user_business_analysis.go diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go new file mode 100644 index 000000000..502cec806 --- /dev/null +++ b/models/user_business_analysis.go @@ -0,0 +1,696 @@ +package models + +import ( + "encoding/json" + "fmt" + "sort" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" +) + +type UserBusinessAnalysis struct { + ID int64 `xorm:"pk"` + + CountDate int64 `xorm:"pk"` + + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCommitRepo // 5 + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCreateIssue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + + //user + Email string `xorm:"NOT NULL"` + + //user + Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisQueryOptions struct { + ListOptions + UserName string + SortType string + StartTime int64 + EndTime int64 + IsAll bool +} + +type UserBusinessAnalysisList []*UserBusinessAnalysis + +func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisList) Less(i, j int) bool { + return ulist[i].ID > ulist[j].ID +} + +func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { + log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + statictisSess.Find(&userBusinessAnalysisList) + + resultMap := make(map[int64]*UserBusinessAnalysis) + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} + for _, v := range resultMap { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) + } + sort.Sort(userBusinessAnalysisReturnList) + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList +} + +func getLastCountDate() int64 { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Limit(1, 0) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").OrderBy("count_date desc").Limit(1, 0). + Find(&userBusinessAnalysisList); err == nil { + for _, userRecord := range userBusinessAnalysisList { + return userRecord.CountDate - 10000 + } + } else { + log.Info("query error." + err.Error()) + } + currentTimeNow := time.Now() + pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + return pageStartTime.Unix() +} + +func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + currentTimeNow := time.Now() + pageStartTime := getLastCountDate() + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() + + var cond = builder.NewCond() + if len(opts.UserName) > 0 { + cond = cond.And( + builder.Like{"name", opts.UserName}, + ) + } + cond = cond.And( + builder.Gte{"count_date": pageStartTime}, + ) + cond = cond.And( + builder.Lte{"count_date": pageEndTime}, + ) + + count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + statictisSess.Limit(opts.PageSize, start) + } + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + + resultMap := make(map[int64]*UserBusinessAnalysis) + + if len(userBusinessAnalysisList) > 0 { + var newAndCond = builder.NewCond() + var newOrCond = builder.NewCond() + for _, userRecord := range userBusinessAnalysisList { + newOrCond = newOrCond.Or( + builder.Eq{"id": userRecord.ID}, + ) + } + newAndCond = newAndCond.And( + newOrCond, + ) + if !opts.IsAll { + newAndCond = newAndCond.And( + builder.Gte{"count_date": opts.StartTime}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": opts.EndTime}, + ) + } + userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc"). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + } + + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} + for _, v := range resultMap { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) + } + sort.Sort(userBusinessAnalysisReturnList) + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, count +} + +func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { + + log.Info("start to count other user info data") + sess := x.NewSession() + defer sess.Close() + sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") + userList := make([]*User, 0) + sess.Find(&userList) + + currentTimeNow := time.Now() + log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) + + //yesterday := currentTimeNow.AddDate(0, 0, -1) + //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) + start_unix := startTime.Unix() + log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) + + //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + end_unix := endTime.Unix() + CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + if isReCount { + CountDate = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + } + + DataDate := startTime.Format("2006-01-02") + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap := queryAction(start_unix, end_unix, 5) + IssueCountMap := queryAction(start_unix, end_unix, 6) + + CommentCountMap := queryComment(start_unix, end_unix) + FocusRepoCountMap := queryWatch(start_unix, end_unix) + StarRepoCountMap := queryStar(start_unix, end_unix) + WatchedCountMap := queryFollow(start_unix, end_unix) + + CommitCodeSizeMap, err := GetAllUserKPIStats() + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysis + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate + if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { + dateRecord.CodeMergeCount = 0 + } else { + dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] + } + + if _, ok := CommitCountMap[dateRecord.ID]; !ok { + dateRecord.CommitCount = 0 + } else { + dateRecord.CommitCount = CommitCountMap[dateRecord.ID] + } + + if _, ok := IssueCountMap[dateRecord.ID]; !ok { + dateRecord.IssueCount = 0 + } else { + dateRecord.IssueCount = IssueCountMap[dateRecord.ID] + } + + if _, ok := CommentCountMap[dateRecord.ID]; !ok { + dateRecord.CommentCount = 0 + } else { + dateRecord.CommentCount = CommentCountMap[dateRecord.ID] + } + + if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { + dateRecord.FocusRepoCount = 0 + } else { + dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] + } + + if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { + dateRecord.StarRepoCount = 0 + } else { + dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] + } + + if _, ok := WatchedCountMap[dateRecord.ID]; !ok { + dateRecord.WatchedCount = 0 + } else { + dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] + } + + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) + } + + if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { + dateRecord.CommitDatasetSize = 0 + } else { + dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] + } + + if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { + dateRecord.SolveIssueCount = 0 + } else { + dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] + } + + if _, ok := wikiCountMap[dateRecord.Name]; !ok { + dateRecord.EncyclopediasCount = 0 + } else { + dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] + } + + if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { + dateRecord.CreateRepoCount = 0 + } else { + dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] + } + + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } + + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] + } + + dateRecord.CommitModelCount = 0 + + _, err = statictisSess.Insert(&dateRecord) + if err != nil { + log.Info("insert daterecord failed." + err.Error()) + return err + } + } + return nil +} + +func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { + CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) +} + +func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { + //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id + sess := x.NewSession() + defer sess.Close() + sess.Select("issue_assignees.*").Table("issue_assignees"). + Join("inner", "issue", "issue.id=issue_assignees.issue_id"). + Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) + issueAssigneesList := make([]*IssueAssignees, 0) + sess.Find(&issueAssigneesList) + resultMap := make(map[int64]int) + log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) + for _, issueAssigneesRecord := range issueAssigneesList { + if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { + resultMap[issueAssigneesRecord.AssigneeID] = 1 + } else { + resultMap[issueAssigneesRecord.AssigneeID] += 1 + } + } + return resultMap + +} + +func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + + sess.Select("issue.*").Table("issue"). + Join("inner", "pull_request", "issue.id=pull_request.issue_id"). + Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) + + issueList := make([]*Issue, 0) + sess.Find(&issueList) + resultMap := make(map[int64]int) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + return resultMap +} + +func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + resultMap := make(map[int64]int) + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + return resultMap +} + +func queryComment(start_unix int64, end_unix int64) map[int64]int { + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + commentList := make([]*Comment, 0) + sess.Find(&commentList) + resultMap := make(map[int64]int) + log.Info("query Comment size=" + fmt.Sprint(len(commentList))) + for _, commentRecord := range commentList { + if _, ok := resultMap[commentRecord.PosterID]; !ok { + resultMap[commentRecord.PosterID] = 1 + } else { + resultMap[commentRecord.PosterID] += 1 + } + } + return resultMap +} + +func queryWatch(start_unix int64, end_unix int64) map[int64]int { + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + watchList := make([]*Watch, 0) + sess.Find(&watchList) + resultMap := make(map[int64]int) + log.Info("query Watch size=" + fmt.Sprint(len(watchList))) + for _, watchRecord := range watchList { + if _, ok := resultMap[watchRecord.UserID]; !ok { + resultMap[watchRecord.UserID] = 1 + } else { + resultMap[watchRecord.UserID] += 1 + } + } + return resultMap + +} + +func queryStar(start_unix int64, end_unix int64) map[int64]int { + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + starList := make([]*Star, 0) + sess.Find(&starList) + resultMap := make(map[int64]int) + log.Info("query Star size=" + fmt.Sprint(len(starList))) + for _, starRecord := range starList { + if _, ok := resultMap[starRecord.UID]; !ok { + resultMap[starRecord.UID] = 1 + } else { + resultMap[starRecord.UID] += 1 + } + } + return resultMap + +} + +func queryFollow(start_unix int64, end_unix int64) map[int64]int { + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + followList := make([]*Follow, 0) + sess.Find(&followList) + resultMap := make(map[int64]int) + log.Info("query Follow size=" + fmt.Sprint(len(followList))) + for _, followRecord := range followList { + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 + } else { + resultMap[followRecord.FollowID] += 1 + } + } + return resultMap +} + +func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + attachmentList := make([]*Attachment, 0) + sess.Find(&attachmentList) + resultMap := make(map[int64]int) + log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) + for _, attachRecord := range attachmentList { + if _, ok := resultMap[attachRecord.UploaderID]; !ok { + resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + } else { + resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + } + } + return resultMap + +} + +func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + resultMap := make(map[int64]int) + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := resultMap[repoRecord.OwnerID]; !ok { + resultMap[repoRecord.OwnerID] = 1 + } else { + resultMap[repoRecord.OwnerID] += 1 + } + } + return resultMap +} + +func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoStatisticList := make([]*RepoStatistic, 0) + statictisSess.Find(&repoStatisticList) + repoOpenIIndexMap := make(map[int64]float64) + log.Info("query repo_statistic size=" + fmt.Sprint(len(repoStatisticList))) + for _, repoRecord := range repoStatisticList { + if _, ok := repoOpenIIndexMap[repoRecord.RepoID]; !ok { + repoOpenIIndexMap[repoRecord.RepoID] = repoRecord.RadarTotal + } + } + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false") + repoList := make([]*Repository, 0) + sess.Find(&repoList) + + userMap := make(map[int64]float64) + + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := userMap[repoRecord.OwnerID]; !ok { + if _, ok := repoOpenIIndexMap[repoRecord.ID]; ok { + userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID] + } + } + } + + //query collaboration + sess.Select("repo_id,user_id,mode").Table("collaboration") + collaborationList := make([]*Collaboration, 0) + sess.Find(&collaborationList) + + log.Info("query collaborationList size=" + fmt.Sprint(len(collaborationList))) + + for _, collaborationRecord := range collaborationList { + if _, ok := userMap[collaborationRecord.UserID]; !ok { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { + userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID] + } + } else { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { + userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID] + } + } + } + + userMapJson, _ := json.Marshal(userMap) + log.Info("userMapJson=" + string(userMapJson)) + + return userMap +} + +func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + return resultMap +} + +func subMonth(t1, t2 time.Time) (month int) { + y1 := t1.Year() + y2 := t2.Year() + m1 := int(t1.Month()) + m2 := int(t2.Month()) + d1 := t1.Day() + d2 := t2.Day() + + yearInterval := y1 - y2 + // 如果 d1的 月-日 小于 d2的 月-日 那么 yearInterval-- 这样就得到了相差的年数 + if m1 < m2 || m1 == m2 && d1 < d2 { + yearInterval-- + } + // 获取月数差值 + monthInterval := (m1 + 12) - m2 + if d1 < d2 { + monthInterval-- + } + monthInterval %= 12 + month = yearInterval*12 + monthInterval + if month == 0 { + month = 1 + } + return month +} From 27f96746c879c18314ee12436e68f75192c75b7c Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 23 Nov 2021 14:53:05 +0800 Subject: [PATCH 03/16] modify code --- web_src/js/components/ProAnalysis.vue | 44 +++++++++++++++++++++++++++------- web_src/js/components/UserAnalysis.vue | 38 ++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/web_src/js/components/ProAnalysis.vue b/web_src/js/components/ProAnalysis.vue index 6371fa884..b3abd1d04 100755 --- a/web_src/js/components/ProAnalysis.vue +++ b/web_src/js/components/ProAnalysis.vue @@ -8,13 +8,13 @@
统计周期： - + - +