diff --git a/go.mod b/go.mod index 9f93281c3..697e61728 100755 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( gitea.com/macaron/macaron v1.4.0 gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 + github.com/360EntSecGroup-Skylar/excelize v1.4.1 // indirect github.com/BurntSushi/toml v0.3.1 github.com/PuerkitoBio/goquery v1.5.0 github.com/RichardKnop/machinery v1.6.9 diff --git a/go.sum b/go.sum index 20096f4bf..a85c68479 100755 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 h1:N9QFoeNsUXLhl14m gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7/go.mod h1:kgsbFPPS4P+acDYDOPDa3N4IWWOuDJt5/INKRUz7aks= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= +github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0gHwBYyfo5Jcjv/Ks= +github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -538,6 +540,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c h1:3wkDRdxK92dF+c1ke2dtj7ZzemFWBHB9plnJOtlwdFA= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= @@ -713,6 +717,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/models/custom_migrations.go b/models/custom_migrations.go index dd72e7934..1b3802ef5 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -22,6 +22,7 @@ var customMigrations = []CustomMigration{ } var customMigrationsStatic = []CustomMigrationStatic{ + {"Alter user static table field type open_i_index ", alterUserStaticTable}, {"Delete zuzhi user history data ", deleteNotDisplayUser}, } @@ -57,10 +58,17 @@ func syncTopicStruct(x *xorm.Engine) error { return err } +func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error { + alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision" + + _, err := static.Exec(alterSql) + return err + +} + func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { querySQL := "select id,name from public.user where type=1" - rows, err := x.Query(querySQL) if err != nil { log.Info("select db failed,err:", err) @@ -68,8 +76,8 @@ func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { } for i, userRow := range rows { - log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["id"])) - deleteSql := "delete from user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" + log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["name"])) + deleteSql := "delete from public.user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" static.Exec(deleteSql) } diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ac51c2816..25e6dcc2f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -3,6 +3,7 @@ package models import ( "encoding/json" "fmt" + "sort" "time" "code.gitea.io/gitea/modules/log" @@ -65,13 +66,15 @@ type UserBusinessAnalysis struct { LoginCount int `xorm:"NOT NULL DEFAULT 0"` //openi index - OpenIIndex int `xorm:"NOT NULL DEFAULT 0"` + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` //user Email string `xorm:"NOT NULL"` //user Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NOT NULL"` } type UserBusinessAnalysisQueryOptions struct { @@ -82,6 +85,22 @@ type UserBusinessAnalysisQueryOptions struct { EndTime int64 } +type UserBusinessAnalysisList []*UserBusinessAnalysis + +func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisList) Less(i, j int) bool { + if ulist[i].CommitCount > ulist[j].CommitCount { + return true + } else { + if ulist[i].CommitCount == ulist[j].CommitCount { + return ulist[i].ID > ulist[j].ID + } else { + return false + } + } +} + func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) statictisSess := xStatistic.NewSession() @@ -115,12 +134,11 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis } } - userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) - index := 0 + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} for _, v := range resultMap { - userBusinessAnalysisReturnList[index] = v - index += 1 + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) } + sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList } @@ -138,7 +156,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus var cond = builder.NewCond() if len(opts.UserName) > 0 { cond = cond.And( - builder.Eq{"name": opts.UserName}, + builder.Like{"name", opts.UserName}, ) } cond = cond.And( @@ -218,17 +236,17 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus } } - userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) - index := 0 + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} for _, v := range resultMap { - userBusinessAnalysisReturnList[index] = v - index += 1 + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) } + sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, count } -func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { +func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) { + log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() @@ -246,9 +264,12 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() - CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + if isReCount { + CountDate = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 1, 0, 0, startTime.Location()) + } + DataDate := startTime.Format("2006-01-02") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryAction(start_unix, end_unix, 5) IssueCountMap := queryAction(start_unix, end_unix, 10) @@ -285,6 +306,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { dateRecord.CodeMergeCount = 0 } else { @@ -366,7 +388,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { dateRecord.OpenIIndex = 0 } else { - dateRecord.OpenIIndex = int(OpenIIndexMap[dateRecord.ID] * 100) + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] } dateRecord.CommitModelCount = 0 @@ -376,6 +398,10 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } +func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { + CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) +} + func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() @@ -507,10 +533,10 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { resultMap := make(map[int64]int) log.Info("query Follow size=" + fmt.Sprint(len(followList))) for _, followRecord := range followList { - if _, ok := resultMap[followRecord.UserID]; !ok { - resultMap[followRecord.UserID] = 1 + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 } else { - resultMap[followRecord.UserID] += 1 + resultMap[followRecord.FollowID] += 1 } } return resultMap diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index e24416d4f..3f529272d 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/360EntSecGroup-Skylar/excelize" ) func QueryUserStaticData(ctx *context.Context) { @@ -18,6 +19,7 @@ func QueryUserStaticData(ctx *context.Context) { log.Info("startDate=" + startDate + " endDate=" + endDate) startTime, _ := time.Parse("2006-01-02", startDate) endTime, _ := time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) } @@ -29,17 +31,24 @@ func QueryUserStaticDataPage(ctx *context.Context) { if page <= 0 { page = 1 } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } userName := ctx.Query("userName") + IsReturnFile := ctx.QueryBool("IsReturnFile") + log.Info("startDate=" + startDate + " endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page)) startTime, _ := time.Parse("2006-01-02", startDate) endTime, _ := time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) pageOpts := &models.UserBusinessAnalysisQueryOptions{ ListOptions: models.ListOptions{ Page: page, - PageSize: setting.UI.IssuePagingNum, + PageSize: pageSize, }, UserName: userName, StartTime: startTime.Unix(), @@ -49,11 +58,71 @@ func QueryUserStaticDataPage(ctx *context.Context) { re, count := models.QueryUserStaticDataPage(pageOpts) mapInterface["data"] = re mapInterface["count"] = count - ctx.JSON(http.StatusOK, mapInterface) -} -func TimingCountDataByDate(date string) { + if IsReturnFile { + //writer exec file. + xlsx := excelize.NewFile() + sheetName := "用户分析" + index := xlsx.NewSheet(sheetName) + dataHeader := map[string]string{ + //学科 + "A1": "ID", + "B1": "用户名", + "C1": "PR数", + "D1": "提出任务数", + "E1": "评论数", + "F1": "关注项目数", + "G1": "点赞项目数", + "H1": "登录次数", + "I1": "关注者数", + "J1": "commit代码行数", + "K1": "已解决任务数", + "L1": "百科页面贡献次数", + "M1": "创建项目", + "N1": "OpenI指数", + "O1": "用户注册时间", + "P1": "系统统计时间", + } + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + + for i, userRecord := range re { + rows := fmt.Sprint(i + 2) + + xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) + xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) + xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.IssueCount) + xlsx.SetCellValue(sheetName, "E"+rows, userRecord.CommentCount) + xlsx.SetCellValue(sheetName, "F"+rows, userRecord.FocusRepoCount) + xlsx.SetCellValue(sheetName, "G"+rows, userRecord.StarRepoCount) + xlsx.SetCellValue(sheetName, "H"+rows, userRecord.LoginCount) + xlsx.SetCellValue(sheetName, "I"+rows, userRecord.WatchedCount) + xlsx.SetCellValue(sheetName, "J"+rows, userRecord.CommitCodeSize) + xlsx.SetCellValue(sheetName, "K"+rows, userRecord.SolveIssueCount) + xlsx.SetCellValue(sheetName, "L"+rows, userRecord.EncyclopediasCount) + xlsx.SetCellValue(sheetName, "M"+rows, userRecord.CreateRepoCount) + xlsx.SetCellValue(sheetName, "N"+rows, userRecord.OpenIIndex) + xlsx.SetCellValue(sheetName, "O"+rows, userRecord.RegistDate.Format("2006-01-02")) + xlsx.SetCellValue(sheetName, "P"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02")) + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + err := xlsx.SaveAs("./成绩表.xlsx") + if err != nil { + log.Error(err) + } + + } else { + ctx.JSON(http.StatusOK, mapInterface) + } +} + +func TimingCountDataByDateAndReCount(date string, isReCount bool) { t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -100,16 +169,17 @@ func TimingCountDataByDate(date string) { } } //other user info data - models.CounDataByDate(wikiMap, startTime, endTime) + models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount) +} +func TimingCountDataByDate(date string) { + TimingCountDataByDateAndReCount(date, true) } func TimingCountData() { - log.Info("start to time count data") currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02") - - TimingCountDataByDate(startTime) + TimingCountDataByDateAndReCount(startTime, false) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index f827685a7..994cbf36c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,8 +792,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static", repo.QueryUserStaticData) - m.Get("/tool/query_user_static_page", repo.QueryUserStaticDataPage) + m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData) + m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Get("/contributors", repo.Contributors)