diff --git a/go.mod b/go.mod index 9f93281c3..69f788413 100755 --- a/go.mod +++ b/go.mod @@ -107,6 +107,7 @@ require ( github.com/unknwon/paginater v0.0.0-20151104151617-7748a72e0141 github.com/urfave/cli v1.22.1 github.com/xanzy/go-gitlab v0.31.0 + github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2 github.com/yohcop/openid-go v1.0.0 github.com/yuin/goldmark v1.1.27 github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60 diff --git a/go.sum b/go.sum index 20096f4bf..7346b9050 100755 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 h1:N9QFoeNsUXLhl14m gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7/go.mod h1:kgsbFPPS4P+acDYDOPDa3N4IWWOuDJt5/INKRUz7aks= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= +github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2 h1:StMrA6UQ5Cm6206DxXGuV/NMqSIOIDoMXMYt8JPe1lE= +github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2/go.mod h1:EfRHD2k+Kd7ijnqlwOrH1IifwgWB9yYJ0pdXtBZmlpU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -538,6 +540,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c h1:3wkDRdxK92dF+c1ke2dtj7ZzemFWBHB9plnJOtlwdFA= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= @@ -818,6 +822,7 @@ golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/models/custom_migrations.go b/models/custom_migrations.go index b196c5a8b..fe40fdf13 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -1,6 +1,8 @@ package models import ( + "fmt" + "code.gitea.io/gitea/modules/log" "xorm.io/xorm" ) @@ -10,10 +12,21 @@ type CustomMigration struct { Migrate func(*xorm.Engine) error } +type CustomMigrationStatic struct { + Description string + Migrate func(*xorm.Engine, *xorm.Engine) error +} + var customMigrations = []CustomMigration{ {"Custom v1 Topic struct change to support chinese", syncTopicStruct}, } +var customMigrationsStatic = []CustomMigrationStatic{ + {"Alter user static table field type ", alterUserStaticTable}, + {"Delete organization user history data ", deleteNotDisplayUser}, + {"update issue_fixed_rate to 1 if num_issues is 0 ", updateIssueFixedRate}, +} + func MigrateCustom(x *xorm.Engine) { for _, m := range customMigrations { @@ -27,6 +40,17 @@ func MigrateCustom(x *xorm.Engine) { } +func MigrateCustomStatic(x *xorm.Engine, static *xorm.Engine) { + for _, m := range customMigrationsStatic { + log.Info("Migration: %s", m.Description) + if err := m.Migrate(x, static); err != nil { + + log.Error("Migration: %v", err) + + } + } +} + func syncTopicStruct(x *xorm.Engine) error { query := "ALTER TABLE topic ALTER COLUMN name TYPE varchar(105);" @@ -34,3 +58,35 @@ func syncTopicStruct(x *xorm.Engine) error { _, err := x.Exec(query) return err } + +func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error { + alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision" + + _, err := static.Exec(alterSql) + return err + +} + +func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { + + querySQL := "select id,name from public.user where type=1" + rows, err := x.Query(querySQL) + if err != nil { + log.Info("select db failed,err:", err) + return err + } + + for i, userRow := range rows { + log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["name"])) + deleteSql := "delete from user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" + static.Exec(deleteSql) + } + + return nil +} + +func updateIssueFixedRate(x *xorm.Engine, static *xorm.Engine) error { + updateSQL := "update repo_statistic set issue_fixed_rate=1.0 where num_issues=0" + _, err := static.Exec(updateSQL) + return err +} diff --git a/models/models.go b/models/models.go index 696d0949b..4d39e2f14 100755 --- a/models/models.go +++ b/models/models.go @@ -190,7 +190,7 @@ func setEngine(engine *xorm.Engine, table []interface{}, database *setting.DBInf engine.SetMaxIdleConns(setting.Database.MaxIdleConns) engine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) engine.Sync2(table...) - MigrateCustom(engine) + return nil } @@ -222,7 +222,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e if err = newEngine(ctx, migrateFunc, x, tables, setting.Database); err != nil { return fmt.Errorf("newEngine failed: %v", err) } - + MigrateCustom(x) xStatistic, err = getEngine(setting.DatabaseStatistic) if err != nil { return fmt.Errorf("Failed to connect to database: %v", err) @@ -230,6 +230,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil { return fmt.Errorf("newEngine statistic failed: %v", err) } + MigrateCustomStatic(x, xStatistic) HasEngine = true diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index 7cfed2359..04f8f7ae1 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -11,8 +11,10 @@ import ( type ContributorWithUserId struct { git.Contributor - UserId int64 - IsAdmin bool + UserId int64 + IsAdmin bool + RelAvatarLink string + Email string } func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) { @@ -146,6 +148,8 @@ func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { contributor, user.ID, user.IsAdmin, + user.RelAvatarLink(), + user.Email, } } else { @@ -159,6 +163,8 @@ func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { contributor, -1, false, + "", + contributor.Email, } } else { value.CommitCnt += contributor.CommitCnt diff --git a/models/repo_statistic.go b/models/repo_statistic.go index df065bb79..0b2ded07a 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -12,6 +12,7 @@ type RepoStatistic struct { ID int64 `xorm:"pk autoincr" json:"-"` RepoID int64 `xorm:"unique(s) NOT NULL" json:"repo_id"` Name string `xorm:"INDEX" json:"name"` + OwnerName string `json:"ownerName"` IsPrivate bool `json:"isPrivate"` Date string `xorm:"unique(s) NOT NULL" json:"date"` NumWatches int64 `xorm:"NOT NULL DEFAULT 0" json:"watch"` diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index bb6726a2c..fb639459f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1,11 +1,14 @@ package models import ( + "encoding/json" "fmt" + "sort" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) type UserBusinessAnalysis struct { @@ -19,7 +22,7 @@ type UserBusinessAnalysis struct { //action :ActionCommitRepo // 5 CommitCount int `xorm:"NOT NULL DEFAULT 0"` - //action :ActionCommentIssue // 10 + //action :ActionCreateIssue // 10 IssueCount int `xorm:"NOT NULL DEFAULT 0"` //comment table current date @@ -62,13 +65,39 @@ type UserBusinessAnalysis struct { LoginCount int `xorm:"NOT NULL DEFAULT 0"` //openi index - OpenIIndex int `xorm:"NOT NULL DEFAULT 0"` + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` //user Email string `xorm:"NOT NULL"` //user Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisQueryOptions struct { + ListOptions + UserName string + SortType string + StartTime int64 + EndTime int64 +} + +type UserBusinessAnalysisList []*UserBusinessAnalysis + +func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisList) Less(i, j int) bool { + if ulist[i].CommitCount > ulist[j].CommitCount { + return true + } else { + if ulist[i].CommitCount == ulist[j].CommitCount { + return ulist[i].ID > ulist[j].ID + } else { + return false + } + } } func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { @@ -104,21 +133,125 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis } } - userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) - index := 0 + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} for _, v := range resultMap { - userBusinessAnalysisReturnList[index] = v - index += 1 + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) } + sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList } -func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { +func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + currentTimeNow := time.Now() + pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + + var cond = builder.NewCond() + if len(opts.UserName) > 0 { + cond = cond.And( + builder.Like{"name", opts.UserName}, + ) + } + cond = cond.And( + builder.Gte{"count_date": pageStartTime.Unix()}, + ) + cond = cond.And( + builder.Lte{"count_date": pageEndTime.Unix()}, + ) + + count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + statictisSess.Limit(opts.PageSize, start) + } + statictisSess.OrderBy("count_date desc") + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(cond). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + + resultMap := make(map[int64]*UserBusinessAnalysis) + + if opts.Page >= 0 && opts.PageSize > 0 { + var newAndCond = builder.NewCond() + var newOrCond = builder.NewCond() + for _, userRecord := range userBusinessAnalysisList { + newOrCond = newOrCond.Or( + builder.Eq{"id": userRecord.ID}, + ) + } + newAndCond = newAndCond.And( + newOrCond, + ) + newAndCond = newAndCond.And( + builder.Gte{"count_date": opts.StartTime}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": opts.EndTime}, + ) + + userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + } + + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} + for _, v := range resultMap { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) + } + sort.Sort(userBusinessAnalysisReturnList) + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, count +} + +func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) { + log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user") + sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") userList := make([]*User, 0) sess.Find(&userList) @@ -132,12 +265,15 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() - CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + if isReCount { + CountDate = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + } + DataDate := startTime.Format("2006-01-02") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryAction(start_unix, end_unix, 5) - IssueCountMap := queryAction(start_unix, end_unix, 10) + IssueCountMap := queryAction(start_unix, end_unix, 6) CommentCountMap := queryComment(start_unix, end_unix) FocusRepoCountMap := queryWatch(start_unix, end_unix) @@ -154,6 +290,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) + OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -170,6 +307,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { dateRecord.CodeMergeCount = 0 } else { @@ -248,6 +386,12 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti dateRecord.LoginCount = LoginCountMap[dateRecord.ID] } + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] + } + dateRecord.CommitModelCount = 0 statictisSess.Insert(&dateRecord) @@ -255,6 +399,10 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } +func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { + CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) +} + func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() @@ -386,10 +534,10 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { resultMap := make(map[int64]int) log.Info("query Follow size=" + fmt.Sprint(len(followList))) for _, followRecord := range followList { - if _, ok := resultMap[followRecord.UserID]; !ok { - resultMap[followRecord.UserID] = 1 + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 } else { - resultMap[followRecord.UserID] += 1 + resultMap[followRecord.FollowID] += 1 } } return resultMap @@ -432,6 +580,62 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoStatisticList := make([]*RepoStatistic, 0) + statictisSess.Find(&repoStatisticList) + repoOpenIIndexMap := make(map[int64]float64) + log.Info("query repo_statistic size=" + fmt.Sprint(len(repoStatisticList))) + for _, repoRecord := range repoStatisticList { + if _, ok := repoOpenIIndexMap[repoRecord.RepoID]; !ok { + repoOpenIIndexMap[repoRecord.RepoID] = repoRecord.RadarTotal + } + } + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false") + repoList := make([]*Repository, 0) + sess.Find(&repoList) + + userMap := make(map[int64]float64) + + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := userMap[repoRecord.OwnerID]; !ok { + if _, ok := repoOpenIIndexMap[repoRecord.ID]; ok { + userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID] + } + } + } + + //query collaboration + sess.Select("repo_id,user_id,mode").Table("collaboration") + collaborationList := make([]*Collaboration, 0) + sess.Find(&collaborationList) + + log.Info("query collaborationList size=" + fmt.Sprint(len(collaborationList))) + + for _, collaborationRecord := range collaborationList { + if _, ok := userMap[collaborationRecord.UserID]; !ok { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { + userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID] + } + } else { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { + userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID] + } + } + } + + userMapJson, _ := json.Marshal(userMap) + log.Info("userMapJson=" + string(userMapJson)) + + return userMap +} + func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -470,5 +674,8 @@ func subMonth(t1, t2 time.Time) (month int) { } monthInterval %= 12 month = yearInterval*12 + monthInterval + if month == 0 { + month = 1 + } return month } diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index eac081a8f..207018c20 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -11,7 +11,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/migrations" repository_service "code.gitea.io/gitea/modules/repository" - api_repo "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/repo" mirror_service "code.gitea.io/gitea/services/mirror" ) @@ -164,13 +163,13 @@ func registerHandleBlockChainUnSuccessCommits() { }) } -func registerHandleRepoStatistic() { - RegisterTaskFatal("handle_repo_statistic", &BaseConfig{ +func registerHandleRepoAndUserStatistic() { + RegisterTaskFatal("handle_repo_and_user_statistic", &BaseConfig{ Enabled: true, RunAtStart: false, Schedule: "@daily", }, func(ctx context.Context, _ *models.User, _ Config) error { - repo.RepoStatisticAuto() + repo.StatisticAuto() return nil }) } @@ -185,27 +184,6 @@ func registerHandleSummaryStatistic() { return nil }) } -func registerHandleUserStatistic() { - RegisterTaskFatal("handle_user_statistic", &BaseConfig{ - Enabled: true, - RunAtStart: false, - Schedule: "@daily", - }, func(ctx context.Context, _ *models.User, _ Config) error { - repo.TimingCountData() - return nil - }) -} - -func registerHandleClearRepoStatisticFile() { - RegisterTaskFatal("handle_repo_clear_statistic_file", &BaseConfig{ - Enabled: true, - RunAtStart: false, - Schedule: "@daily", - }, func(ctx context.Context, _ *models.User, _ Config) error { - api_repo.ClearUnusedStatisticsFile() - return nil - }) -} func initBasicTasks() { registerUpdateMirrorTask() @@ -222,7 +200,6 @@ func initBasicTasks() { registerHandleBlockChainMergedPulls() registerHandleBlockChainUnSuccessCommits() - registerHandleRepoStatistic() - registerHandleUserStatistic() + registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9f31612b6..c533eb924 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -547,8 +547,9 @@ var ( GrowthCommit float64 GrowthComments float64 RecordBeginTime string - Path string }{} + + Warn_Notify_Mails []string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1292,6 +1293,9 @@ func NewContext() { ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") SetRadarMapConfig() + + sec = Cfg.Section("warn_mail") + Warn_Notify_Mails = strings.Split(sec.Key("mails").MustString(""), ",") } func SetRadarMapConfig() { @@ -1329,7 +1333,6 @@ func SetRadarMapConfig() { RadarMap.GrowthCommit = sec.Key("growth_commit").MustFloat64(0.2) RadarMap.GrowthComments = sec.Key("growth_comments").MustFloat64(0.2) RadarMap.RecordBeginTime = sec.Key("record_beigin_time").MustString("2021-11-05") - RadarMap.Path = sec.Key("PATH").MustString("data/projectborad") } diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 239580adb..acf660e56 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -6,7 +6,9 @@ package storage import ( "io" + "net/url" "path" + "sort" "strconv" "strings" @@ -221,6 +223,9 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } fileInfos = append(fileInfos, fileInfo) } + sort.Slice(fileInfos, func(i, j int) bool { + return fileInfos[i].ModTime > fileInfos[j].ModTime + }) return fileInfos, err } else { if obsError, ok := err.(obs.ObsError); ok { @@ -262,6 +267,7 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) input.Method = obs.HttpMethodGet reqParams := make(map[string]string) + fileName = url.QueryEscape(fileName) reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index bc71693b8..cac2153bb 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -402,6 +402,26 @@ form.name_reserved = The username '%s' is reserved. form.name_pattern_not_allowed = The pattern '%s' is not allowed in a username. form.name_chars_not_allowed = User name '%s' contains invalid characters. +static.sheetname=User Analysis +static.id=ID +static.name=User Name +static.codemergecount=PR Count +static.commitcount=Commit Count +static.issuecount=Issue Count +static.commentcount=Comment Count +static.focusrepocount=Focus Repo Count +static.starrepocount=Repo Star Count +static.logincount=Login Count +static.watchedcount=Watched Count +static.commitcodesize=Commit Code Line +static.solveissuecount=Solve Issue Count +static.encyclopediascount=Encyclopedias Count +static.createrepocount=Create Repo Count +static.openiindex=OpenI Index +static.registdate=Regist Date +static.countdate=Count Date + + [settings] profile = Profile account = Account @@ -793,6 +813,9 @@ total_count_get_error=Can not get the total page. last_update_time_error=Can not get the last updated time. get_repo_stat_error=Can not get the statistics of the repository. get_repo_info_error=Can not get the information of the repository. +generate_statistic_file_error=Fail to generate file. +repo_stat_inspect=ProjectAnalysis +all=all modelarts.notebook=Debug Task modelarts.train_job=Train Task modelarts.train_job.new_debug= New Debug Task diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 6702a0434..9918d47e0 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -405,6 +405,24 @@ form.name_reserved='%s' 用户名被保留。 form.name_pattern_not_allowed=用户名中不允许使用 "%s"。 form.name_chars_not_allowed=用户名 '%s' 包含无效字符。 +static.sheetname=用户分析 +static.id=ID +static.name=用户名 +static.codemergecount=PR数 +static.commitcount=commit次数 +static.issuecount=提出任务数 +static.commentcount=评论数 +static.focusrepocount=关注项目数 +static.starrepocount=点赞项目数 +static.logincount=登录次数 +static.watchedcount=关注者数 +static.commitcodesize=commit代码行数 +static.solveissuecount=已解决任务数 +static.encyclopediascount=百科页面贡献次数 +static.createrepocount=创建项目数 +static.openiindex=OpenI指数 +static.registdate=用户注册时间 +static.countdate=系统统计时间 [settings] profile=个人信息 account=账号 @@ -797,6 +815,9 @@ total_count_get_error=查询总页数失败。 last_update_time_error=查询最新更新时间失败。 get_repo_stat_error=查询当前仓库的统计信息失败。 get_repo_info_error=查询当前仓库信息失败。 +generate_statistic_file_error=生成文件失败。 +repo_stat_inspect=项目分析 +all=所有 modelarts.notebook=调试任务 modelarts.train_job=训练任务 modelarts.train_job.new_debug=新建调试任务 diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 975c3aa00..9862a7744 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -1,15 +1,14 @@ package repo import ( - "encoding/csv" "fmt" - "io/ioutil" "net/http" - "os" - "path" + "net/url" "strconv" "time" + "github.com/360EntSecGroup-Skylar/excelize/v2" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" @@ -19,6 +18,7 @@ import ( const DEFAULT_PAGE_SIZE = 10 const DATE_FORMAT = "2006-01-02" +const EXCEL_DATE_FORMAT = "20060102" type ProjectsPeriodData struct { RecordBeginTime string `json:"recordBeginTime"` @@ -30,10 +30,12 @@ type ProjectsPeriodData struct { } type UserInfo struct { - User string `json:"user"` - Mode int `json:"mode"` - PR int64 `json:"pr"` - Commit int `json:"commit"` + User string `json:"user"` + Mode int `json:"mode"` + PR int64 `json:"pr"` + Commit int `json:"commit"` + RelAvatarLink string `json:"relAvatarLink"` + Email string `json:"email"` } type ProjectLatestData struct { @@ -107,6 +109,7 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) return } + sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) projectsPeriodData := ProjectsPeriodData{ RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), @@ -114,13 +117,23 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) { TotalPage: getTotalPage(total, pageSize), TotalCount: total, LastUpdatedTime: latestUpdatedTime, - PageRecords: models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)), + PageRecords: models.GetRepoStatisticByRawSql(sql), } ctx.JSON(http.StatusOK, projectsPeriodData) } +func generateSqlByType(ctx *context.Context, beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { + sql := "" + if ctx.QueryTrim("type") == "all" { + sql = generateTypeAllSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize) + } else { + sql = generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize) + } + return sql +} + func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() @@ -157,86 +170,82 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) return } + var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") + fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) - fileName := getFileName(ctx, beginTime, endTime) + totalPage := getTotalPage(total, pageSize) - if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil { - ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error()) - } + f := excelize.NewFile() - totalPage := getTotalPage(total, pageSize) + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") - f, e := os.Create(fileName) - defer f.Close() - if e != nil { - log.Warn("Failed to create file", e) + for k, v := range allProjectsPeroidHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) } - writer := csv.NewWriter(f) - writer.Write(allProjectsPeroidHeader(ctx)) + + var row = 2 for i := 0; i <= totalPage; i++ { - pageRecords := models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) for _, record := range pageRecords { - e = writer.Write(allProjectsPeroidValues(record, ctx)) - if e != nil { - log.Warn("Failed to write record", e) + + for k, v := range allProjectsPeroidValues(row, record, ctx) { + f.SetCellValue(projectAnalysis, k, v) } + row++ + } - writer.Flush() } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.ServeFile(fileName) + f.WriteTo(ctx.Resp) } -func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) string { - baseName := setting.RadarMap.Path + "/" - if ctx.QueryTrim("type") != "" { - baseName = baseName + ctx.QueryTrim("type") + "_" - } +func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, projectAnalysis string) string { + baseName := projectAnalysis + "_" + if ctx.QueryTrim("q") != "" { baseName = baseName + ctx.QueryTrim("q") + "_" } - baseName = baseName + beginTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_to_" + endTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv" - return baseName -} - -func ClearUnusedStatisticsFile() { - fileInfos, err := ioutil.ReadDir(setting.RadarMap.Path) - if err != nil { - log.Warn("can not read dir: "+setting.RadarMap.Path, err) - return - } - - for _, fileInfo := range fileInfos { - if !fileInfo.IsDir() && fileInfo.ModTime().Before(time.Now().AddDate(0, 0, -1)) { - os.Remove(path.Join(setting.RadarMap.Path, fileInfo.Name())) - } + if ctx.QueryTrim("type") == "all" { + baseName = baseName + ctx.Tr("repo.all") + } else { + baseName = baseName + beginTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) } - + frontName := baseName + ".xlsx" + return frontName } -func allProjectsPeroidHeader(ctx *context.Context) []string { +func allProjectsPeroidHeader(ctx *context.Context) map[string]string { - return []string{ctx.Tr("repos.id"), ctx.Tr("repos.projectName"), ctx.Tr("repos.isPrivate"), ctx.Tr("repos.openi"), ctx.Tr("repos.visit"), ctx.Tr("repos.download"), ctx.Tr("repos.pr"), ctx.Tr("repos.commit"), - ctx.Tr("repos.watches"), ctx.Tr("repos.stars"), ctx.Tr("repos.forks"), ctx.Tr("repos.issues"), ctx.Tr("repos.closedIssues"), ctx.Tr("repos.contributor")} + return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"), + "J1": ctx.Tr("admin.repos.watches"), "K1": ctx.Tr("admin.repos.stars"), "L1": ctx.Tr("admin.repos.forks"), "M1": ctx.Tr("admin.repos.issues"), "N1": ctx.Tr("admin.repos.closedIssues"), "O1": ctx.Tr("admin.repos.contributor")} } -func allProjectsPeroidValues(rs *models.RepoStatistic, ctx *context.Context) []string { - return []string{strconv.FormatInt(rs.RepoID, 10), rs.Name, getIsPrivateDisplay(rs.IsPrivate, ctx), strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), - strconv.FormatInt(rs.NumVisits, 10), strconv.FormatInt(rs.NumDownloads, 10), strconv.FormatInt(rs.NumPulls, 10), strconv.FormatInt(rs.NumCommits, 10), - strconv.FormatInt(rs.NumWatches, 10), strconv.FormatInt(rs.NumStars, 10), strconv.FormatInt(rs.NumForks, 10), strconv.FormatInt(rs.NumIssues, 10), - strconv.FormatInt(rs.NumClosedIssues, 10), strconv.FormatInt(rs.NumContributor, 10), +func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string { + return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.Name, getCellName("C", row): rs.OwnerName, getCellName("D", row): getIsPrivateDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), + getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10), + getCellName("J", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("K", row): strconv.FormatInt(rs.NumStars, 10), getCellName("L", row): strconv.FormatInt(rs.NumForks, 10), getCellName("M", row): strconv.FormatInt(rs.NumIssues, 10), + getCellName("N", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("O", row): strconv.FormatInt(rs.NumContributor, 10), } } +func getCellName(col string, row int) string { + return col + strconv.Itoa(row) +} + func getIsPrivateDisplay(private bool, ctx *context.Context) string { if private { - return ctx.Tr("repos.yes") + return ctx.Tr("admin.repos.yes") } else { - return ctx.Tr("repos.no") + return ctx.Tr("admin.repos.no") } } @@ -300,10 +309,11 @@ func GetProjectLatestStatistics(ctx *context.Context) { pr := models.GetPullCountByUserAndRepoId(repoIdInt, contributor.UserId) userInfo := UserInfo{ - User: contributor.Committer, - Commit: contributor.CommitCnt, - Mode: mode, - PR: pr, + User: contributor.Committer, + Commit: contributor.CommitCnt, + Mode: mode, + PR: pr, + RelAvatarLink: contributor.RelAvatarLink, } users = append(users, userInfo) @@ -345,7 +355,7 @@ func GetProjectPeriodStatistics(ctx *context.Context) { func generateRadarSql(beginTime time.Time, endTime time.Time, repoId int64) string { sql := "SELECT date, impact, completeness, liveness, project_health, team_health, growth, radar_total FROM repo_statistic" + " where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " order by created_unix" return sql } @@ -353,16 +363,16 @@ func generateRadarSql(beginTime time.Time, endTime time.Time, repoId int64) stri func generateTargetSql(beginTime time.Time, endTime time.Time, repoId int64) string { sql := "SELECT date, num_visits,num_downloads,num_commits FROM repo_statistic" + " where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " order by created_unix desc" return sql } -func generateCountSql(beginTime time.Time, endTime time.Time, yesterday string, q string) string { +func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, q string) string { countSql := "SELECT count(*) FROM " + "(SELECT repo_id FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + - "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + yesterday + "') B" + + "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { countSql = countSql + " and B.name like '%" + q + "%'" @@ -370,18 +380,34 @@ func generateCountSql(beginTime time.Time, endTime time.Time, yesterday string, return countSql } -func generatePageSql(beginTime time.Time, endTime time.Time, yesterday string, q string, orderBy string, page int, pageSize int) string { - countSql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + +func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { + sql := "SELECT A.repo_id,name,owner_name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + + "(SELECT repo_id,sum(num_visits) as num_visits " + + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + + "(SELECT repo_id,name,owner_name,is_private,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" + + " where A.repo_id=B.repo_id" + + if q != "" { + sql = sql + " and name like '%" + q + "%'" + } + sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) + return sql +} + +func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { + + sql := "SELECT A.repo_id,name,owner_name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + "(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + - "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + yesterday + "') B" + + "(SELECT repo_id,name,owner_name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { - countSql = countSql + " and B.name like '%" + q + "%'" + sql = sql + " and B.name like '%" + q + "%'" } - countSql = countSql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) - return countSql + sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) + return sql } func getOrderBy(ctx *context.Context) string { @@ -435,7 +461,7 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) } else if queryType == "current_week" { - beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now } else if queryType == "current_month" { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 529d576bc..1e1f7bf38 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "regexp" + "sort" "strconv" "strings" "time" @@ -528,6 +529,15 @@ func CloudBrainShowModels(ctx *context.Context) { return } + for i, fileInfo := range fileInfos { + temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime) + fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05") + } + + sort.Slice(fileInfos, func(i, j int) bool { + return fileInfos[i].ModTime > fileInfos[j].ModTime + }) + ctx.Data["Path"] = dirArray ctx.Data["Dirs"] = fileInfos ctx.Data["task"] = task diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index c1edb0fed..066b29772 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -3,18 +3,22 @@ package repo import ( "time" - "code.gitea.io/gitea/modules/setting" - - "code.gitea.io/gitea/modules/normalization" + "code.gitea.io/gitea/services/mailer" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/normalization" "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" ) -//auto daily or manually +func StatisticAuto() { + RepoStatisticAuto() + TimingCountData() +} + +//auto daily func RepoStatisticAuto() { - log.Info("", time.Now()) yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") setting.UpdateRadarMap() RepoStatisticDaily(yesterday) @@ -24,14 +28,17 @@ func RepoStatisticDaily(date string) { log.Info("%s", date) log.Info("begin Repo Statistic") t, _ := time.Parse("2006-01-02", date) + warnEmailMessage := "项目统计信息入库失败,请尽快定位。" if err := models.DeleteRepoStatDaily(date); err != nil { log.Error("DeleteRepoStatDaily failed: %v", err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) return } repos, err := models.GetAllRepositories() if err != nil { log.Error("GetAllRepositories failed: %v", err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) return } @@ -45,7 +52,7 @@ func RepoStatisticDaily(date string) { var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth int64 repoGitStat, err := models.GetRepoKPIStats(repo) if err != nil { - log.Error("GetRepoKPIStats failed: %s", repo.Name) + log.Error("GetRepoKPIStats failed: %s", getDistinctProjectName(repo)) } else { numDevMonths = repoGitStat.DevelopAge numKeyContributor = repoGitStat.KeyContributors @@ -59,41 +66,45 @@ func RepoStatisticDaily(date string) { var issueFixedRate float32 if repo.NumIssues != 0 { issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues) + } else { + issueFixedRate = 1.0 } var numVersions int64 numVersions, err = models.GetReleaseCountByRepoID(repo.ID, models.FindReleasesOptions{}) if err != nil { - log.Error("GetReleaseCountByRepoID failed(%s): %v", repo.Name, err) + log.Error("GetReleaseCountByRepoID failed(%s): %v", getDistinctProjectName(repo), err) } var datasetSize int64 datasetSize, err = getDatasetSize(repo) if err != nil { - log.Error("getDatasetSize failed(%s): %v", repo.Name, err) + log.Error("getDatasetSize failed(%s): %v", getDistinctProjectName(repo), err) } var numComments int64 numComments, err = models.GetCommentCountByRepoID(repo.ID) if err != nil { - log.Error("GetCommentCountByRepoID failed(%s): %v", repo.Name, err) + log.Error("GetCommentCountByRepoID failed(%s): %v", getDistinctProjectName(repo), err) } beginTime, endTime := getStatTime(date) var numVisits int numVisits, err = repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) if err != nil { - log.Error("AppointProjectView failed(%s): %v", repo.Name, err) + log.Error("AppointProjectView failed(%s): %v", getDistinctProjectName(repo), err) } repoStat := models.RepoStatistic{ - RepoID: repo.ID, - Date: date, - Name: repo.Name, - IsPrivate: repo.IsPrivate, - NumWatches: int64(repo.NumWatches), - NumStars: int64(repo.NumStars), - NumForks: int64(repo.NumForks), + RepoID: repo.ID, + Date: date, + Name: repo.Name, + IsPrivate: repo.IsPrivate, + OwnerName: repo.OwnerName, + NumWatches: int64(repo.NumWatches), + NumStars: int64(repo.NumStars), + NumForks: int64(repo.NumForks), + NumDownloads: repo.CloneCnt, NumComments: numComments, NumVisits: int64(numVisits), @@ -145,8 +156,9 @@ func RepoStatisticDaily(date string) { } if _, err = models.InsertRepoStat(&repoStat); err != nil { - log.Error("InsertRepoStat failed(%s): %v", repo.Name, err) - log.Error("failed statistic: %s", repo.Name) + log.Error("InsertRepoStat failed(%s): %v", getDistinctProjectName(repo), err) + log.Error("failed statistic: %s", getDistinctProjectName(repo)) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) continue } @@ -238,6 +250,10 @@ func RepoStatisticDaily(date string) { } +func getDistinctProjectName(repo *models.Repository) string { + return repo.OwnerName + "/" + repo.Name +} + func getDatasetSize(repo *models.Repository) (int64, error) { dataset, err := models.GetDatasetByRepo(repo) if err != nil { diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 68cccd478..134896177 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -3,12 +3,16 @@ package repo import ( "fmt" "net/http" + "net/url" + "strings" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "github.com/360EntSecGroup-Skylar/excelize/v2" ) func QueryUserStaticData(ctx *context.Context) { @@ -17,13 +21,119 @@ func QueryUserStaticData(ctx *context.Context) { log.Info("startDate=" + startDate + " endDate=" + endDate) startTime, _ := time.Parse("2006-01-02", startDate) endTime, _ := time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) - } -func TimingCountDataByDate(date string) { +func QueryUserStaticDataPage(ctx *context.Context) { + startDate := ctx.Query("startDate") + endDate := ctx.Query("endDate") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + userName := ctx.Query("userName") + IsReturnFile := ctx.QueryBool("IsReturnFile") + + log.Info("startDate=" + startDate + " endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page)) + startTime, _ := time.Parse("2006-01-02", startDate) + endTime, _ := time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) + log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) + + if IsReturnFile { + page = -1 + pageSize = -1 + } + + pageOpts := &models.UserBusinessAnalysisQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pageSize, + }, + UserName: userName, + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + } + mapInterface := make(map[string]interface{}) + re, count := models.QueryUserStaticDataPage(pageOpts) + mapInterface["data"] = re + mapInterface["count"] = count + if IsReturnFile { + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.static.sheetname") + index := xlsx.NewSheet(sheetName) + dataHeader := map[string]string{ + "A1": ctx.Tr("user.static.id"), + "B1": ctx.Tr("user.static.name"), + "C1": ctx.Tr("user.static.codemergecount"), + "D1": ctx.Tr("user.static.commitcount"), + "E1": ctx.Tr("user.static.issuecount"), + "F1": ctx.Tr("user.static.commentcount"), + "G1": ctx.Tr("user.static.focusrepocount"), + "H1": ctx.Tr("user.static.starrepocount"), + "I1": ctx.Tr("user.static.logincount"), + "J1": ctx.Tr("user.static.watchedcount"), + "K1": ctx.Tr("user.static.commitcodesize"), + "L1": ctx.Tr("user.static.solveissuecount"), + "M1": ctx.Tr("user.static.encyclopediascount"), + "N1": ctx.Tr("user.static.createrepocount"), + "O1": ctx.Tr("user.static.openiindex"), + "P1": ctx.Tr("user.static.registdate"), + "Q1": ctx.Tr("user.static.countdate"), + } + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + + for i, userRecord := range re { + rows := fmt.Sprint(i + 2) + + xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) + xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) + xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount) + xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount) + xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount) + xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount) + xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount) + xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount) + xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount) + xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize) + xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount) + xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) + xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) + xlsx.SetCellValue(sheetName, "O"+rows, userRecord.OpenIIndex) + xlsx.SetCellValue(sheetName, "P"+rows, userRecord.RegistDate.Format("2006-01-02")) + xlsx.SetCellValue(sheetName, "Q"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02")) + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + filename := sheetName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" + if len(userName) > 0 { + filename = sheetName + "_" + userName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" + } + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + if _, err := xlsx.WriteTo(ctx.Resp); err != nil { + log.Info("writer exel error." + err.Error()) + } + } else { + ctx.JSON(http.StatusOK, mapInterface) + } +} + +func TimingCountDataByDateAndReCount(date string, isReCount bool) { t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -70,16 +180,17 @@ func TimingCountDataByDate(date string) { } } //other user info data - models.CounDataByDate(wikiMap, startTime, endTime) + models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount) +} +func TimingCountDataByDate(date string) { + TimingCountDataByDateAndReCount(date, true) } func TimingCountData() { - log.Info("start to time count data") currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02") - - TimingCountDataByDate(startTime) + TimingCountDataByDateAndReCount(startTime, false) } diff --git a/routers/repo/view.go b/routers/repo/view.go index 50a53f68b..a5f2e1477 100755 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -916,6 +916,7 @@ func Forks(ctx *context.Context) { } func Contributors(ctx *context.Context) { + ctx.Data["PageIsViewCode"] = true ctx.HTML(http.StatusOK, tplContributors) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a3de68373..c28e76a47 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,7 +792,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static", repo.QueryUserStaticData) + m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData) + m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Get("/contributors", repo.Contributors) diff --git a/services/mailer/mail.go b/services/mailer/mail.go index e9395f130..ef7447be2 100755 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -115,6 +115,22 @@ func SendActivateEmailMail(locale Locale, u *models.User, email *models.EmailAdd SendAsync(msg) } +func SendWarnNotifyMail(emails []string, message string) { + if setting.MailService == nil { + log.Warn("SendWarnNotifyMail is being invoked but mail service hasn't been initialized") + return + } + if len(emails) == 0 { + log.Warn("SendWarnNotifyMail is being invoked but do not have email to send") + return + } + msg := NewMessage(emails, message, message) + msg.Info = fmt.Sprintf(message) + + SendAsync(msg) + +} + // SendRegisterNotifyMail triggers a notify e-mail by admin created a account. func SendRegisterNotifyMail(locale Locale, u *models.User) { if setting.MailService == nil { diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 67d8fc1b7..49b3181e7 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -37,7 +37,9 @@ {{.i18n.Tr "explore.users"}} {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} - {{.i18n.Tr "explore.data_analysis"}} + {{if .IsAdmin}} + {{.i18n.Tr "explore.data_analysis"}} + {{end}} {{else if .IsLandingPageHome}} @@ -53,6 +55,9 @@ {{.i18n.Tr "explore.users"}} {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} + {{if .IsAdmin}} + {{.i18n.Tr "explore.data_analysis"}} + {{end}} {{else if .IsLandingPageExplore}} diff --git a/templates/base/head_navbar_home.tmpl b/templates/base/head_navbar_home.tmpl index e334efba8..c94ff269f 100644 --- a/templates/base/head_navbar_home.tmpl +++ b/templates/base/head_navbar_home.tmpl @@ -29,7 +29,9 @@ {{.i18n.Tr "explore.users"}} {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} - {{.i18n.Tr "explore.data_analysis"}} + {{if .IsAdmin}} + {{.i18n.Tr "explore.data_analysis"}} + {{end}} {{else if .IsLandingPageHome}} @@ -45,6 +47,9 @@ {{.i18n.Tr "explore.users"}} {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} + {{if .IsAdmin}} + {{.i18n.Tr "explore.data_analysis"}} + {{end}} {{else if .IsLandingPageExplore}} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 6dc4d8ee5..d5b911a79 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -93,7 +93,7 @@