diff --git a/models/custom_migrations.go b/models/custom_migrations.go index b196c5a8b..0b9a0d552 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -14,6 +14,8 @@ var customMigrations = []CustomMigration{ {"Custom v1 Topic struct change to support chinese", syncTopicStruct}, } +var customMigrationsStatic = []CustomMigration{} + func MigrateCustom(x *xorm.Engine) { for _, m := range customMigrations { @@ -27,6 +29,17 @@ func MigrateCustom(x *xorm.Engine) { } +func MigrateCustomStatic(x *xorm.Engine) { + for _, m := range customMigrationsStatic { + log.Info("Migration: %s", m.Description) + if err := m.Migrate(x); err != nil { + + log.Error("Migration: %v", err) + + } + } +} + func syncTopicStruct(x *xorm.Engine) error { query := "ALTER TABLE topic ALTER COLUMN name TYPE varchar(105);" diff --git a/models/models.go b/models/models.go index 696d0949b..86a091bca 100755 --- a/models/models.go +++ b/models/models.go @@ -190,7 +190,7 @@ func setEngine(engine *xorm.Engine, table []interface{}, database *setting.DBInf engine.SetMaxIdleConns(setting.Database.MaxIdleConns) engine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) engine.Sync2(table...) - MigrateCustom(engine) + return nil } @@ -222,7 +222,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e if err = newEngine(ctx, migrateFunc, x, tables, setting.Database); err != nil { return fmt.Errorf("newEngine failed: %v", err) } - + MigrateCustom(x) xStatistic, err = getEngine(setting.DatabaseStatistic) if err != nil { return fmt.Errorf("Failed to connect to database: %v", err) @@ -230,6 +230,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil { return fmt.Errorf("newEngine statistic failed: %v", err) } + MigrateCustomStatic(xStatistic) HasEngine = true diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index bb6726a2c..a4a155ba7 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -5,7 +5,9 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) type UserBusinessAnalysis struct { @@ -71,6 +73,14 @@ type UserBusinessAnalysis struct { Name string `xorm:"NOT NULL"` } +type UserBusinessAnalysisQueryOptions struct { + ListOptions + UserName string + SortType string + StartTime int64 + EndTime int64 +} + func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) statictisSess := xStatistic.NewSession() @@ -114,11 +124,114 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis return userBusinessAnalysisReturnList } +func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + currentTimeNow := time.Now() + pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + + var cond = builder.NewCond() + if len(opts.UserName) > 0 { + cond = cond.And( + builder.Eq{"name": opts.UserName}, + ) + } + cond = cond.And( + builder.Gte{"count_date": fmt.Sprint(pageStartTime)}, + ) + cond = cond.And( + builder.Lte{"count_date": fmt.Sprint(pageEndTime)}, + ) + + count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + statictisSess.Limit(opts.PageSize, start) + } + statictisSess.OrderBy("count_date desc") + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0, setting.UI.IssuePagingNum) + if err := statictisSess.Table("user_business_analysis").Where(cond). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + + resultMap := make(map[int64]*UserBusinessAnalysis) + + var newAndCond = builder.NewCond() + var newOrCond = builder.NewCond() + for _, userRecord := range userBusinessAnalysisList { + newOrCond.Or( + builder.Eq{"id": userRecord.ID}, + ) + } + newAndCond = newAndCond.And( + newOrCond, + ) + newAndCond = newAndCond.And( + builder.Gte{"count_date": fmt.Sprint(opts.StartTime)}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": fmt.Sprint(opts.EndTime)}, + ) + + userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) + index := 0 + for _, v := range resultMap { + userBusinessAnalysisReturnList[index] = v + index += 1 + } + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, count +} + func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user") + sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") userList := make([]*User, 0) sess.Find(&userList) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 239580adb..c25787703 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -6,6 +6,7 @@ package storage import ( "io" + "net/url" "path" "strconv" "strings" @@ -262,6 +263,7 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) input.Method = obs.HttpMethodGet reqParams := make(map[string]string) + fileName = url.QueryEscape(fileName) reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 975c3aa00..0fcd70a99 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "path" "strconv" @@ -107,6 +108,7 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) return } + sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) projectsPeriodData := ProjectsPeriodData{ RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), @@ -114,13 +116,23 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) { TotalPage: getTotalPage(total, pageSize), TotalCount: total, LastUpdatedTime: latestUpdatedTime, - PageRecords: models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)), + PageRecords: models.GetRepoStatisticByRawSql(sql), } ctx.JSON(http.StatusOK, projectsPeriodData) } +func generateSqlByType(ctx *context.Context, beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { + sql := "" + if ctx.QueryTrim("type") == "all" { + sql = generateTypeAllSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize) + } else { + sql = generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize) + } + return sql +} + func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() @@ -158,7 +170,7 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { return } - fileName := getFileName(ctx, beginTime, endTime) + fileName, frontName := getFileName(ctx, beginTime, endTime) if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil { ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error()) @@ -175,7 +187,7 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { writer.Write(allProjectsPeroidHeader(ctx)) for i := 0; i <= totalPage; i++ { - pageRecords := models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) for _, record := range pageRecords { e = writer.Write(allProjectsPeroidValues(record, ctx)) if e != nil { @@ -186,20 +198,24 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { } - ctx.ServeFile(fileName) + ctx.ServeFile(fileName, url.QueryEscape(frontName)) } -func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) string { - baseName := setting.RadarMap.Path + "/" - if ctx.QueryTrim("type") != "" { - baseName = baseName + ctx.QueryTrim("type") + "_" - } +func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) (string, string) { + baseName := setting.RadarMap.Path + "/项目分析_" + if ctx.QueryTrim("q") != "" { baseName = baseName + ctx.QueryTrim("q") + "_" } - baseName = baseName + beginTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_to_" + endTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv" - return baseName + if ctx.QueryTrim("type") == "all" { + baseName = baseName + "所有" + } else { + baseName = baseName + beginTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + } + frontName := baseName + ".csv" + localName := baseName + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv" + return localName, path.Base(frontName) } func ClearUnusedStatisticsFile() { @@ -219,8 +235,8 @@ func ClearUnusedStatisticsFile() { func allProjectsPeroidHeader(ctx *context.Context) []string { - return []string{ctx.Tr("repos.id"), ctx.Tr("repos.projectName"), ctx.Tr("repos.isPrivate"), ctx.Tr("repos.openi"), ctx.Tr("repos.visit"), ctx.Tr("repos.download"), ctx.Tr("repos.pr"), ctx.Tr("repos.commit"), - ctx.Tr("repos.watches"), ctx.Tr("repos.stars"), ctx.Tr("repos.forks"), ctx.Tr("repos.issues"), ctx.Tr("repos.closedIssues"), ctx.Tr("repos.contributor")} + return []string{ctx.Tr("admin.repos.id"), ctx.Tr("admin.repos.projectName"), ctx.Tr("admin.repos.isPrivate"), ctx.Tr("admin.repos.openi"), ctx.Tr("admin.repos.visit"), ctx.Tr("admin.repos.download"), ctx.Tr("admin.repos.pr"), ctx.Tr("admin.repos.commit"), + ctx.Tr("admin.repos.watches"), ctx.Tr("admin.repos.stars"), ctx.Tr("admin.repos.forks"), ctx.Tr("admin.repos.issues"), ctx.Tr("admin.repos.closedIssues"), ctx.Tr("admin.repos.contributor")} } @@ -234,9 +250,9 @@ func allProjectsPeroidValues(rs *models.RepoStatistic, ctx *context.Context) []s func getIsPrivateDisplay(private bool, ctx *context.Context) string { if private { - return ctx.Tr("repos.yes") + return ctx.Tr("admin.repos.yes") } else { - return ctx.Tr("repos.no") + return ctx.Tr("admin.repos.no") } } @@ -358,11 +374,11 @@ func generateTargetSql(beginTime time.Time, endTime time.Time, repoId int64) str return sql } -func generateCountSql(beginTime time.Time, endTime time.Time, yesterday string, q string) string { +func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, q string) string { countSql := "SELECT count(*) FROM " + "(SELECT repo_id FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + - "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + yesterday + "') B" + + "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { countSql = countSql + " and B.name like '%" + q + "%'" @@ -370,18 +386,34 @@ func generateCountSql(beginTime time.Time, endTime time.Time, yesterday string, return countSql } -func generatePageSql(beginTime time.Time, endTime time.Time, yesterday string, q string, orderBy string, page int, pageSize int) string { - countSql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + +func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { + sql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + + "(SELECT repo_id,sum(num_visits) as num_visits " + + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + + "(SELECT repo_id,name,is_private,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" + + " where A.repo_id=B.repo_id" + + if q != "" { + sql = sql + " and name like '%" + q + "%'" + } + sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) + return sql +} + +func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { + + sql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + "(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + - "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + yesterday + "') B" + + "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { - countSql = countSql + " and B.name like '%" + q + "%'" + sql = sql + " and B.name like '%" + q + "%'" } - countSql = countSql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) - return countSql + sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) + return sql } func getOrderBy(ctx *context.Context) string { diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 68cccd478..e24416d4f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) func QueryUserStaticData(ctx *context.Context) { @@ -19,7 +20,36 @@ func QueryUserStaticData(ctx *context.Context) { endTime, _ := time.Parse("2006-01-02", endDate) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) +} + +func QueryUserStaticDataPage(ctx *context.Context) { + startDate := ctx.Query("startDate") + endDate := ctx.Query("endDate") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + userName := ctx.Query("userName") + log.Info("startDate=" + startDate + " endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page)) + startTime, _ := time.Parse("2006-01-02", startDate) + endTime, _ := time.Parse("2006-01-02", endDate) + log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) + + pageOpts := &models.UserBusinessAnalysisQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + UserName: userName, + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + } + mapInterface := make(map[string]interface{}) + re, count := models.QueryUserStaticDataPage(pageOpts) + mapInterface["data"] = re + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) } func TimingCountDataByDate(date string) { diff --git a/routers/repo/view.go b/routers/repo/view.go index 8b9039087..13dcf22aa 100755 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -919,6 +919,7 @@ func Forks(ctx *context.Context) { } func Contributors(ctx *context.Context) { + ctx.Data["PageIsViewCode"] = true ctx.HTML(http.StatusOK, tplContributors) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a3de68373..0ce670c17 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -793,6 +793,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) m.Get("/tool/query_user_static", repo.QueryUserStaticData) + m.Get("/tool/query_user_static_page", repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Get("/contributors", repo.Contributors) diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 6dc4d8ee5..d5b911a79 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -93,7 +93,7 @@
{{if .Permission.CanRead $.UnitTypeCode}}
- + {{svg "octicon-code" 16}} {{.i18n.Tr "repo.code"}}