From afad3923f43ac3c5889e42d6e7dda993aa4ae95b Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 23 Nov 2021 19:53:38 +0800 Subject: [PATCH 1/4] add tool --- go.mod | 2 +- models/repo_statistic.go | 7 +++++++ routers/private/internal.go | 3 ++- routers/private/tool.go | 29 ++++++++++++++++++++++++++++- routers/repo/repo_statistic.go | 24 ++++++++++++++++++++++-- 5 files changed, 60 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 69f788413..337aabc8e 100755 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( gitea.com/macaron/macaron v1.4.0 gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 + github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2 github.com/BurntSushi/toml v0.3.1 github.com/PuerkitoBio/goquery v1.5.0 github.com/RichardKnop/machinery v1.6.9 @@ -107,7 +108,6 @@ require ( github.com/unknwon/paginater v0.0.0-20151104151617-7748a72e0141 github.com/urfave/cli v1.22.1 github.com/xanzy/go-gitlab v0.31.0 - github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2 github.com/yohcop/openid-go v1.0.0 github.com/yuin/goldmark v1.1.27 github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60 diff --git a/models/repo_statistic.go b/models/repo_statistic.go index 0b2ded07a..5e872fa38 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -172,3 +172,10 @@ func UpdateRepoStat(repoStat *RepoStatistic) error { _, err := xStatistic.Exec(sql, repoStat.Impact, repoStat.Completeness, repoStat.Liveness, repoStat.ProjectHealth, repoStat.TeamHealth, repoStat.Growth, repoStat.RadarTotal, repoStat.RepoID, repoStat.Date) return err } + +func UpdateRepoStatVisits(repoStat *RepoStatistic) error { + sql := "update repo_statistic set num_visits=? where repo_id=? and date=?" + + _, err := xStatistic.Exec(sql, repoStat.NumVisits, repoStat.RepoID, repoStat.Date) + return err +} diff --git a/routers/private/internal.go b/routers/private/internal.go index b254d48ba..0dd725ca3 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -43,7 +43,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/manager/restart", Restart) m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues) m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) - m.Post("/tool/repo_stat", RepoStatisticManually) + m.Post("/tool/repo_stat/:date", RepoStatisticManually) + m.Post("/tool/update_repo_visit/:date", UpdateRepoVisit) }, CheckInternalToken) } diff --git a/routers/private/tool.go b/routers/private/tool.go index b93f17090..d01c5b2ab 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -39,8 +39,35 @@ func UpdateAllRepoCommitCnt(ctx *macaron.Context) { } func RepoStatisticManually(ctx *macaron.Context) { - date := ctx.Query("date") + date := ctx.Params("date") repo.RepoStatisticDaily(date) repo.SummaryStatisticDaily(date) repo.TimingCountDataByDate(date) } + +func UpdateRepoVisit(ctx *macaron.Context) { + date := ctx.Params("date") + log.Info("date(%s)", date) + + repos, err := models.GetAllRepositories() + if err != nil { + log.Error("GetAllRepositories failed:%v", err.Error(), ctx.Data["MsgID"]) + ctx.JSON(http.StatusInternalServerError, map[string]string{ + "error_msg": "GetAllRepositories failed", + }) + return + } + + for i, repoStat := range repos { + log.Info("%d:begin UpdateRepoVisits(id = %d, name = %s)", i, repoStat.ID, repoStat.Name) + if err = repo.UpdateRepoVisits(ctx, repoStat, date); err != nil { + log.Error("UpdateRepoVisits(id = %d, name = %s) failed:%v", repoStat.ID, repoStat.Name, err.Error()) + continue + } + log.Info("%d:finish UpdateRepoVisits(id = %d, name = %s)", i, repoStat.ID, repoStat.Name) + } + + ctx.JSON(http.StatusOK, map[string]string{ + "error_msg": "", + }) +} diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 066b29772..d6f7424ea 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -3,13 +3,14 @@ package repo import ( "time" - "code.gitea.io/gitea/services/mailer" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/normalization" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer" + + "gitea.com/macaron/macaron" ) func StatisticAuto() { @@ -274,3 +275,22 @@ func getStatTime(timeStr string) (string, string) { return beginTime, endTime } + +func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string) error { + beginTime, endTime := getStatTime(date) + var numVisits int + numVisits, err := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) + if err != nil { + log.Error("AppointProjectView failed(%s): %v", getDistinctProjectName(repo), err) + return err + } + + repoStat, err := models.GetRepoStatisticByDate(date, repo.ID) + repoStat[0].NumVisits = int64(numVisits) + + if err = models.UpdateRepoStatVisits(repoStat[0]); err != nil { + log.Error("UpdateRepoStatVisits failed(%s): %v", getDistinctProjectName(repo), err) + return err + } + return nil +} From b092e7077ff7f4322bbbc7aca8f1fedf07362371 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 24 Nov 2021 09:51:49 +0800 Subject: [PATCH 2/4] opt --- routers/repo/repo_statistic.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index d6f7424ea..4c8c1b68a 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -1,6 +1,7 @@ package repo import ( + "errors" "time" "code.gitea.io/gitea/models" @@ -286,6 +287,13 @@ func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string } repoStat, err := models.GetRepoStatisticByDate(date, repo.ID) + if err != nil { + log.Error("GetRepoStatisticByDate failed(%s): %v", getDistinctProjectName(repo), err) + return err + } else if len(repoStat) != 1 { + log.Error("GetRepoStatisticByDate failed(%s): %v", getDistinctProjectName(repo), err) + return errors.New("not find repo") + } repoStat[0].NumVisits = int64(numVisits) if err = models.UpdateRepoStatVisits(repoStat[0]); err != nil { From 02546c41026886bca1268d60d9c830241b51995a Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 24 Nov 2021 10:33:14 +0800 Subject: [PATCH 3/4] opt --- routers/repo/repo_statistic.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 4c8c1b68a..7aadd431f 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -50,7 +50,7 @@ func RepoStatisticDaily(date string) { var maxRepoRadar models.RepoStatistic for i, repo := range repos { - log.Info("start statistic: %s", repo.Name) + log.Info("start statistic: %s", getDistinctProjectName(repo)) var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth int64 repoGitStat, err := models.GetRepoKPIStats(repo) if err != nil { @@ -232,7 +232,7 @@ func RepoStatisticDaily(date string) { } - log.Info("finish statistic: %s", repo.Name) + log.Info("finish statistic: %s", getDistinctProjectName(repo)) } //radar map From b6f6d57b18e9cc23c6c9e9faa54bece8650b6bff Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 10:38:53 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E8=A1=A8=E7=BB=93=E6=A3=8D=E7=9A=84=E4=BB=A3=E7=A0=81=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E6=97=B6=E7=BB=A7=E7=BB=AD=E4=BF=AE=E6=94=B9=E5=88=86?= =?UTF-8?q?=E9=A1=B5=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 9 ----- models/user_business_analysis.go | 70 ++++++++++++++++++++++++++++++++++++++ routers/repo/user_data_analysis.go | 11 +++--- 3 files changed, 77 insertions(+), 13 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index fe40fdf13..d0158530b 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -22,7 +22,6 @@ var customMigrations = []CustomMigration{ } var customMigrationsStatic = []CustomMigrationStatic{ - {"Alter user static table field type ", alterUserStaticTable}, {"Delete organization user history data ", deleteNotDisplayUser}, {"update issue_fixed_rate to 1 if num_issues is 0 ", updateIssueFixedRate}, } @@ -59,14 +58,6 @@ func syncTopicStruct(x *xorm.Engine) error { return err } -func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error { - alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision" - - _, err := static.Exec(alterSql) - return err - -} - func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { querySQL := "select id,name from public.user where type=1" diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index e5f92c07b..08be81241 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -153,6 +153,76 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } +func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + resultMap := make(map[int64]*UserBusinessAnalysis) + + var newAndCond = builder.NewCond() + if len(opts.UserName) > 0 { + newAndCond = newAndCond.And( + builder.Like{"name", opts.UserName}, + ) + } + if !opts.IsAll { + newAndCond = newAndCond.And( + builder.Gte{"count_date": opts.StartTime}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": opts.EndTime}, + ) + } + + allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + log.Info("query return total:" + fmt.Sprint(allCount)) + pageSize := 200 + totalPage := int(allCount) / pageSize + + for i := 0; i <= int(totalPage); i++ { + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + } + + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} + for _, v := range resultMap { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) + } + sort.Sort(userBusinessAnalysisReturnList) + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, allCount +} + func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 8d7f72ce0..5a6b3d724 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -76,11 +76,10 @@ func QueryUserStaticDataPage(ctx *context.Context) { EndTime: endTime.Unix(), IsAll: isAll, } - mapInterface := make(map[string]interface{}) - re, count := models.QueryUserStaticDataPage(pageOpts) - mapInterface["data"] = re - mapInterface["count"] = count + if IsReturnFile { + re, count := models.QueryUserStaticDataAll(pageOpts) + log.Info("return count=" + fmt.Sprint(count)) //writer exec file. xlsx := excelize.NewFile() sheetName := ctx.Tr("user.static.sheetname") @@ -163,6 +162,10 @@ func QueryUserStaticDataPage(ctx *context.Context) { } } else { + mapInterface := make(map[string]interface{}) + re, count := models.QueryUserStaticDataPage(pageOpts) + mapInterface["data"] = re + mapInterface["count"] = count ctx.JSON(http.StatusOK, mapInterface) } }