From 0b591cd76bdd1805e06d4f06a04e904966e9370f Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 09:23:22 +0800 Subject: [PATCH 01/16] =?UTF-8?q?-m=E4=B8=8B=E8=BD=BD=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=EF=BC=8C=E5=88=86=E9=A1=B5=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 57 ++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 502cec806..e5f92c07b 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -219,32 +219,43 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus builder.Lte{"count_date": opts.EndTime}, ) } - userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc"). - Find(&userBusinessAnalysisList); err != nil { + + allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) return nil, 0 } - } - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + pageSize := 1000 + totalPage := int(allCount) / pageSize + + for i := 0; i <= int(totalPage); i++ { + userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } } } From 0bb06ded98b161cad5079bb4229d9884d1bb41f5 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 10:09:39 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 70 ++++++++++++++++++++++++++++++++++++++ routers/repo/user_data_analysis.go | 11 +++--- routers/routes/routes.go | 1 + 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index e5f92c07b..08be81241 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -153,6 +153,76 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } +func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + resultMap := make(map[int64]*UserBusinessAnalysis) + + var newAndCond = builder.NewCond() + if len(opts.UserName) > 0 { + newAndCond = newAndCond.And( + builder.Like{"name", opts.UserName}, + ) + } + if !opts.IsAll { + newAndCond = newAndCond.And( + builder.Gte{"count_date": opts.StartTime}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": opts.EndTime}, + ) + } + + allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + log.Info("query return total:" + fmt.Sprint(allCount)) + pageSize := 200 + totalPage := int(allCount) / pageSize + + for i := 0; i <= int(totalPage); i++ { + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + } + + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} + for _, v := range resultMap { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) + } + sort.Sort(userBusinessAnalysisReturnList) + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, allCount +} + func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 8d7f72ce0..5a6b3d724 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -76,11 +76,10 @@ func QueryUserStaticDataPage(ctx *context.Context) { EndTime: endTime.Unix(), IsAll: isAll, } - mapInterface := make(map[string]interface{}) - re, count := models.QueryUserStaticDataPage(pageOpts) - mapInterface["data"] = re - mapInterface["count"] = count + if IsReturnFile { + re, count := models.QueryUserStaticDataAll(pageOpts) + log.Info("return count=" + fmt.Sprint(count)) //writer exec file. xlsx := excelize.NewFile() sheetName := ctx.Tr("user.static.sheetname") @@ -163,6 +162,10 @@ func QueryUserStaticDataPage(ctx *context.Context) { } } else { + mapInterface := make(map[string]interface{}) + re, count := models.QueryUserStaticDataPage(pageOpts) + mapInterface["data"] = re + mapInterface["count"] = count ctx.JSON(http.StatusOK, mapInterface) } } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 19b8da196..90847b2e5 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -975,6 +975,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Put("/modify_model", repo.ModifyModelInfo) m.Get("/show_model", reqRepoCloudBrainReader, repo.ShowModelPageInfo) m.Get("/query_train_job", reqRepoCloudBrainReader, repo.QueryTrainJobList) + m.Get("/query_train_job_version", reqRepoCloudBrainReader, repo.QueryTrainJobVersionList) m.Group("/:ID", func() { m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel) m.Get("/downloadsingle", reqRepoCloudBrainReader, repo.DownloadSingleModelFile) From b799ca409cfaf2c349ec7459b1ad607d667c8a05 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 10:12:00 +0800 Subject: [PATCH 03/16] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 1c7febdf1..692bc6a05 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -257,6 +257,30 @@ func DownloadMultiModelFile(ctx *context.Context) { } } +func QueryTrainJobVersionList(ctx *context.Context) { + log.Info("query train job version list. start.") + JobID := ctx.Query("JobID") + + VersionListTasks, count, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: -1, + PageSize: -1, + }, + RepoID: -1, + Type: -1, + JobType: "", + JobID: JobID, + }) + + log.Info("query return count=" + fmt.Sprint(count)) + + if err != nil { + ctx.ServerError("QueryTrainJobList:", err) + } else { + ctx.JSON(200, VersionListTasks) + } +} + func QueryTrainJobList(ctx *context.Context) { log.Info("query train job list. start.") repoId := ctx.QueryInt64("repoId") From 9f4fc77c51ed0bd541dcdb445c9eb2761fc61bc9 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 10:32:03 +0800 Subject: [PATCH 04/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/cloudbrain.go | 46 +++++++++++++++++++++++++++++++++++++++++ models/custom_migrations.go | 9 -------- routers/repo/ai_model_manage.go | 22 ++------------------ 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 44b669181..812be2238 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -903,6 +903,52 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { return cloudbrains, count, nil } +func QueryModelTrainJobVersionList(jobId string) ([]*CloudbrainInfo, int, error) { + sess := x.NewSession() + defer sess.Close() + + var cond = builder.NewCond() + + cond = cond.And( + builder.Eq{"cloudbrain.job_id": jobId}, + ) + cond = cond.And( + builder.Eq{"cloudbrain.Status": "COMPLETED"}, + ) + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*CloudbrainInfo, 0) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + + return cloudbrains, int(len(cloudbrains)), nil +} + +func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) { + sess := x.NewSession() + defer sess.Close() + + var cond = builder.NewCond() + + cond = cond.And( + builder.Eq{"cloudbrain.repo_id": repoId}, + ) + cond = cond.And( + builder.Eq{"cloudbrain.Status": "COMPLETED"}, + ) + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*CloudbrainInfo, 0) + if err := sess.Distinct("job_id").Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + + return cloudbrains, int(len(cloudbrains)), nil +} + func CloudbrainsVersionList(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int, error) { sess := x.NewSession() defer sess.Close() diff --git a/models/custom_migrations.go b/models/custom_migrations.go index fe40fdf13..d0158530b 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -22,7 +22,6 @@ var customMigrations = []CustomMigration{ } var customMigrationsStatic = []CustomMigrationStatic{ - {"Alter user static table field type ", alterUserStaticTable}, {"Delete organization user history data ", deleteNotDisplayUser}, {"update issue_fixed_rate to 1 if num_issues is 0 ", updateIssueFixedRate}, } @@ -59,14 +58,6 @@ func syncTopicStruct(x *xorm.Engine) error { return err } -func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error { - alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision" - - _, err := static.Exec(alterSql) - return err - -} - func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { querySQL := "select id,name from public.user where type=1" diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 692bc6a05..cb9b15a8c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -261,16 +261,7 @@ func QueryTrainJobVersionList(ctx *context.Context) { log.Info("query train job version list. start.") JobID := ctx.Query("JobID") - VersionListTasks, count, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: -1, - PageSize: -1, - }, - RepoID: -1, - Type: -1, - JobType: "", - JobID: JobID, - }) + VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) log.Info("query return count=" + fmt.Sprint(count)) @@ -285,16 +276,7 @@ func QueryTrainJobList(ctx *context.Context) { log.Info("query train job list. start.") repoId := ctx.QueryInt64("repoId") - VersionListTasks, count, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: -1, - PageSize: -1, - }, - RepoID: repoId, - Type: -1, - JobType: "", - JobID: "", - }) + VersionListTasks, count, err := models.QueryModelTrainJobList(repoId) log.Info("query return count=" + fmt.Sprint(count)) From 26dcb2a1f246744ce28a9afd753801efafa8ccfd Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 15:25:51 +0800 Subject: [PATCH 05/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 1 + models/user_business_analysis.go | 189 +++++++++++++++++++++------------------ 2 files changed, 103 insertions(+), 87 deletions(-) diff --git a/models/models.go b/models/models.go index 4858ec104..e8a71bbd8 100755 --- a/models/models.go +++ b/models/models.go @@ -140,6 +140,7 @@ func init() { new(RepoStatistic), new(SummaryStatistic), new(UserBusinessAnalysis), + new(UserBusinessAnalysisAll), new(UserLoginLog), ) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 08be81241..608d5fcaa 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -4,13 +4,19 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) +type UserBusinessAnalysisAll struct { + UserBusinessAnalysis +} + type UserBusinessAnalysis struct { ID int64 `xorm:"pk"` @@ -93,46 +99,12 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool { return ulist[i].ID > ulist[j].ID } -func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { - log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") - - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - statictisSess.Find(&userBusinessAnalysisList) +type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll - resultMap := make(map[int64]*UserBusinessAnalysis) - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } - } - - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList +func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisAllList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool { + return ulist[i].ID > ulist[j].ID } func getLastCountDate() int64 { @@ -153,71 +125,33 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } -func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { +func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) statictisSess := xStatistic.NewSession() defer statictisSess.Close() - resultMap := make(map[int64]*UserBusinessAnalysis) - - var newAndCond = builder.NewCond() - if len(opts.UserName) > 0 { - newAndCond = newAndCond.And( - builder.Like{"name", opts.UserName}, - ) - } - if !opts.IsAll { - newAndCond = newAndCond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - newAndCond = newAndCond.And( - builder.Lte{"count_date": opts.EndTime}, - ) - } - - allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll)) if err != nil { log.Info("query error." + err.Error()) return nil, 0 } log.Info("query return total:" + fmt.Sprint(allCount)) - pageSize := 200 + pageSize := 1000 totalPage := int(allCount) / pageSize - + userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} for i := 0; i <= int(totalPage); i++ { - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). - Find(&userBusinessAnalysisList); err != nil { + userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) + if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisAllList); err != nil { return nil, 0 } - log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } + log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList))) + for _, userRecord := range userBusinessAnalysisAllList { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord) } } - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, allCount @@ -491,10 +425,91 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("insert daterecord failed." + err.Error()) return err } + + if isExistUserInAllTable(dateRecord.ID, statictisSess) { + updateCurrentData(dateRecord.ID, statictisSess, dateRecord) + } else { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) + allData := getAllData(dateRecord.ID, statictisSess) + allData.ID = dateRecord.ID + allData.CountDate = 0 + allData.DataDate = dateRecord.DataDate + allData.Email = dateRecord.Email + allData.OpenIIndex = dateRecord.OpenIIndex + allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth + allData.Name = dateRecord.Name + allData.RegistDate = dateRecord.RegistDate + + _, err = statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + return err + } + } + } return nil } +func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { + + _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + + ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + + ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + + ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + + ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + + ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + + ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + + ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + + ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + + ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + + ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + + ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + + ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + + ",login_count+=" + fmt.Sprint(currentData.LoginCount) + + " where id=" + fmt.Sprint(userId)) + + if err != nil { + log.Info("update table failed." + err.Error()) + } + +} + +func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { + + allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) + if err != nil { + return false + } + return allCount > 0 +} + +func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { + var dateRecord UserBusinessAnalysisAll + + rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") + if err != nil { + for i, row := range rows { + log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) + dateRecord.CodeMergeCount = strconv.FormatInt(string(row["code_merge_count"]), 10) + dateRecord.CommitCount = strconv.FormatInt(string(row["commit_count"]), 10) + dateRecord.IssueCount = strconv.FormatInt(string(row["issue_count"]), 10) + dateRecord.CommentCount = strconv.FormatInt(string(row["comment_count"]), 10) + dateRecord.FocusRepoCount = strconv.FormatInt(string(row["focus_repo_count"]), 10) + dateRecord.StarRepoCount = strconv.FormatInt(string(row["star_repo_count"]), 10) + dateRecord.WatchedCount = strconv.FormatInt(string(row["watched_count"]), 10) + dateRecord.CommitCodeSize = strconv.FormatInt(string(row["commit_code_size"]), 10) + dateRecord.CommitDatasetSize = strconv.FormatInt(string(row["commit_dataset_size"]), 10) + dateRecord.CommitModelCount = strconv.FormatInt(string(row["commit_model_count"]), 10) + dateRecord.SolveIssueCount = strconv.FormatInt(string(row["solve_issue_count"]), 10) + dateRecord.EncyclopediasCount = strconv.FormatInt(string(row["encyclopedias_count"]), 10) + dateRecord.CreateRepoCount = strconv.FormatInt(string(row["create_repo_count"]), 10) + dateRecord.LoginCount = strconv.FormatInt(string(row["login_count"]), 10) + } + } + return dateRecord +} + func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) } From faf1330a6a8193f5363d99cdab69c6cbb2f05f27 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 15:35:11 +0800 Subject: [PATCH 06/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 608d5fcaa..eb3f50b9f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -491,24 +491,31 @@ func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisA if err != nil { for i, row := range rows { log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) - dateRecord.CodeMergeCount = strconv.FormatInt(string(row["code_merge_count"]), 10) - dateRecord.CommitCount = strconv.FormatInt(string(row["commit_count"]), 10) - dateRecord.IssueCount = strconv.FormatInt(string(row["issue_count"]), 10) - dateRecord.CommentCount = strconv.FormatInt(string(row["comment_count"]), 10) - dateRecord.FocusRepoCount = strconv.FormatInt(string(row["focus_repo_count"]), 10) - dateRecord.StarRepoCount = strconv.FormatInt(string(row["star_repo_count"]), 10) - dateRecord.WatchedCount = strconv.FormatInt(string(row["watched_count"]), 10) - dateRecord.CommitCodeSize = strconv.FormatInt(string(row["commit_code_size"]), 10) - dateRecord.CommitDatasetSize = strconv.FormatInt(string(row["commit_dataset_size"]), 10) - dateRecord.CommitModelCount = strconv.FormatInt(string(row["commit_model_count"]), 10) - dateRecord.SolveIssueCount = strconv.FormatInt(string(row["solve_issue_count"]), 10) - dateRecord.EncyclopediasCount = strconv.FormatInt(string(row["encyclopedias_count"]), 10) - dateRecord.CreateRepoCount = strconv.FormatInt(string(row["create_repo_count"]), 10) - dateRecord.LoginCount = strconv.FormatInt(string(row["login_count"]), 10) + dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) + dateRecord.CommitCount = getInt(string(row["commit_count"])) + dateRecord.IssueCount = getInt(string(row["issue_count"])) + dateRecord.CommentCount = getInt(string(row["comment_count"])) + dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) + dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) + dateRecord.WatchedCount = getInt(string(row["watched_count"])) + dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) + dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) + dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) + dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) + dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) + dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) + dateRecord.LoginCount = getInt(string(row["login_count"])) } } return dateRecord } +func getInt(str string) int { + re, err := strconv.ParseInt(str, 10, 32) + if err != nil { + return 0 + } + return int(re) +} func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) From 077d658627eaeb1c246d445fe2506b51f4bf71ee Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 15:43:07 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 5a6b3d724..1a48de014 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -16,17 +16,6 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) -func QueryUserStaticData(ctx *context.Context) { - startDate := ctx.Query("startDate") - endDate := ctx.Query("endDate") - log.Info("startDate=" + startDate + " endDate=" + endDate) - startTime, _ := time.Parse("2006-01-02", startDate) - endTime, _ := time.Parse("2006-01-02", endDate) - endTime = endTime.AddDate(0, 0, 1) - log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) - ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) -} - func QueryUserStaticDataPage(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") From 3fdb8921a2d3418927f1555a1bd91d512cec2d26 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 15:45:04 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/routes/routes.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 90847b2e5..960284844 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,7 +792,6 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData) m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { From a212a897af865f4e0e1e1c3bd83c1bc9e14e103b Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 15:53:54 +0800 Subject: [PATCH 09/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 66 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index eb3f50b9f..f0eea6ca6 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -14,7 +14,68 @@ import ( ) type UserBusinessAnalysisAll struct { - UserBusinessAnalysis + ID int64 `xorm:"pk"` + + CountDate int64 `xorm:"pk"` + + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCommitRepo // 5 + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCreateIssue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + + //user + Email string `xorm:"NOT NULL"` + + //user + Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NULL"` } type UserBusinessAnalysis struct { @@ -488,7 +549,7 @@ func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisA var dateRecord UserBusinessAnalysisAll rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") - if err != nil { + if err == nil { for i, row := range rows { log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) @@ -509,6 +570,7 @@ func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisA } return dateRecord } + func getInt(str string) int { re, err := strconv.ParseInt(str, 10, 32) if err != nil { From c5e3e7ab67db4d99bb92768bb68fa252613107c8 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 16:28:34 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 42 ++++++++++++++++++++ routers/repo/user_data_analysis.go | 81 +++++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index f0eea6ca6..80c36fc42 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -333,6 +333,48 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } +func RefreshUserStaticAllTabel() { + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + statictisSess.Exec("delete from user_business_analysis_all") + + currentTimeNow := time.Now() + pageStartTime := getLastCountDate() + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() + + var cond = builder.NewCond() + cond = cond.And( + builder.Gte{"count_date": pageStartTime}, + ) + cond = cond.And( + builder.Lte{"count_date": pageEndTime}, + ) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). + Find(&userBusinessAnalysisList); err != nil { + return + } + for _, userRecord := range userBusinessAnalysisList { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) + allData := getAllData(userRecord.ID, statictisSess) + allData.ID = userRecord.ID + allData.CountDate = 0 + allData.DataDate = userRecord.DataDate + allData.Email = userRecord.Email + allData.OpenIIndex = userRecord.OpenIIndex + allData.GiteaAgeMonth = userRecord.GiteaAgeMonth + allData.Name = userRecord.Name + allData.RegistDate = userRecord.RegistDate + + _, err := statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } +} + func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { log.Info("start to count other user info data") diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 1a48de014..6d5c96752 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -16,6 +16,77 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) +func DownloadAllUserData(ctx *context.Context) { + re, count := models.QueryUserStaticDataAll(pageOpts) + log.Info("return count=" + fmt.Sprint(count)) + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.static.sheetname") + index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + dataHeader := map[string]string{ + "A1": ctx.Tr("user.static.id"), + "B1": ctx.Tr("user.static.name"), + "C1": ctx.Tr("user.static.codemergecount"), + "D1": ctx.Tr("user.static.commitcount"), + "E1": ctx.Tr("user.static.issuecount"), + "F1": ctx.Tr("user.static.commentcount"), + "G1": ctx.Tr("user.static.focusrepocount"), + "H1": ctx.Tr("user.static.starrepocount"), + "I1": ctx.Tr("user.static.logincount"), + "J1": ctx.Tr("user.static.watchedcount"), + "K1": ctx.Tr("user.static.commitcodesize"), + "L1": ctx.Tr("user.static.solveissuecount"), + "M1": ctx.Tr("user.static.encyclopediascount"), + "N1": ctx.Tr("user.static.createrepocount"), + "O1": ctx.Tr("user.static.openiindex"), + "P1": ctx.Tr("user.static.registdate"), + "Q1": ctx.Tr("user.static.countdate"), + } + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + + for i, userRecord := range re { + rows := fmt.Sprint(i + 2) + + xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) + xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) + xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount) + xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount) + xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount) + xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount) + xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount) + xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount) + xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount) + xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize) + xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount) + xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) + xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) + xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) + + formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) + + formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + var filename string + + filename = sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx" + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + if _, err := xlsx.WriteTo(ctx.Resp); err != nil { + log.Info("writer exel error." + err.Error()) + } +} + func QueryUserStaticDataPage(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") @@ -120,8 +191,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) - formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") - xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) + formatTime = userRecord.DataDate + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01") } //设置默认打开的表单 @@ -160,6 +231,12 @@ func QueryUserStaticDataPage(ctx *context.Context) { } func TimingCountDataByDateAndReCount(date string, isReCount bool) { + + if date == "refreshAll" { + models.RefreshUserStaticAllTabel() + return + } + t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) From 778782ba9fa6a567313446860bab2534f963ceac Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 16:30:21 +0800 Subject: [PATCH 11/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 80c36fc42..84b201441 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -337,7 +337,7 @@ func RefreshUserStaticAllTabel() { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - + log.Info("delete all data from table: user_business_analysis_all") statictisSess.Exec("delete from user_business_analysis_all") currentTimeNow := time.Now() @@ -356,6 +356,7 @@ func RefreshUserStaticAllTabel() { Find(&userBusinessAnalysisList); err != nil { return } + log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) for _, userRecord := range userBusinessAnalysisList { log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) allData := getAllData(userRecord.ID, statictisSess) @@ -373,6 +374,7 @@ func RefreshUserStaticAllTabel() { log.Info("insert all data failed." + err.Error()) } } + log.Info("refresh all data finished.") } func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { From 430b9f08dd047799c0dd61ce63d70fca96050254 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 16:32:11 +0800 Subject: [PATCH 12/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 71 -------------------------------------- 1 file changed, 71 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 6d5c96752..d27d76b9c 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -16,77 +16,6 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) -func DownloadAllUserData(ctx *context.Context) { - re, count := models.QueryUserStaticDataAll(pageOpts) - log.Info("return count=" + fmt.Sprint(count)) - //writer exec file. - xlsx := excelize.NewFile() - sheetName := ctx.Tr("user.static.sheetname") - index := xlsx.NewSheet(sheetName) - xlsx.DeleteSheet("Sheet1") - dataHeader := map[string]string{ - "A1": ctx.Tr("user.static.id"), - "B1": ctx.Tr("user.static.name"), - "C1": ctx.Tr("user.static.codemergecount"), - "D1": ctx.Tr("user.static.commitcount"), - "E1": ctx.Tr("user.static.issuecount"), - "F1": ctx.Tr("user.static.commentcount"), - "G1": ctx.Tr("user.static.focusrepocount"), - "H1": ctx.Tr("user.static.starrepocount"), - "I1": ctx.Tr("user.static.logincount"), - "J1": ctx.Tr("user.static.watchedcount"), - "K1": ctx.Tr("user.static.commitcodesize"), - "L1": ctx.Tr("user.static.solveissuecount"), - "M1": ctx.Tr("user.static.encyclopediascount"), - "N1": ctx.Tr("user.static.createrepocount"), - "O1": ctx.Tr("user.static.openiindex"), - "P1": ctx.Tr("user.static.registdate"), - "Q1": ctx.Tr("user.static.countdate"), - } - for k, v := range dataHeader { - //设置单元格的值 - xlsx.SetCellValue(sheetName, k, v) - } - - for i, userRecord := range re { - rows := fmt.Sprint(i + 2) - - xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) - xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) - xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) - xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount) - xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount) - xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount) - xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount) - xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount) - xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount) - xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount) - xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize) - xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount) - xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) - xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) - xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) - - formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") - xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) - - formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") - xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) - } - - //设置默认打开的表单 - xlsx.SetActiveSheet(index) - var filename string - - filename = sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx" - - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - if _, err := xlsx.WriteTo(ctx.Resp); err != nil { - log.Info("writer exel error." + err.Error()) - } -} - func QueryUserStaticDataPage(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") From 61df860930b0d9444ca64e6811dc4a0ac450bb22 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 16:41:02 +0800 Subject: [PATCH 13/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 84b201441..f81683242 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -198,6 +198,9 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return nil, 0 } log.Info("query return total:" + fmt.Sprint(allCount)) + if allCount == 0 { + RefreshUserStaticAllTabel() + } pageSize := 1000 totalPage := int(allCount) / pageSize userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} From 89b3f1b8a3db15c05f3ec0668670324eb8a08220 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 17:46:27 +0800 Subject: [PATCH 14/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index d27d76b9c..22f748ad4 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -46,6 +46,10 @@ func QueryUserStaticDataPage(ctx *context.Context) { } endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local) endTime = endTime.AddDate(0, 0, 1) + if startDate == endDate { + startTime = startTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 1) + } isAll = false log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) } @@ -218,6 +222,10 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } + + if isReCount { + models.RefreshUserStaticAllTabel() + } } func TimingCountDataByDate(date string) { From b33034b9c8b95b05715cec51d10507abed2d9090 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 25 Nov 2021 09:00:55 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=98=A8=E5=A4=A9=E6=95=B0=E6=8D=AE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 22f748ad4..c893594bc 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -39,6 +39,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { endTime = time.Now() } else { startTime, _ = time.ParseInLocation("2006-01-02", startDate, time.Local) + startTime = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 12, 0, 0, 0, startTime.Location()) settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime) if startTime.Unix() < settingStartTime.Unix() { startTime = settingStartTime @@ -46,10 +47,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { } endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local) endTime = endTime.AddDate(0, 0, 1) - if startDate == endDate { - startTime = startTime.AddDate(0, 0, 1) - endTime = endTime.AddDate(0, 0, 1) - } + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 23, 59, 59, 0, startTime.Location()) + isAll = false log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) } From be4a352947c33cfdf5783f347c59ae18bdc95a32 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 25 Nov 2021 10:06:11 +0800 Subject: [PATCH 16/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 22 ++++++++++++++++++++-- routers/repo/user_data_analysis.go | 18 +----------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index f81683242..15f67e858 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -89,7 +89,7 @@ type UserBusinessAnalysis struct { //action :ActionCommitRepo // 5 CommitCount int `xorm:"NOT NULL DEFAULT 0"` - //action :ActionCreateIssue // 10 + //action :ActionCreateIssue // 6 IssueCount int `xorm:"NOT NULL DEFAULT 0"` //comment table current date @@ -406,7 +406,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, DataDate := startTime.Format("2006-01-02") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap := queryAction(start_unix, end_unix, 5) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryAction(start_unix, end_unix, 6) CommentCountMap := queryComment(start_unix, end_unix) @@ -674,6 +674,24 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + resultMap := make(map[int64]int) + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + return resultMap +} + func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index c893594bc..2f37dc15e 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "time" "code.gitea.io/gitea/models" @@ -129,23 +128,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { //设置默认打开的表单 xlsx.SetActiveSheet(index) - var filename string - nowTime := time.Now() - nowZeroTime := time.Date(nowTime.Year(), nowTime.Month(), nowTime.Day(), 0, 0, 0, 0, nowTime.Location()) - if endTime.Unix() >= nowZeroTime.Unix() { - endDate = nowZeroTime.AddDate(0, 0, -1).Format("2006-01-02") - } - - if isAll { - filename = sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx" - } else { - filename = sheetName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" - } - - if len(userName) > 0 { - filename = sheetName + "_" + userName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" - } + filename := sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx" ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) ctx.Resp.Header().Set("Content-Type", "application/octet-stream")