From 7ea98940a408cd7410d1621e97a7fa9403e822cb Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 10:28:21 +0800 Subject: [PATCH 01/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 134 +++++++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 28 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 1bebb9f3e..56a88856f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -186,6 +186,75 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode formatTime = userRecord.DataDate xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime) } + +func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord *models.UserBusinessAnalysis) { + rows := fmt.Sprint(row) + var tmp byte + tmp = 0 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ID) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Name) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, fmt.Sprintf("%.2f", userRecord.UserIndexPrimitive)) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CodeMergeCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.IssueCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommentCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusRepoCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.StarRepoCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.LoginCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.WatchedCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitCodeSize) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.SolveIssueCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.EncyclopediasCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CreateRepoCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CloudBrainTaskNum) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600)) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitDatasetNum) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectedDataset) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.RecommendDataset) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectImage) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectedImage) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.RecommendImage) + tmp = tmp + 1 + formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime[0:len(formatTime)-3]) + tmp = tmp + 1 + + formatTime = userRecord.DataDate + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime) +} + func getColumn(tmp byte) string { var tmpA byte tmpA = 'A' @@ -481,36 +550,13 @@ func QueryUserStaticDataPage(ctx *context.Context) { } if IsReturnFile { - re, count := models.QueryUserStaticDataAll(pageOpts) - log.Info("return count=" + fmt.Sprint(count)) - //writer exec file. - xlsx := excelize.NewFile() + //re, count := models.QueryUserStaticDataAll(pageOpts) + re, count := models.QueryUserStaticDataPage(pageOpts) sheetName := ctx.Tr("user.static.sheetname") - index := xlsx.NewSheet(sheetName) - xlsx.DeleteSheet("Sheet1") - - dataHeader := getExcelHeader(ctx) - for k, v := range dataHeader { - //设置单元格的值 - xlsx.SetCellValue(sheetName, k, v) - } - - for i, userRecord := range re { - row := i + 2 - writeExcel(row, xlsx, sheetName, userRecord) - } - - //设置默认打开的表单 - xlsx.SetActiveSheet(index) - - filename := sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx" - - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - if _, err := xlsx.WriteTo(ctx.Resp); err != nil { - log.Info("writer exel error." + err.Error()) - } + filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" + go writeFileToDisk(ctx, count, re, filename) + ctx.JSON(http.StatusOK, filename) } else { mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) @@ -520,6 +566,38 @@ func QueryUserStaticDataPage(ctx *context.Context) { } } +func writeFileToDisk(ctx *context.Context, count int64, re []*models.UserBusinessAnalysis, filename string) { + log.Info("return count=" + fmt.Sprint(count)) + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.static.sheetname") + index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + + dataHeader := getExcelHeader(ctx) + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + + for i, userRecord := range re { + row := i + 2 + writeExcelPage(row, xlsx, sheetName, userRecord) + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + + //ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) + //ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + filename = setting.AppDataPath + "/useranalysis/" + filename + if err := xlsx.SaveAs(filename); err != nil { + log.Info("writer exel error." + err.Error()) + } else { + log.Info("write to file succeed, filepath=" + filename) + } +} + func TimingCountDataByDateAndReCount(date string, isReCount bool) { t, _ := time.Parse("2006-01-02", date) From 859f5ed7a18361ea71a997b25eaff1fb9139b11f Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 10:53:57 +0800 Subject: [PATCH 02/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 1 + routers/repo/user_data_analysis.go | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index d2c6e3633..8bf12c9d0 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -559,6 +559,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_metrics_all", operationReq, repo_ext.QueryUserMetricsAll) m.Get("/query_user_metrics_page", operationReq, repo_ext.QueryUserMetricDataPage) + m.Get("/download_user_define_file", operationReq, repo_ext.DownloadUserDefineFile) m.Get("/query_user_rank_list", operationReq, repo_ext.QueryRankingList) m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) m.Get("/query_user_current_month", operationReq, repo_ext.QueryUserStaticCurrentMonth) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 56a88856f..15032df6b 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/url" + "os" "time" "code.gitea.io/gitea/models" @@ -16,7 +17,8 @@ import ( ) const ( - PAGE_SIZE = 2000 + PAGE_SIZE = 2000 + Excel_File_Path = "/useranalysis/" ) func getUserMetricsExcelHeader(ctx *context.Context) map[string]string { @@ -399,6 +401,25 @@ func QueryRankingList(ctx *context.Context) { ctx.JSON(http.StatusOK, mapInterface) } +func DownloadUserDefineFile(ctx *context.Context) { + filename := ctx.Query("filename") + length := len(filename) + if filename[0:1] == "\"" { + filename = filename[1 : length-1] + } + allFilename := setting.AppDataPath + Excel_File_Path + filename + log.Info("allFilename=" + allFilename) + _, err := os.Stat(allFilename) + if err != nil { //文件不存在 + log.Info("file not exist.") + ctx.JSON(http.StatusOK, "File Not Exist.") + } else { + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+filename) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + ctx.ServeFile(allFilename, filename) + } +} + func QueryUserMetricsCurrentMonth(ctx *context.Context) { currentTimeNow := time.Now() @@ -590,7 +611,8 @@ func writeFileToDisk(ctx *context.Context, count int64, re []*models.UserBusines //ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) //ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - filename = setting.AppDataPath + "/useranalysis/" + filename + filename = setting.AppDataPath + Excel_File_Path + filename + os.Mkdir(setting.AppDataPath+Excel_File_Path, 0755) if err := xlsx.SaveAs(filename); err != nil { log.Info("writer exel error." + err.Error()) } else { From 8e6fcafca3f0742f0486584c1eb5cb262762df75 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 11:00:48 +0800 Subject: [PATCH 03/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 15032df6b..fa2d43589 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -577,7 +577,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" go writeFileToDisk(ctx, count, re, filename) - ctx.JSON(http.StatusOK, filename) + ctx.Redirect("/api/v1/download_user_define_file?filename=" + filename) } else { mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) From f648f0982788e59d5d2d4c0766fc5e49fa29d330 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 11:05:42 +0800 Subject: [PATCH 04/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index fa2d43589..4042955ec 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -575,9 +575,9 @@ func QueryUserStaticDataPage(ctx *context.Context) { re, count := models.QueryUserStaticDataPage(pageOpts) sheetName := ctx.Tr("user.static.sheetname") filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" - + os.Remove(setting.AppDataPath + Excel_File_Path + filename) go writeFileToDisk(ctx, count, re, filename) - ctx.Redirect("/api/v1/download_user_define_file?filename=" + filename) + ctx.JSON(http.StatusOK, filename) } else { mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) From 0689e0047650ef28b0d6d0e5a818ed33e328b214 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 11:08:04 +0800 Subject: [PATCH 05/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 4042955ec..5d2e336fd 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -577,7 +577,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" os.Remove(setting.AppDataPath + Excel_File_Path + filename) go writeFileToDisk(ctx, count, re, filename) - ctx.JSON(http.StatusOK, filename) + ctx.JSON(http.StatusOK, setting.AppURL+"/api/v1/download_user_define_file?filename="+filename) } else { mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) From 821d9088431864405e144589aba8fbffc5dc8e76 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 11:09:35 +0800 Subject: [PATCH 06/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 5d2e336fd..e0c72da57 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -577,7 +577,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" os.Remove(setting.AppDataPath + Excel_File_Path + filename) go writeFileToDisk(ctx, count, re, filename) - ctx.JSON(http.StatusOK, setting.AppURL+"/api/v1/download_user_define_file?filename="+filename) + ctx.JSON(http.StatusOK, setting.AppURL+"api/v1/download_user_define_file?filename="+filename) } else { mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) From 1d4be6af12a80ac948f4aef54ca600f527d098c2 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 11:32:02 +0800 Subject: [PATCH 07/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index d0158530b..412bedce1 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -1,8 +1,6 @@ package models import ( - "fmt" - "code.gitea.io/gitea/modules/log" "xorm.io/xorm" ) @@ -22,7 +20,6 @@ var customMigrations = []CustomMigration{ } var customMigrationsStatic = []CustomMigrationStatic{ - {"Delete organization user history data ", deleteNotDisplayUser}, {"update issue_fixed_rate to 1 if num_issues is 0 ", updateIssueFixedRate}, } @@ -36,7 +33,6 @@ func MigrateCustom(x *xorm.Engine) { } } - } func MigrateCustomStatic(x *xorm.Engine, static *xorm.Engine) { @@ -58,24 +54,6 @@ func syncTopicStruct(x *xorm.Engine) error { return err } -func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { - - querySQL := "select id,name from public.user where type=1" - rows, err := x.Query(querySQL) - if err != nil { - log.Info("select db failed,err:", err) - return err - } - - for i, userRow := range rows { - log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["name"])) - deleteSql := "delete from user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" - static.Exec(deleteSql) - } - - return nil -} - func updateIssueFixedRate(x *xorm.Engine, static *xorm.Engine) error { updateSQL := "update repo_statistic set issue_fixed_rate=1.0 where num_issues=0" _, err := static.Exec(updateSQL) From d90706658f4345008ec978b6694be9eb026508bf Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 11:58:33 +0800 Subject: [PATCH 08/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 59 ++++++++++++++++++++++++++++++++++++++ routers/repo/user_data_analysis.go | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 56e591dea..02317198a 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -407,6 +407,65 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return userBusinessAnalysisReturnList, allCount } +func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Gte{"count_date": opts.StartTime}, + ) + cond = cond.And( + builder.Lte{"count_date": opts.EndTime}, + ) + count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + resultMap := make(map[int64]*UserBusinessAnalysis) + if count > 0 { + pageSize := 1000 + totalPage := int(count) / pageSize + for i := 0; i <= int(totalPage); i++ { + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + } + } + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} + for _, v := range resultMap { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) + } + sort.Sort(userBusinessAnalysisReturnList) + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, count +} + func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index e0c72da57..1f03fbf44 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -572,7 +572,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { if IsReturnFile { //re, count := models.QueryUserStaticDataAll(pageOpts) - re, count := models.QueryUserStaticDataPage(pageOpts) + re, count := models.QueryUserStaticDataForUserDefine(pageOpts) sheetName := ctx.Tr("user.static.sheetname") filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" os.Remove(setting.AppDataPath + Excel_File_Path + filename) From 3efb1697f1c15b5b5576ae9ce8dc52205dafd1bc Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:01:25 +0800 Subject: [PATCH 09/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 1 + models/user_business_analysis.go | 178 +++++++++++++++++++++++---------- models/user_business_struct.go | 66 ++++++++++++ options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/api/v1/api.go | 1 + routers/repo/user_data_analysis.go | 37 ++++--- web_src/js/components/UserAnalysis.vue | 6 ++ 8 files changed, 225 insertions(+), 66 deletions(-) diff --git a/models/models.go b/models/models.go index 2a2e119fb..9d255c5e6 100755 --- a/models/models.go +++ b/models/models.go @@ -157,6 +157,7 @@ func init() { new(UserBusinessAnalysisCurrentMonth), new(UserBusinessAnalysisCurrentWeek), new(UserBusinessAnalysisYesterday), + new(UserBusinessAnalysisLastWeek), new(UserLoginLog), new(UserMetrics), new(UserAnalysisPara), diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 02317198a..b986baab5 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -407,63 +407,132 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return userBusinessAnalysisReturnList, allCount } -func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { - log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - var cond = builder.NewCond() - cond = cond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - cond = cond.And( - builder.Lte{"count_date": opts.EndTime}, - ) - count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) +func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]*UserBusinessAnalysis, int64) { + log.Info("start to count other user info data") + sess := x.NewSession() + defer sess.Close() + + currentTimeNow := time.Now() + log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) + + start_unix := opts.StartTime + + end_unix := opts.EndTime + CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryCreateIssue(start_unix, end_unix) + + CommentCountMap := queryComment(start_unix, end_unix) + FocusRepoCountMap := queryWatch(start_unix, end_unix) + StarRepoCountMap := queryStar(start_unix, end_unix) + WatchedCountMap, WatchOtherMap := queryFollow(start_unix, end_unix) + + CommitCodeSizeMap, err := GetAllUserKPIStats() if err != nil { - log.Info("query error." + err.Error()) - return nil, 0 + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + CommitCodeSizeMapJson, _ := json.Marshal(CommitCodeSizeMap) + log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson)) } - resultMap := make(map[int64]*UserBusinessAnalysis) - if count > 0 { - pageSize := 1000 - totalPage := int(count) / pageSize - for i := 0; i <= int(totalPage); i++ { - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). - Find(&userBusinessAnalysisList); err != nil { - return nil, 0 + CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + AiModelManageMap := queryUserModel(start_unix, end_unix) + + CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) + RecommendDataset := queryRecommedDataSet(start_unix, end_unix) + CollectImage, CollectedImage := queryImageStars(start_unix, end_unix) + RecommendImage := queryRecommedImage(start_unix, end_unix) + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + cond := "type != 1 and and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + + ParaWeight := getParaWeight() + ResultList := make([]*UserBusinessAnalysis, 0) + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysis + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.CountDate = CountDate.Unix() + + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + + dateRecord.CodeMergeCount = getMapValue(dateRecord.ID, CodeMergeCountMap) + dateRecord.CommitCount = getMapValue(dateRecord.ID, CommitCountMap) + dateRecord.IssueCount = getMapValue(dateRecord.ID, IssueCountMap) + dateRecord.CommentCount = getMapValue(dateRecord.ID, CommentCountMap) + dateRecord.FocusRepoCount = getMapValue(dateRecord.ID, FocusRepoCountMap) + dateRecord.StarRepoCount = getMapValue(dateRecord.ID, StarRepoCountMap) + dateRecord.WatchedCount = getMapValue(dateRecord.ID, WatchedCountMap) + dateRecord.FocusOtherUser = getMapValue(dateRecord.ID, WatchOtherMap) + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) } - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } + dateRecord.CommitDatasetSize = getMapValue(dateRecord.ID, CommitDatasetSizeMap) + dateRecord.CommitDatasetNum = getMapValue(dateRecord.ID, CommitDatasetNumMap) + dateRecord.SolveIssueCount = getMapValue(dateRecord.ID, SolveIssueCountMap) + + dateRecord.EncyclopediasCount = getMapKeyStringValue(dateRecord.Name, wikiCountMap) + + dateRecord.CreateRepoCount = getMapValue(dateRecord.ID, CreateRepoCountMap) + + dateRecord.LoginCount = getMapValue(dateRecord.ID, LoginCountMap) + + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] } + + dateRecord.CloudBrainTaskNum = getMapValue(dateRecord.ID, CloudBrainTaskMap) + dateRecord.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecord.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecord.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecord.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecord.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) + dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + + dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) + dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) + dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset) + dateRecord.CollectImage = getMapValue(dateRecord.ID, CollectImage) + dateRecord.CollectedImage = getMapValue(dateRecord.ID, CollectedImage) + dateRecord.RecommendImage = getMapValue(dateRecord.ID, RecommendImage) + + dateRecord.UserIndexPrimitive = getUserIndex(dateRecord, ParaWeight) + ResultList = append(ResultList, &dateRecord) + } + + indexTotal += PAGE_SIZE + if indexTotal >= count { + break } } - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList, count + + return ResultList, int64(len(ResultList)) } func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { @@ -792,7 +861,12 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, userMetrics map[stri pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) refreshUserStaticTable(wikiCountMap, "user_business_analysis_current_week", pageStartTime, pageEndTime, userMetrics) + pageEndTime = pageStartTime + pageStartTime = pageStartTime.AddDate(0, 0, -7) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_last_week", pageStartTime, pageEndTime, userMetrics) + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30) + pageEndTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) refreshUserStaticTable(wikiCountMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime, userMetrics) pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -1) @@ -937,7 +1011,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("has activity." + userRecord.Name) addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID) } - if userRecord.IsActive { + if !userRecord.IsActive { continue } statictisSess.Delete(&dateRecord) diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 70f806c78..fec361bca 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -394,6 +394,72 @@ type UserBusinessAnalysisYesterday struct { RecommendImage int `xorm:"NOT NULL DEFAULT 0"` } +type UserBusinessAnalysisLastWeek struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserIndexPrimitive float64 `xorm:"NOT NULL DEFAULT 0"` + + UserLocation string `xorm:"NULL"` + + FocusOtherUser int `xorm:"NOT NULL DEFAULT 0"` + CollectDataset int `xorm:"NOT NULL DEFAULT 0"` + CollectedDataset int `xorm:"NOT NULL DEFAULT 0"` + RecommendDataset int `xorm:"NOT NULL DEFAULT 0"` + CollectImage int `xorm:"NOT NULL DEFAULT 0"` + CollectedImage int `xorm:"NOT NULL DEFAULT 0"` + RecommendImage int `xorm:"NOT NULL DEFAULT 0"` +} + type UserAnalysisPara struct { Key string `xorm:"NOT NULL"` Value float64 `xorm:"NOT NULL DEFAULT 0"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ef3ad7705..265e7ed36 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -522,6 +522,7 @@ static.RecommendImage=Recommended Image Count static.all=All static.public.user_business_analysis_current_month=Current_Month static.public.user_business_analysis_current_week=Current_Week +static.public.user_business_analysis_last_week=Last_Week static.public.user_business_analysis_current_year=Current_Year static.public.user_business_analysis_last30_day=Last_30_day static.public.user_business_analysis_last_month=Last_Month diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2f0bbe91d..1b26a930c 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -527,6 +527,7 @@ static.RecommendImage=被推荐镜像数 static.all=所有 static.public.user_business_analysis_current_month=本月 static.public.user_business_analysis_current_week=本周 +static.public.user_business_analysis_last_week=上周 static.public.user_business_analysis_current_year=今年 static.public.user_business_analysis_last30_day=近30天 static.public.user_business_analysis_last_month=上月 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8bf12c9d0..03f13b195 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -564,6 +564,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) m.Get("/query_user_current_month", operationReq, repo_ext.QueryUserStaticCurrentMonth) m.Get("/query_user_current_week", operationReq, repo_ext.QueryUserStaticCurrentWeek) + m.Get("/query_user_last_week", operationReq, repo_ext.QueryUserStaticCurrentWeek) m.Get("/query_user_current_year", operationReq, repo_ext.QueryUserStaticCurrentYear) m.Get("/query_user_last30_day", operationReq, repo_ext.QueryUserStaticLast30Day) m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 1f03fbf44..65764d58a 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -455,6 +455,10 @@ func QueryUserMetricsCurrentWeek(ctx *context.Context) { func QueryUserStaticCurrentWeek(ctx *context.Context) { queryUserDataPage(ctx, "public.user_business_analysis_current_week", new(models.UserBusinessAnalysisCurrentWeek)) } +func QueryUserStaticLastWeek(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_last_week", new(models.UserBusinessAnalysisLastWeek)) +} + func QueryUserMetricsCurrentYear(ctx *context.Context) { currentTimeNow := time.Now() pageStartTime := time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location()) @@ -572,7 +576,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { if IsReturnFile { //re, count := models.QueryUserStaticDataAll(pageOpts) - re, count := models.QueryUserStaticDataForUserDefine(pageOpts) + wikiMap, _ := queryWikiCountMap(startTime, endTime) + re, count := models.QueryUserStaticDataForUserDefine(pageOpts, wikiMap) sheetName := ctx.Tr("user.static.sheetname") filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" os.Remove(setting.AppDataPath + Excel_File_Path + filename) @@ -620,22 +625,14 @@ func writeFileToDisk(ctx *context.Context, count int64, re []*models.UserBusines } } -func TimingCountDataByDateAndReCount(date string, isReCount bool) { - - t, _ := time.Parse("2006-01-02", date) - startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) - - endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location()) - - //query wiki data - log.Info("start to time count data") +func queryWikiCountMap(startTime time.Time, endTime time.Time) (map[string]int, error) { wikiMap := make(map[string]int) warnEmailMessage := "用户统计信息入库失败，请尽快定位。" repoList, err := models.GetAllRepositories() if err != nil { log.Error("query repo error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) - return + return nil, err } log.Info("start to query wiki data") for _, repoRecord := range repoList { @@ -643,7 +640,7 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { time, err := git.GetLatestCommitTime(wikiPath) if err == nil { log.Info("last commit time:" + time.Format("2006-01-02 15:04:05") + " wikiPath=" + wikiPath) - if time.After(startTime) { + if time.After(startTime) && time.Before(endTime) { wikiRepo, _, err := FindWikiRepoCommitByWikiPath(wikiPath) if err != nil { log.Error("wiki not exist. wikiPath=" + wikiPath) @@ -668,14 +665,26 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { } } } + return wikiMap, nil +} + +func TimingCountDataByDateAndReCount(date string, isReCount bool) { + + t, _ := time.Parse("2006-01-02", date) + startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + + endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location()) + warnEmailMessage := "用户统计信息入库失败，请尽快定位。" + //query wiki data + log.Info("start to time count data") + wikiMap, err := queryWikiCountMap(startTime, endTime) //other user info data err = models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount) if err != nil { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } - log.Info("start to count all user info data") - //models.RefreshUserStaticAllTabel(wikiMap) + log.Info("end to count all user info data") } diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index 784d375fb..4d48bd3b6 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -7,6 +7,7 @@ 统计周期： + @@ -375,6 +376,11 @@ this.dataUrl = '../../api/v1/query_user_current_week'; break } + case "last_week_usr":{ + this.value_time=[] + this.dataUrl = '../../api/v1/query_user_last_week'; + break + } case "current_month_usr":{ this.value_time=[] this.dataUrl = '../../api/v1/query_user_current_month'; From f3dffd7db87110b24fa4cb3c61cc91297a06a04e Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:09:05 +0800 Subject: [PATCH 10/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 2 +- models/user_business_analysis.go | 8 ++++---- routers/repo/user_data_analysis.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/models/models.go b/models/models.go index 9d255c5e6..46444c58b 100755 --- a/models/models.go +++ b/models/models.go @@ -157,7 +157,7 @@ func init() { new(UserBusinessAnalysisCurrentMonth), new(UserBusinessAnalysisCurrentWeek), new(UserBusinessAnalysisYesterday), - new(UserBusinessAnalysisLastWeek), + //new(UserBusinessAnalysisLastWeek), new(UserLoginLog), new(UserMetrics), new(UserAnalysisPara), diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index b986baab5..2984fdc3f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -407,7 +407,7 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return userBusinessAnalysisReturnList, allCount } -func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]*UserBusinessAnalysis, int64) { +func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]UserBusinessAnalysis, int64) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() @@ -457,7 +457,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi count, err := sess.Where(cond).Count(new(User)) ParaWeight := getParaWeight() - ResultList := make([]*UserBusinessAnalysis, 0) + ResultList := make([]UserBusinessAnalysis, 0) var indexTotal int64 indexTotal = 0 for { @@ -523,7 +523,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.RecommendImage = getMapValue(dateRecord.ID, RecommendImage) dateRecord.UserIndexPrimitive = getUserIndex(dateRecord, ParaWeight) - ResultList = append(ResultList, &dateRecord) + ResultList = append(ResultList, dateRecord) } indexTotal += PAGE_SIZE @@ -531,7 +531,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi break } } - + log.Info("query user define,count=" + fmt.Sprint((ResultList))) return ResultList, int64(len(ResultList)) } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 65764d58a..430c8b2cc 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -189,7 +189,7 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime) } -func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord *models.UserBusinessAnalysis) { +func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord models.UserBusinessAnalysis) { rows := fmt.Sprint(row) var tmp byte tmp = 0 @@ -592,7 +592,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { } } -func writeFileToDisk(ctx *context.Context, count int64, re []*models.UserBusinessAnalysis, filename string) { +func writeFileToDisk(ctx *context.Context, count int64, re []models.UserBusinessAnalysis, filename string) { log.Info("return count=" + fmt.Sprint(count)) //writer exec file. xlsx := excelize.NewFile() From 16ad0e80426449ba13f868871dce6a30eef68904 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:12:40 +0800 Subject: [PATCH 11/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 8 ++++---- routers/repo/user_data_analysis.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 2984fdc3f..1dadb8e64 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -407,7 +407,7 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return userBusinessAnalysisReturnList, allCount } -func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]UserBusinessAnalysis, int64) { +func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]*UserBusinessAnalysis, int64) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() @@ -453,11 +453,11 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi statictisSess := xStatistic.NewSession() defer statictisSess.Close() - cond := "type != 1 and and is_active=true" + cond := "type != 1 and is_active=true" count, err := sess.Where(cond).Count(new(User)) ParaWeight := getParaWeight() - ResultList := make([]UserBusinessAnalysis, 0) + ResultList := make([]*UserBusinessAnalysis, 0) var indexTotal int64 indexTotal = 0 for { @@ -523,7 +523,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.RecommendImage = getMapValue(dateRecord.ID, RecommendImage) dateRecord.UserIndexPrimitive = getUserIndex(dateRecord, ParaWeight) - ResultList = append(ResultList, dateRecord) + ResultList = append(ResultList, &dateRecord) } indexTotal += PAGE_SIZE diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 430c8b2cc..65764d58a 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -189,7 +189,7 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime) } -func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord models.UserBusinessAnalysis) { +func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord *models.UserBusinessAnalysis) { rows := fmt.Sprint(row) var tmp byte tmp = 0 @@ -592,7 +592,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { } } -func writeFileToDisk(ctx *context.Context, count int64, re []models.UserBusinessAnalysis, filename string) { +func writeFileToDisk(ctx *context.Context, count int64, re []*models.UserBusinessAnalysis, filename string) { log.Info("return count=" + fmt.Sprint(count)) //writer exec file. xlsx := excelize.NewFile() From 516aa059244e033e2892966e73cfbe9b778a2c08 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:15:47 +0800 Subject: [PATCH 12/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 65764d58a..f631591d1 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -551,7 +551,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { startDate = settingStartTime.Format("2006-01-02") } endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local) - endTime = endTime.AddDate(0, 0, 1) + //endTime = endTime.AddDate(0, 0, 1) endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 23, 59, 59, 0, startTime.Location()) isAll = false From 4a2c126a7272e4c4e1713b0088d6af840afcc5bd Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:16:42 +0800 Subject: [PATCH 13/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index 4d48bd3b6..587784712 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -7,7 +7,6 @@ 统计周期： - From df2603806fdf8b9cac2643b3cd4e5ebc7fc71372 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:26:08 +0800 Subject: [PATCH 14/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 1dadb8e64..a6caaa733 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -541,9 +541,9 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus statictisSess := xStatistic.NewSession() defer statictisSess.Close() - currentTimeNow := time.Now() - pageStartTime := getLastCountDate() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() + //currentTimeNow := time.Now() + //pageStartTime := getLastCountDate() + //pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() var cond = builder.NewCond() if len(opts.UserName) > 0 { @@ -552,10 +552,10 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus ) } cond = cond.And( - builder.Gte{"count_date": pageStartTime}, + builder.Gte{"count_date": opts.StartTime}, ) cond = cond.And( - builder.Lte{"count_date": pageEndTime}, + builder.Lte{"count_date": opts.EndTime}, ) count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) From 07bba7e8b88149c5399b813cc009bfa7ebcf2bb5 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:28:22 +0800 Subject: [PATCH 15/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index 587784712..d67daac46 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -27,11 +27,9 @@ - - + - 下载报告 - 下载报告 + 下载报告 From d9f1aa156e609c6f638c24c45c7b63250bc70866 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:35:08 +0800 Subject: [PATCH 16/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a6caaa733..59a9268ba 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -575,7 +575,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus } userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("count_date,id desc"). Find(&userBusinessAnalysisList); err != nil { return nil, 0 } From 645bde4ff378d6b16cd901263eb2420b037ae61a Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 15:42:15 +0800 Subject: [PATCH 17/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index d67daac46..587784712 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -27,9 +27,11 @@ - + + - 下载报告 + 下载报告 + 下载报告 From eac3bc9dbf14eb903f2a4b7f991be84c3675c09a Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 16:42:21 +0800 Subject: [PATCH 18/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 2 +- models/repo_activity_custom.go | 4 ++-- models/user_business_analysis.go | 23 +++++++++++++++++++---- modules/git/repo_stats_custom.go | 19 +++++++++++++------ 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/models/models.go b/models/models.go index 46444c58b..9d255c5e6 100755 --- a/models/models.go +++ b/models/models.go @@ -157,7 +157,7 @@ func init() { new(UserBusinessAnalysisCurrentMonth), new(UserBusinessAnalysisCurrentWeek), new(UserBusinessAnalysisYesterday), - //new(UserBusinessAnalysisLastWeek), + new(UserBusinessAnalysisLastWeek), new(UserLoginLog), new(UserMetrics), new(UserAnalysisPara), diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index ac39a8de7..cbe00b9d9 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -211,7 +211,7 @@ func setKeyContributerDict(contributorDistinctDict map[string]int, email string, } } -func GetAllUserKPIStats() (map[string]*git.UserKPIStats, error) { +func GetAllUserKPIStats(startTime time.Time, endTime time.Time) (map[string]*git.UserKPIStats, error) { authors := make(map[string]*git.UserKPIStats) repositorys, err := GetAllRepositoriesByFilterCols("owner_name", "name") if err != nil { @@ -219,7 +219,7 @@ func GetAllUserKPIStats() (map[string]*git.UserKPIStats, error) { } for _, repository := range repositorys { - authorsOneRepo, err1 := git.GetUserKPIStats(repository.RepoPath()) + authorsOneRepo, err1 := git.GetUserKPIStats(repository.RepoPath(), startTime, endTime) if err1 != nil { log.Warn("get user kpi status err:"+repository.RepoPath(), err1.Error()) continue diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 59a9268ba..177b1caba 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -429,7 +429,9 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap, WatchOtherMap := queryFollow(start_unix, end_unix) - CommitCodeSizeMap, err := GetAllUserKPIStats() + StartTime := time.Unix(start_unix, 0) + EndTime := time.Unix(end_unix, 0) + CommitCodeSizeMap, err := GetAllUserKPIStats(StartTime, EndTime) if err != nil { log.Info("query commit code errr.") } else { @@ -679,7 +681,15 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS FocusRepoCountMap := queryWatch(start_unix, end_unix) StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap, WatchOtherMap := queryFollow(start_unix, end_unix) - CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix()) + CommitCodeSizeMap, err := GetAllUserKPIStats(StartTimeNextDay, EndTimeNextDay) + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + CommitCodeSizeMapJson, _ := json.Marshal(CommitCodeSizeMap) + log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson)) + } + //CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix()) CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) @@ -733,7 +743,12 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.FocusOtherUser = getMapValue(dateRecordAll.ID, WatchOtherMap) dateRecordAll.StarRepoCount = getMapValue(dateRecordAll.ID, StarRepoCountMap) dateRecordAll.WatchedCount = getMapValue(dateRecordAll.ID, WatchedCountMap) - dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap) + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + //dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap) dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap) dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap) dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap) @@ -907,7 +922,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap, WatchOtherMap := queryFollow(start_unix, end_unix) - CommitCodeSizeMap, err := GetAllUserKPIStats() + CommitCodeSizeMap, err := GetAllUserKPIStats(startTime, endTime) if err != nil { log.Info("query commit code errr.") } else { diff --git a/modules/git/repo_stats_custom.go b/modules/git/repo_stats_custom.go index d70a17052..1a7b657d5 100644 --- a/modules/git/repo_stats_custom.go +++ b/modules/git/repo_stats_custom.go @@ -58,12 +58,11 @@ func SetDevelopAge(repoPath string, stats *RepoKPIStats, fromTime time.Time) err return nil } -//获取一天内的用户贡献指标 -func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { - timeUntil := time.Now() - oneDayAgo := timeUntil.AddDate(0, 0, -1) - since := oneDayAgo.Format(time.RFC3339) - args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} +func GetUserKPIStats(repoPath string, startTime time.Time, endTime time.Time) (map[string]*UserKPIStats, error) { + + after := startTime.Format(time.RFC3339) + until := endTime.Format(time.RFC3339) + args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--after='%s'", after), fmt.Sprintf("--until=='%s'", until)} stdout, err := NewCommand(args...).RunInDirBytes(repoPath) if err != nil { return nil, err @@ -124,6 +123,14 @@ func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { } +//获取一天内的用户贡献指标 +func getUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { + timeUntil := time.Now() + oneDayAgo := timeUntil.AddDate(0, 0, -1) + + return GetUserKPIStats(repoPath, oneDayAgo, oneDayAgo) +} + func SetRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { since := fromTime.Format(time.RFC3339) args := []string{"log", "--numstat", "--no-merges", "HEAD", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} From 141762edff7bd683fc477aa2b6b43aa4b89d77ed Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 16:46:16 +0800 Subject: [PATCH 19/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 177b1caba..dd50cb964 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -419,6 +419,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi end_unix := opts.EndTime CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + DataDate := currentTimeNow.Format("2006-01-02 15:04") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryCommitAction(start_unix, end_unix, 5) @@ -472,7 +473,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.ID = userRecord.ID log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) dateRecord.CountDate = CountDate.Unix() - + dateRecord.DataDate = DataDate dateRecord.Email = userRecord.Email dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name From 4af9b1d1f0e924f4e63569b922513eb0e7c35ac0 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 16:55:12 +0800 Subject: [PATCH 20/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index dd50cb964..0775ac899 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1789,7 +1789,7 @@ func queryImageStars(start_unix int64, end_unix int64) (map[int64]int, map[int64 var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uid,dataset_id").Table(new(ImageStar)).Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + sess.Select("id,uid,image_id").Table(new(ImageStar)).Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) imageStarList := make([]*ImageStar, 0) sess.Find(&imageStarList) log.Info("query imageStarList size=" + fmt.Sprint(len(imageStarList))) From 4f999089e02c91e5c38f70bb2bc1776f749c9327 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 17:18:04 +0800 Subject: [PATCH 21/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E7=95=8C=E9=9D=A2=E4=B8=8A=E4=B8=8A?= =?UTF-8?q?=E5=91=A8=E6=A6=9C=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index 587784712..cfc91d449 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -6,6 +6,7 @@
统计周期： + From 961797d915565d895779733d734d6277cef0f8a8 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 17:21:07 +0800 Subject: [PATCH 22/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E7=95=8C=E9=9D=A2=E4=B8=8A=E4=B8=8A?= =?UTF-8?q?=E5=91=A8=E6=A6=9C=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 03f13b195..9a05aa8ae 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -564,7 +564,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) m.Get("/query_user_current_month", operationReq, repo_ext.QueryUserStaticCurrentMonth) m.Get("/query_user_current_week", operationReq, repo_ext.QueryUserStaticCurrentWeek) - m.Get("/query_user_last_week", operationReq, repo_ext.QueryUserStaticCurrentWeek) + m.Get("/query_user_last_week", operationReq, repo_ext.QueryUserStaticLastWeek) m.Get("/query_user_current_year", operationReq, repo_ext.QueryUserStaticCurrentYear) m.Get("/query_user_last30_day", operationReq, repo_ext.QueryUserStaticLast30Day) m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) From 4527452232560686714999ab9f12526cb78610b0 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 17:26:32 +0800 Subject: [PATCH 23/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E7=95=8C=E9=9D=A2=E4=B8=8A=E4=B8=8A?= =?UTF-8?q?=E5=91=A8=E6=A6=9C=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 0775ac899..a8520c10f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1030,7 +1030,12 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, if !userRecord.IsActive { continue } - statictisSess.Delete(&dateRecord) + + var deleteDateRecord UserBusinessAnalysis + deleteDateRecord.ID = userRecord.ID + deleteDateRecord.CountDate = CountDate.Unix() + statictisSess.Delete(&deleteDateRecord) + _, err = statictisSess.Insert(&dateRecord) if err != nil { log.Info("insert daterecord failed." + err.Error()) From 6180a1ee639f139f61a708729dac423f7519edea Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 17:47:43 +0800 Subject: [PATCH 24/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E6=AD=A3=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a8520c10f..b45ffa104 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -663,10 +663,8 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS log.Info("truncate all data from table: " + tableName) statictisSess.Exec("TRUNCATE TABLE " + tableName) - StartTimeNextDay := pageStartTime.AddDate(0, 0, 1) - EndTimeNextDay := pageEndTime.AddDate(0, 0, 1) - log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05") + " nextDay:" + StartTimeNextDay.Format("2006-01-02 15:04:05")) - log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05") + " nextDay:" + EndTimeNextDay.Format("2006-01-02 15:04:05")) + log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) start_unix := pageStartTime.Unix() end_unix := pageEndTime.Unix() @@ -682,7 +680,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS FocusRepoCountMap := queryWatch(start_unix, end_unix) StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap, WatchOtherMap := queryFollow(start_unix, end_unix) - CommitCodeSizeMap, err := GetAllUserKPIStats(StartTimeNextDay, EndTimeNextDay) + CommitCodeSizeMap, err := GetAllUserKPIStats(pageStartTime, pageEndTime) if err != nil { log.Info("query commit code errr.") } else { From d81e34bee04bf12549bab3bd8fff585c5a40abbe Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 23 May 2022 17:50:11 +0800 Subject: [PATCH 25/25] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E6=AD=A3=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index b45ffa104..bef23e109 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -768,6 +768,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset) dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset)