From ac7aae247d6c959c0080833fe1748206f1455a8f Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Jul 2022 14:44:53 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 47 ++++++++++++++++++++++++++++++++++---- routers/repo/user_data_analysis.go | 10 +++++++- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index cb503d669..99a9edfe8 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -110,9 +110,9 @@ type UserBusinessAnalysisAll struct { } type UserBusinessAnalysis struct { - ID int64 `xorm:"pk"` - - CountDate int64 `xorm:"pk"` + ID int64 `xorm:"pk"` + DataDate string `xorm:"pk"` + CountDate int64 `xorm:"NULL"` //action :ActionMergePullRequest // 11 CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` @@ -171,8 +171,6 @@ type UserBusinessAnalysis struct { //user Name string `xorm:"NOT NULL"` - DataDate string `xorm:"NULL"` - CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` @@ -411,6 +409,42 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return userBusinessAnalysisReturnList, allCount } +func QueryDataForUserDefineFromDb(opts *UserBusinessAnalysisQueryOptions, key string) ([]*UserBusinessAnalysis, int64) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + allCount, err := statictisSess.Where("data_date=" + key).Count(new(UserBusinessAnalysis)) + if err == nil { + if allCount > 0 { + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where("data_date="+key).OrderBy("id desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + return userBusinessAnalysisList, allCount + } + } + return nil, 0 +} + +func WriteDataToDb(dataList []*UserBusinessAnalysis, key string) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + log.Info("write to db, size=" + fmt.Sprint(len(dataList))) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + for _, data := range dataList { + data.DataDate = key + userBusinessAnalysisList = append(userBusinessAnalysisList, data) + if len(userBusinessAnalysisList) > BATCH_INSERT_SIZE { + statictisSess.Insert(userBusinessAnalysisList) + userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) + } + } + if len(userBusinessAnalysisList) > 0 { + statictisSess.Insert(userBusinessAnalysisList) + } +} + func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]*UserBusinessAnalysis, int64) { log.Info("start to count other user info data") sess := x.NewSession() @@ -954,6 +988,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, statictisSess := xStatistic.NewSession() defer statictisSess.Close() + log.Info("truncate all data from table:user_business_analysis ") + statictisSess.Exec("TRUNCATE TABLE user_business_analysis") + cond := "type != 1" count, err := sess.Where(cond).Count(new(User)) if err != nil { diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index ac1265d04..bafe46878 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -611,7 +611,15 @@ func QueryUserStaticDataPage(ctx *context.Context) { ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+"/api/v1/download_user_define_file?filename="+filename) } else { mapInterface := make(map[string]interface{}) - re, count := models.QueryUserStaticDataPage(pageOpts) + key := startTime.Format("2006-01-02") + endTime.Format("2006-01-02") + log.Info("db key =" + key) + re, count := models.QueryDataForUserDefineFromDb(pageOpts, key) + if count == 0 { + wikiMap, _ := queryWikiCountMap(startTime, endTime) + re, count = models.QueryUserStaticDataForUserDefine(pageOpts, wikiMap) + models.WriteDataToDb(re, key) + } + re, count = models.QueryDataForUserDefineFromDb(pageOpts, key) mapInterface["data"] = re mapInterface["count"] = count ctx.JSON(http.StatusOK, mapInterface) From e6abe7283b53eb1a13f1a328f9b92b699d8ca2b9 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Jul 2022 15:04:15 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 99a9edfe8..b514a4f33 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -412,12 +412,12 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi func QueryDataForUserDefineFromDb(opts *UserBusinessAnalysisQueryOptions, key string) ([]*UserBusinessAnalysis, int64) { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - - allCount, err := statictisSess.Where("data_date=" + key).Count(new(UserBusinessAnalysis)) + cond := "data_date='" + key + "'" + allCount, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) if err == nil { if allCount > 0 { userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where("data_date="+key).OrderBy("id desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize). + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize). Find(&userBusinessAnalysisList); err != nil { return nil, 0 } From 74ff43b76ac70bc086ee309b1409958974396f52 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 26 Jul 2022 17:10:51 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E4=BF=A1=E6=81=AF=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_analysis_for_activity.go | 14 +++++++++ routers/api/v1/api.go | 1 + routers/repo/user_data_analysis.go | 60 ++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go index d8e4a5500..2066697d2 100644 --- a/models/user_analysis_for_activity.go +++ b/models/user_analysis_for_activity.go @@ -6,6 +6,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) type UserBusinessAnalysisForActivity struct { @@ -435,3 +436,16 @@ func queryUserModelPublic(start_unix int64, end_unix int64, publicAllRepo map[in } return resultMap } + +func QueryUserLoginInfo(userIds []int64) []*UserLoginLog { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + var cond = builder.NewCond() + cond = cond.And(builder.In("u_id", userIds)) + statictisSess.Select("*").Table(new(UserLoginLog)).Where(cond) + loginList := make([]*UserLoginLog, 0) + + statictisSess.Find(&loginList) + + return loginList +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 3d9452f93..aa51c6e1a 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -571,6 +571,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday) m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) m.Get("/query_user_activity", operationReq, repo_ext.QueryUserActivity) + m.Get("/query_user_login", operationReq, repo_ext.QueryUserLoginInfo) //cloudbrain board m.Group("/cloudbrainboard", func() { m.Get("/downloadAll", repo.DownloadCloudBrainBoard) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index bafe46878..be64fddf9 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -5,6 +5,8 @@ import ( "net/http" "net/url" "os" + "strconv" + "strings" "time" "code.gitea.io/gitea/models" @@ -847,3 +849,61 @@ func writeUserActivityToExcel(startTime time.Time, endTime time.Time, filePath s log.Info("write to file succeed, filepath=" + filePath) } } + +// URL: /api/v1/query_user_login?userId=1,2,3,4 +func QueryUserLoginInfo(ctx *context.Context) { + userId := ctx.Query("userId") + userIds := strings.Split(userId, ",") + userIdInt := make([]int64, 0) + for _, id := range userIds { + idInt, err := strconv.ParseInt(id, 10, 64) + if err == nil { + userIdInt = append(userIdInt, idInt) + } + } + result := models.QueryUserLoginInfo(userIdInt) + + xlsx := excelize.NewFile() + sheetName := ctx.Tr("用户登录信息") + index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + + excelHeader := make([]string, 0) + excelHeader = append(excelHeader, "用户ID") + excelHeader = append(excelHeader, "登录IP") + excelHeader = append(excelHeader, "登录时间") + + excelHeaderMap := make(map[string]string, 0) + var j byte + j = 0 + for _, value := range excelHeader { + excelColumn := getColumn(j) + fmt.Sprint(1) + log.Info("excelColumn=" + excelColumn) + excelHeaderMap[excelColumn] = value + j++ + } + for k, v := range excelHeaderMap { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + for i, userLogin := range result { + row := i + 2 + rows := fmt.Sprint(row) + var tmp byte + tmp = 0 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userLogin.UId) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userLogin.IpAddr) + tmp = tmp + 1 + formatTime := userLogin.CreatedUnix.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime[0:len(formatTime)-3]) + } + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + filename := sheetName + "_" + time.Now().Format("2006-01-02 15:04:05") + ".xlsx" + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + if _, err := xlsx.WriteTo(ctx.Resp); err != nil { + log.Info("writer exel error." + err.Error()) + } +} From d91d790e1d693d415194d9e79f7216043e8f7d3f Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 27 Jul 2022 14:37:23 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E8=B6=8B=E5=8A=BF=E6=B4=BB=E5=8A=A8=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 1 + routers/repo/user_data_analysis.go | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 06a15c884..a36bd4736 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1140,6 +1140,7 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, userA ",activate_regist_user=" + fmt.Sprint(useMetrics.ActivateRegistUser) + ",not_activate_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser-useMetrics.ActivateRegistUser) + ",current_day_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser) + + ",activate_index=" + fmt.Sprint(float64(useMetrics.ActivateRegistUser)/float64(useMetrics.CurrentDayRegistUser)) + ",data_date='" + time.Unix(key, 0).Format("2006-01-02") + "'" + " where count_date=" + fmt.Sprint(key) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index be64fddf9..ed967af73 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -452,7 +452,7 @@ func DownloadUserDefineFile(ctx *context.Context) { func QueryUserMetricsCurrentMonth(ctx *context.Context) { currentTimeNow := time.Now() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()) pageStartTime = getStartTime(pageStartTime) queryMetrics(ctx, "public.user_business_analysis_current_month", pageStartTime, pageEndTime) @@ -478,7 +478,7 @@ func QueryUserMetricsCurrentWeek(ctx *context.Context) { } pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) pageStartTime = getStartTime(pageStartTime) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) queryMetrics(ctx, "public.user_business_analysis_current_week", pageStartTime, pageEndTime) } func QueryUserStaticCurrentWeek(ctx *context.Context) { @@ -492,7 +492,7 @@ func QueryUserMetricsCurrentYear(ctx *context.Context) { currentTimeNow := time.Now() pageStartTime := time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location()) pageStartTime = getStartTime(pageStartTime) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) queryMetrics(ctx, "public.user_business_analysis_current_year", pageStartTime, pageEndTime) } func QueryUserStaticCurrentYear(ctx *context.Context) { @@ -502,7 +502,7 @@ func QueryUserMetricsLast30Day(ctx *context.Context) { currentTimeNow := time.Now() pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30) pageStartTime = getStartTime(pageStartTime) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) queryMetrics(ctx, "public.user_business_analysis_last30_day", pageStartTime, pageEndTime) } func QueryUserStaticLast30Day(ctx *context.Context) { @@ -513,14 +513,14 @@ func QueryUserMetricsLastMonth(ctx *context.Context) { thisMonth := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()) pageStartTime := thisMonth.AddDate(0, -1, 0) pageStartTime = getStartTime(pageStartTime) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 23, 59, 59, 0, currentTimeNow.Location()).AddDate(0, 0, -1) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()).AddDate(0, 0, -1) queryMetrics(ctx, "public.user_business_analysis_last_month", pageStartTime, pageEndTime) } func QueryUserStaticLastMonth(ctx *context.Context) { queryUserDataPage(ctx, "public.user_business_analysis_last_month", new(models.UserBusinessAnalysisLastMonth)) } func QueryUserMetricsYesterday(ctx *context.Context) { - currentTimeNow := time.Now() + currentTimeNow := time.Now().AddDate(0, 0, -1) pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local) pageStartTime = getStartTime(pageStartTime) pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) @@ -533,7 +533,7 @@ func QueryUserMetricsAll(ctx *context.Context) { currentTimeNow := time.Now() pageStartTime := time.Date(2022, 4, 5, 0, 0, 0, 0, currentTimeNow.Location()) pageStartTime = getStartTime(pageStartTime) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) queryMetrics(ctx, "public.user_business_analysis_all", pageStartTime, pageEndTime) } func QueryUserStaticAll(ctx *context.Context) { From e7f3ccaa63953810e2bd6a7db9e4e37943cebf18 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 27 Jul 2022 15:05:51 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E4=B8=8A=E6=9C=88?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=95=B0=E6=8D=AE=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index ed967af73..557715d3a 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -513,7 +513,7 @@ func QueryUserMetricsLastMonth(ctx *context.Context) { thisMonth := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()) pageStartTime := thisMonth.AddDate(0, -1, 0) pageStartTime = getStartTime(pageStartTime) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()).AddDate(0, 0, -1) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 23, 59, 59, 0, currentTimeNow.Location()).AddDate(0, 0, -1) queryMetrics(ctx, "public.user_business_analysis_last_month", pageStartTime, pageEndTime) } func QueryUserStaticLastMonth(ctx *context.Context) { From 253da886415f24b276ce89bcdbba9c944773b1c8 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 27 Jul 2022 17:03:15 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E8=BF=90=E8=90=A5=E7=9C=8B=E6=9D=BF?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 557715d3a..9063e0458 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -406,7 +406,7 @@ func queryMetrics(ctx *context.Context, tableName string, startTime time.Time, e if tableName == "public.user_business_analysis_yesterday" { mapInterface["datarecordbegintime"] = setting.RadarMap.GrowthBeginTime if len(result) > 0 { - dateTime := time.Unix(result[0].CountDate, 0) + dateTime := time.Unix(result[0].CountDate, 0).AddDate(0, 0, 1) mapInterface["lastUpdatedTime"] = dateTime.Format("2006-01-02 15:04:05") } else { mapInterface["lastUpdatedTime"] = ""