Merge pull request '运营看板中用户统计分析相关问题修改及提供用户登录日志查询接口。' (#2592) from zouap into V20220801

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/2592
Reviewed-by: lewis <747342561@qq.com>

models/user_analysis_for_activity.go

@@ -6,6 +6,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/timeutil"
+	"xorm.io/builder"
 )
 
 type UserBusinessAnalysisForActivity struct {
@@ -435,3 +436,16 @@ func queryUserModelPublic(start_unix int64, end_unix int64, publicAllRepo map[in
 	}
 	return resultMap
 }
+
+func QueryUserLoginInfo(userIds []int64) []*UserLoginLog {
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+	var cond = builder.NewCond()
+	cond = cond.And(builder.In("u_id", userIds))
+	statictisSess.Select("*").Table(new(UserLoginLog)).Where(cond)
+	loginList := make([]*UserLoginLog, 0)
+
+	statictisSess.Find(&loginList)
+
+	return loginList
+}

models/user_business_analysis.go

@@ -110,9 +110,9 @@ type UserBusinessAnalysisAll struct {
 }
 
 type UserBusinessAnalysis struct {
-	ID int64 `xorm:"pk"`
-
-	CountDate int64 `xorm:"pk"`
+	ID        int64  `xorm:"pk"`
+	DataDate  string `xorm:"pk"`
+	CountDate int64  `xorm:"NULL"`
 
 	//action :ActionMergePullRequest                         // 11
 	CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"`
@@ -171,8 +171,6 @@ type UserBusinessAnalysis struct {
 	//user
 	Name string `xorm:"NOT NULL"`
 
-	DataDate string `xorm:"NULL"`
-
 	CloudBrainTaskNum  int     `xorm:"NOT NULL DEFAULT 0"`
 	GpuDebugJob        int     `xorm:"NOT NULL DEFAULT 0"`
 	NpuDebugJob        int     `xorm:"NOT NULL DEFAULT 0"`
@@ -411,6 +409,42 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi
 	return userBusinessAnalysisReturnList, allCount
 }
 
+func QueryDataForUserDefineFromDb(opts *UserBusinessAnalysisQueryOptions, key string) ([]*UserBusinessAnalysis, int64) {
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+	cond := "data_date='" + key + "'"
+	allCount, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis))
+	if err == nil {
+		if allCount > 0 {
+			userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
+			if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).
+				Find(&userBusinessAnalysisList); err != nil {
+				return nil, 0
+			}
+			return userBusinessAnalysisList, allCount
+		}
+	}
+	return nil, 0
+}
+
+func WriteDataToDb(dataList []*UserBusinessAnalysis, key string) {
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+	log.Info("write to db, size=" + fmt.Sprint(len(dataList)))
+	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
+	for _, data := range dataList {
+		data.DataDate = key
+		userBusinessAnalysisList = append(userBusinessAnalysisList, data)
+		if len(userBusinessAnalysisList) > BATCH_INSERT_SIZE {
+			statictisSess.Insert(userBusinessAnalysisList)
+			userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0)
+		}
+	}
+	if len(userBusinessAnalysisList) > 0 {
+		statictisSess.Insert(userBusinessAnalysisList)
+	}
+}
+
 func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]*UserBusinessAnalysis, int64) {
 	log.Info("start to count other user info data")
 	sess := x.NewSession()
@@ -954,6 +988,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 	statictisSess := xStatistic.NewSession()
 	defer statictisSess.Close()
 
+	log.Info("truncate all data from table:user_business_analysis ")
+	statictisSess.Exec("TRUNCATE TABLE user_business_analysis")
+
 	cond := "type != 1"
 	count, err := sess.Where(cond).Count(new(User))
 	if err != nil {
@@ -1103,6 +1140,7 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, userA
 				",activate_regist_user=" + fmt.Sprint(useMetrics.ActivateRegistUser) +
 				",not_activate_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser-useMetrics.ActivateRegistUser) +
 				",current_day_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser) +
+				",activate_index=" + fmt.Sprint(float64(useMetrics.ActivateRegistUser)/float64(useMetrics.CurrentDayRegistUser)) +
 				",data_date='" + time.Unix(key, 0).Format("2006-01-02") + "'" +
 				" where count_date=" + fmt.Sprint(key)
 

routers/api/v1/api.go

@@ -571,6 +571,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday)
 		m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll)
 		m.Get("/query_user_activity", operationReq, repo_ext.QueryUserActivity)
+		m.Get("/query_user_login", operationReq, repo_ext.QueryUserLoginInfo)
 		//cloudbrain board
 		m.Group("/cloudbrainboard", func() {
 			m.Get("/downloadAll", repo.DownloadCloudBrainBoard)

routers/repo/user_data_analysis.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models"
@@ -404,7 +406,7 @@ func queryMetrics(ctx *context.Context, tableName string, startTime time.Time, e
 		if tableName == "public.user_business_analysis_yesterday" {
 			mapInterface["datarecordbegintime"] = setting.RadarMap.GrowthBeginTime
 			if len(result) > 0 {
-				dateTime := time.Unix(result[0].CountDate, 0)
+				dateTime := time.Unix(result[0].CountDate, 0).AddDate(0, 0, 1)
 				mapInterface["lastUpdatedTime"] = dateTime.Format("2006-01-02 15:04:05")
 			} else {
 				mapInterface["lastUpdatedTime"] = ""
@@ -450,7 +452,7 @@ func DownloadUserDefineFile(ctx *context.Context) {
 func QueryUserMetricsCurrentMonth(ctx *context.Context) {
 
 	currentTimeNow := time.Now()
-	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
 	pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location())
 	pageStartTime = getStartTime(pageStartTime)
 	queryMetrics(ctx, "public.user_business_analysis_current_month", pageStartTime, pageEndTime)
@@ -476,7 +478,7 @@ func QueryUserMetricsCurrentWeek(ctx *context.Context) {
 	}
 	pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset)
 	pageStartTime = getStartTime(pageStartTime)
-	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
 	queryMetrics(ctx, "public.user_business_analysis_current_week", pageStartTime, pageEndTime)
 }
 func QueryUserStaticCurrentWeek(ctx *context.Context) {
@@ -490,7 +492,7 @@ func QueryUserMetricsCurrentYear(ctx *context.Context) {
 	currentTimeNow := time.Now()
 	pageStartTime := time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location())
 	pageStartTime = getStartTime(pageStartTime)
-	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
 	queryMetrics(ctx, "public.user_business_analysis_current_year", pageStartTime, pageEndTime)
 }
 func QueryUserStaticCurrentYear(ctx *context.Context) {
@@ -500,7 +502,7 @@ func QueryUserMetricsLast30Day(ctx *context.Context) {
 	currentTimeNow := time.Now()
 	pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30)
 	pageStartTime = getStartTime(pageStartTime)
-	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
 	queryMetrics(ctx, "public.user_business_analysis_last30_day", pageStartTime, pageEndTime)
 }
 func QueryUserStaticLast30Day(ctx *context.Context) {
@@ -518,7 +520,7 @@ func QueryUserStaticLastMonth(ctx *context.Context) {
 	queryUserDataPage(ctx, "public.user_business_analysis_last_month", new(models.UserBusinessAnalysisLastMonth))
 }
 func QueryUserMetricsYesterday(ctx *context.Context) {
-	currentTimeNow := time.Now()
+	currentTimeNow := time.Now().AddDate(0, 0, -1)
 	pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local)
 	pageStartTime = getStartTime(pageStartTime)
 	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
@@ -531,7 +533,7 @@ func QueryUserMetricsAll(ctx *context.Context) {
 	currentTimeNow := time.Now()
 	pageStartTime := time.Date(2022, 4, 5, 0, 0, 0, 0, currentTimeNow.Location())
 	pageStartTime = getStartTime(pageStartTime)
-	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
 	queryMetrics(ctx, "public.user_business_analysis_all", pageStartTime, pageEndTime)
 }
 func QueryUserStaticAll(ctx *context.Context) {
@@ -611,7 +613,15 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 		ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+"/api/v1/download_user_define_file?filename="+filename)
 	} else {
 		mapInterface := make(map[string]interface{})
-		re, count := models.QueryUserStaticDataPage(pageOpts)
+		key := startTime.Format("2006-01-02") + endTime.Format("2006-01-02")
+		log.Info("db key =" + key)
+		re, count := models.QueryDataForUserDefineFromDb(pageOpts, key)
+		if count == 0 {
+			wikiMap, _ := queryWikiCountMap(startTime, endTime)
+			re, count = models.QueryUserStaticDataForUserDefine(pageOpts, wikiMap)
+			models.WriteDataToDb(re, key)
+		}
+		re, count = models.QueryDataForUserDefineFromDb(pageOpts, key)
 		mapInterface["data"] = re
 		mapInterface["count"] = count
 		ctx.JSON(http.StatusOK, mapInterface)
@@ -839,3 +849,61 @@ func writeUserActivityToExcel(startTime time.Time, endTime time.Time, filePath s
 		log.Info("write to file succeed, filepath=" + filePath)
 	}
 }
+
+// URL:  /api/v1/query_user_login?userId=1,2,3,4
+func QueryUserLoginInfo(ctx *context.Context) {
+	userId := ctx.Query("userId")
+	userIds := strings.Split(userId, ",")
+	userIdInt := make([]int64, 0)
+	for _, id := range userIds {
+		idInt, err := strconv.ParseInt(id, 10, 64)
+		if err == nil {
+			userIdInt = append(userIdInt, idInt)
+		}
+	}
+	result := models.QueryUserLoginInfo(userIdInt)
+
+	xlsx := excelize.NewFile()
+	sheetName := ctx.Tr("用户登录信息")
+	index := xlsx.NewSheet(sheetName)
+	xlsx.DeleteSheet("Sheet1")
+
+	excelHeader := make([]string, 0)
+	excelHeader = append(excelHeader, "用户ID")
+	excelHeader = append(excelHeader, "登录IP")
+	excelHeader = append(excelHeader, "登录时间")
+
+	excelHeaderMap := make(map[string]string, 0)
+	var j byte
+	j = 0
+	for _, value := range excelHeader {
+		excelColumn := getColumn(j) + fmt.Sprint(1)
+		log.Info("excelColumn=" + excelColumn)
+		excelHeaderMap[excelColumn] = value
+		j++
+	}
+	for k, v := range excelHeaderMap {
+		//设置单元格的值
+		xlsx.SetCellValue(sheetName, k, v)
+	}
+	for i, userLogin := range result {
+		row := i + 2
+		rows := fmt.Sprint(row)
+		var tmp byte
+		tmp = 0
+		xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userLogin.UId)
+		tmp = tmp + 1
+		xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userLogin.IpAddr)
+		tmp = tmp + 1
+		formatTime := userLogin.CreatedUnix.Format("2006-01-02 15:04:05")
+		xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime[0:len(formatTime)-3])
+	}
+	//设置默认打开的表单
+	xlsx.SetActiveSheet(index)
+	filename := sheetName + "_" + time.Now().Format("2006-01-02 15:04:05") + ".xlsx"
+	ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
+	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+	if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
+		log.Info("writer exel error." + err.Error())
+	}
+}