Merge pull request '提升action表查询速度' (#3544) from zouap into V20221228

Reviewed-on: https://openi.pcl.ac.cn/OpenI/aiforge/pulls/3544
Reviewed-by: ychao_1983 <ychao_1983@sina.com>

models/user_business_analysis.go

@@ -330,7 +330,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
 	DataDate := currentTimeNow.Format("2006-01-02 15:04")
 
 	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
-	CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5)
+	CommitCountMap := queryCommitAction(start_unix, end_unix, 5)
 	IssueCountMap := queryCreateIssue(start_unix, end_unix)
 
 	CommentCountMap := queryComment(start_unix, end_unix)
@@ -586,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
 	startTime := currentTimeNow.AddDate(0, 0, -1)
 
 	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
-	CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5)
+	CommitCountMap := queryCommitAction(start_unix, end_unix, 5)
 	IssueCountMap := queryCreateIssue(start_unix, end_unix)
 
 	CommentCountMap := queryComment(start_unix, end_unix)
@@ -762,7 +762,8 @@ func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) {
 	end_unix := pageEndTime.Unix()
 
 	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
-	CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
+	CommitCountMap := queryCommitAction(start_unix, end_unix, 5)
+	mostActiveMap := queryMostActiveCommitAction(start_unix, end_unix)
 	IssueCountMap := queryCreateIssue(start_unix, end_unix)
 
 	CommentCountMap := queryComment(start_unix, end_unix)
@@ -1246,7 +1247,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 
 	DataDate := CountDate.Format("2006-01-02")
 	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
-	CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5)
+	CommitCountMap := queryCommitAction(start_unix, end_unix, 5)
 	IssueCountMap := queryCreateIssue(start_unix, end_unix)
 
 	CommentCountMap := queryComment(start_unix, end_unix)
@@ -1740,52 +1741,48 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int {
 	return resultMap
 }
 
-func queryCommitAction(start_unix int64, end_unix int64, actionType int64) (map[int64]int, map[int64]map[string]int) {
+func queryMostActiveCommitAction(start_unix int64, end_unix int64) map[int64]map[string]int {
 	sess := x.NewSession()
 	defer sess.Close()
-	resultMap := make(map[int64]int)
+
 	mostActiveMap := make(map[int64]map[string]int)
 	cond := "user_id=act_user_id  and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
 
 	count, err := sess.Where(cond).Count(new(Action))
 	if err != nil {
 		log.Info("query action error. return.")
-		return resultMap, mostActiveMap
+		return mostActiveMap
 	}
 
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,user_id,op_type,act_user_id,created_unix").Table("action").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
-		actionList := make([]*Action, 0)
-		sess.Find(&actionList)
-
-		log.Info("query action size=" + fmt.Sprint(len(actionList)))
+		actionList, err := sess.QueryInterface("select id,user_id,op_type,act_user_id,created_unix from public.action where " + cond + " order by id asc limit " + fmt.Sprint(PAGE_SIZE) + " offset " + fmt.Sprint(indexTotal))
+		if err != nil {
+			log.Info("error:" + err.Error())
+			continue
+		}
+		log.Info("query mostactive action size=" + fmt.Sprint(len(actionList)))
 		for _, actionRecord := range actionList {
-			if int64(actionRecord.OpType) == actionType {
-				if _, ok := resultMap[actionRecord.UserID]; !ok {
-					resultMap[actionRecord.UserID] = 1
-				} else {
-					resultMap[actionRecord.UserID] += 1
-				}
-			}
-			key := getDate(actionRecord.CreatedUnix)
-			if _, ok := mostActiveMap[actionRecord.UserID]; !ok {
+			userId := convertInterfaceToInt64(actionRecord["user_id"])
+			created_unix := timeutil.TimeStamp(convertInterfaceToInt64(actionRecord["created_unix"]))
+			key := getDate(created_unix)
+			if _, ok := mostActiveMap[userId]; !ok {
 				tmpMap := make(map[string]int)
 				tmpMap[key] = 1
-				mostActiveMap[actionRecord.UserID] = tmpMap
+				mostActiveMap[userId] = tmpMap
 			} else {
-				mostActiveMap[actionRecord.UserID][key] = getMapKeyStringValue(key, mostActiveMap[actionRecord.UserID]) + 1
+				mostActiveMap[userId][key] = getMapKeyStringValue(key, mostActiveMap[userId]) + 1
 			}
-			utcTime := actionRecord.CreatedUnix.AsTime()
+			utcTime := created_unix.AsTime()
 			hour := utcTime.Hour()
 			if hour >= 0 && hour <= 5 {
 				key = "hour_hour"
-				if getMapKeyStringValue(key, mostActiveMap[actionRecord.UserID]) < hour {
-					mostActiveMap[actionRecord.UserID][key] = hour
-					mostActiveMap[actionRecord.UserID]["hour_day"] = utcTime.Day()
-					mostActiveMap[actionRecord.UserID]["hour_month"] = int(utcTime.Month())
-					mostActiveMap[actionRecord.UserID]["hour_year"] = utcTime.Year()
+				if getMapKeyStringValue(key, mostActiveMap[userId]) < hour {
+					mostActiveMap[userId][key] = hour
+					mostActiveMap[userId]["hour_day"] = utcTime.Day()
+					mostActiveMap[userId]["hour_month"] = int(utcTime.Month())
+					mostActiveMap[userId]["hour_year"] = utcTime.Year()
 				}
 			}
 		}
@@ -1794,9 +1791,60 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) (map[
 			break
 		}
 	}
+	return mostActiveMap
+}
 
-	return resultMap, mostActiveMap
+func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int {
+	sess := x.NewSession()
+	defer sess.Close()
+	resultMap := make(map[int64]int)
+	cond := "op_type=" + fmt.Sprint(actionType) + " and user_id=act_user_id  and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
+	count, err := sess.Where(cond).Count(new(Action))
+	if err != nil {
+		log.Info("query action error. return.")
+		return resultMap
+	}
+	var indexTotal int64
+	indexTotal = 0
+	for {
+		actionList, err := sess.QueryInterface("select id,user_id,op_type,act_user_id,created_unix from public.action where " + cond + " order by id asc limit " + fmt.Sprint(PAGE_SIZE) + " offset " + fmt.Sprint(indexTotal))
+		if err != nil {
+			log.Info("error:" + err.Error())
+			continue
+		}
+		log.Info("query action size=" + fmt.Sprint(len(actionList)))
+		for _, actionRecord := range actionList {
+			userId := convertInterfaceToInt64(actionRecord["user_id"])
+
+			if _, ok := resultMap[userId]; !ok {
+				resultMap[userId] = 1
+			} else {
+				resultMap[userId] += 1
+			}
+
+		}
+		indexTotal += PAGE_SIZE
+		if indexTotal >= count {
+			break
+		}
+	}
+	return resultMap
 }
+
+func convertInterfaceToInt64(obj interface{}) int64 {
+	switch obj.(type) {
+	case int8:
+		return int64(obj.(int8))
+	case int16:
+		return int64(obj.(int16))
+	case int32:
+		return int64(obj.(int32))
+	case int64:
+		return obj.(int64)
+	}
+	return 0
+}
+
 func getDate(createTime timeutil.TimeStamp) string {
 	return createTime.Format("2006-01-02")
 }