From 7790e9141ef3cb5abe33c6f92e87ea3271f5dd5d Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 14 Jun 2022 11:18:51 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=E4=BA=91=E8=84=91=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=B6=85=E9=95=BF=E5=AF=BC=E8=87=B4=E5=85=A5?= =?UTF-8?q?=E5=BA=93=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 333867fb2..064edf13b 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -719,12 +719,11 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS userIndexMap := make(map[int64]float64, 0) maxUserIndex := 0.0 minUserIndex := 100000000.0 - dateRecordBatch := make([]UserBusinessAnalysisAll, 0) for { sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) - + dateRecordBatch := make([]UserBusinessAnalysisAll, 0) for _, userRecord := range userList { var dateRecordAll UserBusinessAnalysisAll dateRecordAll.ID = userRecord.ID @@ -804,18 +803,19 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS } } } + if len(dateRecordBatch) > 0 { + insertTable(dateRecordBatch, tableName, statictisSess) + insertCount += len(dateRecordBatch) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } indexTotal += PAGE_SIZE if indexTotal >= count { break } } - if len(dateRecordBatch) > 0 { - insertTable(dateRecordBatch, tableName, statictisSess) - insertCount += len(dateRecordBatch) - if err != nil { - log.Info("insert all data failed." + err.Error()) - } - } + if tableName == "user_business_analysis_all" { log.Info("TotalHasActivityUser=" + fmt.Sprint(userMetrics["TotalHasActivityUser"])) } @@ -2057,7 +2057,10 @@ func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[s } else { resultMap[cloudTaskRecord.UserID] += 1 } - setMapKey("CloudBrainRunTime", cloudTaskRecord.UserID, int(cloudTaskRecord.Duration), resultItemMap) + if cloudTaskRecord.Duration < 100000000 && cloudTaskRecord.Duration > 0 { + setMapKey("CloudBrainRunTime", cloudTaskRecord.UserID, int(cloudTaskRecord.Duration), resultItemMap) + } + if cloudTaskRecord.Type == 1 { //npu if cloudTaskRecord.JobType == "TRAIN" { setMapKey("NpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) From bf033c1f0fcc9c5d4d6c244d7dc9b6ab5a18e9ee Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 14 Jun 2022 11:20:47 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E9=94=99=E8=AF=AF=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 064edf13b..4cd3539d7 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -788,7 +788,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordBatch = append(dateRecordBatch, dateRecordAll) if len(dateRecordBatch) >= BATCH_INSERT_SIZE { - insertTable(dateRecordBatch, tableName, statictisSess) + err := insertTable(dateRecordBatch, tableName, statictisSess) insertCount += BATCH_INSERT_SIZE if err != nil { log.Info("insert all data failed." + err.Error()) @@ -804,7 +804,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS } } if len(dateRecordBatch) > 0 { - insertTable(dateRecordBatch, tableName, statictisSess) + err := insertTable(dateRecordBatch, tableName, statictisSess) insertCount += len(dateRecordBatch) if err != nil { log.Info("insert all data failed." + err.Error()) @@ -835,7 +835,7 @@ func updateUserIndex(tableName string, statictisSess *xorm.Session, userId int64 statictisSess.Exec(updateSql) } -func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, statictisSess *xorm.Session) { +func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, statictisSess *xorm.Session) error { insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + @@ -854,7 +854,8 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static } } - statictisSess.Exec(insertBatchSql) + _, err := statictisSess.Exec(insertBatchSql) + return err } func RefreshUserStaticAllTabel(wikiCountMap map[string]int, userMetrics map[string]int) { @@ -2060,7 +2061,6 @@ func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[s if cloudTaskRecord.Duration < 100000000 && cloudTaskRecord.Duration > 0 { setMapKey("CloudBrainRunTime", cloudTaskRecord.UserID, int(cloudTaskRecord.Duration), resultItemMap) } - if cloudTaskRecord.Type == 1 { //npu if cloudTaskRecord.JobType == "TRAIN" { setMapKey("NpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap)