From 00da5e9b7947f3168cd77cee09ee0a80c626bfdb Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 14 Dec 2022 15:50:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A8=A1=E5=9E=8B=E8=BD=AC=E6=8D=A2=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 48 +++++++++++++++++++++++++++++++++++--- models/user_business_struct.go | 10 ++++++++ options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/repo/user_data_analysis.go | 5 ++++ 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index b8e7a422d..e224c6c9b 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -355,6 +355,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) @@ -427,6 +428,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) @@ -546,6 +548,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].ModelConvertCount += userRecord.ModelConvertCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount @@ -607,6 +610,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix) @@ -687,6 +691,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) + dateRecordAll.ModelConvertCount = getMapValue(dateRecordAll.ID, AiModelConvertMap) dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset) dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset) dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset) @@ -998,7 +1003,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + - "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num) " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num,model_convert_count) " + "VALUES" for i, record := range dateRecords { @@ -1007,7 +1012,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," + - fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + ")" + fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + "," + fmt.Sprint(record.ModelConvertCount) + ")" if i < (len(dateRecords) - 1) { insertBatchSql += "," } @@ -1098,6 +1103,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) @@ -1179,7 +1185,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) - + dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset) @@ -1368,6 +1374,7 @@ func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) @@ -1393,6 +1400,7 @@ func getUserActivateAll(dateRecord UserBusinessAnalysisAll) int { result += dateRecord.CreateRepoCount result += dateRecord.CloudBrainTaskNum result += dateRecord.CommitModelCount + result += dateRecord.ModelConvertCount result += dateRecord.CommitDatasetNum result += dateRecord.FocusOtherUser result += dateRecord.CollectDataset @@ -1414,6 +1422,7 @@ func getUserActivate(dateRecord UserBusinessAnalysis) int { result += dateRecord.CreateRepoCount result += dateRecord.CloudBrainTaskNum result += dateRecord.CommitModelCount + result += dateRecord.ModelConvertCount result += dateRecord.CommitDatasetNum result += dateRecord.FocusOtherUser result += dateRecord.CollectDataset @@ -1450,6 +1459,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64 result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) @@ -2278,6 +2288,38 @@ func queryUserModel(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserModelConvert(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(AiModelConvert)) + if err != nil { + log.Info("query AiModelConvert error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id").Table("ai_model_convert").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + aiModelList := make([]*AiModelConvert, 0) + sess.Find(&aiModelList) + log.Info("query AiModelConvert size=" + fmt.Sprint(len(aiModelList))) + for _, aiModelRecord := range aiModelList { + if _, ok := resultMap[aiModelRecord.UserId]; !ok { + resultMap[aiModelRecord.UserId] = 1 + } else { + resultMap[aiModelRecord.UserId] += 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) { sess := x.NewSession() defer sess.Close() diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 9dcc12342..e59945abe 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -89,6 +89,7 @@ type UserBusinessAnalysisCurrentYear struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLast30Day struct { @@ -157,6 +158,7 @@ type UserBusinessAnalysisLast30Day struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLastMonth struct { @@ -225,6 +227,7 @@ type UserBusinessAnalysisLastMonth struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisCurrentMonth struct { @@ -293,6 +296,7 @@ type UserBusinessAnalysisCurrentMonth struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisCurrentWeek struct { @@ -362,6 +366,7 @@ type UserBusinessAnalysisCurrentWeek struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisYesterday struct { @@ -431,6 +436,7 @@ type UserBusinessAnalysisYesterday struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLastWeek struct { @@ -500,6 +506,7 @@ type UserBusinessAnalysisLastWeek struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserAnalysisPara struct { @@ -616,6 +623,8 @@ type UserBusinessAnalysisAll struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysis struct { @@ -704,4 +713,5 @@ type UserBusinessAnalysis struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 082b35ca8..c3a1fa748 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -577,6 +577,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count static.CloudBrainRunTime=CloudBrain Run Time static.CommitDatasetNum=Commit Dataset Count static.CommitModelCount=Commit Model Count +static.ModelConvertCount=Model Convert Count static.UserIndex=Normalized user index static.UserIndexPrimitive=User Index static.countdate=Count Date diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index efaf031c6..363bd6b82 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -581,6 +581,7 @@ static.CloudBrainTaskNum=云脑任务数 static.CloudBrainRunTime=云脑运行时间（小时） static.CommitDatasetNum=上传（提交）数据集文件数 static.CommitModelCount=提交模型数 +static.ModelConvertCount=模型转换数 static.UserIndex=归一化用户指数 static.UserIndexPrimitive=用户指数 static.countdate=系统统计时间 diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index a6de283a4..a6fb7d82c 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -104,6 +104,7 @@ func getExcelHeader(ctx *context.Context) map[string]string { excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.ModelConvertCount")) excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser")) excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset")) @@ -178,6 +179,8 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount) + tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) @@ -256,6 +259,8 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord * tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount) + tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)