OpenI
/
aiforge
Not watched
1
0
Fork 0
Code
Releases 128 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

提交代码，增加模型转换字段。 

Signed-off-by: zouap <zouap@pcl.ac.cn>
pull/3432/head
zouap 2 years ago
parent
f301331d0a
commit
00da5e9b79
5 changed files with 62 additions and 3 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +45
    -3
      models/user_business_analysis.go
  2. +10
    -0
      models/user_business_struct.go
  3. +1
    -0
      options/locale/locale_en-US.ini
  4. +1
    -0
      options/locale/locale_zh-CN.ini
  5. +5
    -0
      routers/repo/user_data_analysis.go

+ 45
- 3
models/user_business_analysis.go View File

@@ -355,6 +355,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)

CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
@@ -427,6 +428,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap)

dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset)
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset)
@@ -546,6 +548,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum
resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
resultMap[userRecord.ID].ModelConvertCount += userRecord.ModelConvertCount
resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
@@ -607,6 +610,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)

CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)
@@ -687,6 +691,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap)
dateRecordAll.ModelConvertCount = getMapValue(dateRecordAll.ID, AiModelConvertMap)
dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset)
dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset)
dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset)
@@ -998,7 +1003,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static

insertBatchSql := "INSERT INTO public." + tableName +
"(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num) " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num,model_convert_count) " +
"VALUES"

for i, record := range dateRecords {
@@ -1007,7 +1012,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static
", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) +
", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) +
", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," +
fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + ")"
fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + "," + fmt.Sprint(record.ModelConvertCount) + ")"
if i < (len(dateRecords) - 1) {
insertBatchSql += ","
}
@@ -1098,6 +1103,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)

CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
@@ -1179,7 +1185,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap)
dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset)
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset)
dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset)
@@ -1368,6 +1374,7 @@ func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight
result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05)
result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3)
result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2)
result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2)
result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1)

result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1)
@@ -1393,6 +1400,7 @@ func getUserActivateAll(dateRecord UserBusinessAnalysisAll) int {
result += dateRecord.CreateRepoCount
result += dateRecord.CloudBrainTaskNum
result += dateRecord.CommitModelCount
result += dateRecord.ModelConvertCount
result += dateRecord.CommitDatasetNum
result += dateRecord.FocusOtherUser
result += dateRecord.CollectDataset
@@ -1414,6 +1422,7 @@ func getUserActivate(dateRecord UserBusinessAnalysis) int {
result += dateRecord.CreateRepoCount
result += dateRecord.CloudBrainTaskNum
result += dateRecord.CommitModelCount
result += dateRecord.ModelConvertCount
result += dateRecord.CommitDatasetNum
result += dateRecord.FocusOtherUser
result += dateRecord.CollectDataset
@@ -1450,6 +1459,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64
result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05)
result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3)
result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2)
result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2)
result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1)

result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1)
@@ -2278,6 +2288,38 @@ func queryUserModel(start_unix int64, end_unix int64) map[int64]int {
return resultMap
}

func queryUserModelConvert(start_unix int64, end_unix int64) map[int64]int {
sess := x.NewSession()
defer sess.Close()
resultMap := make(map[int64]int)
cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
count, err := sess.Where(cond).Count(new(AiModelConvert))
if err != nil {
log.Info("query AiModelConvert error. return.")
return resultMap
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("id,user_id").Table("ai_model_convert").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
aiModelList := make([]*AiModelConvert, 0)
sess.Find(&aiModelList)
log.Info("query AiModelConvert size=" + fmt.Sprint(len(aiModelList)))
for _, aiModelRecord := range aiModelList {
if _, ok := resultMap[aiModelRecord.UserId]; !ok {
resultMap[aiModelRecord.UserId] = 1
} else {
resultMap[aiModelRecord.UserId] += 1
}
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}
return resultMap
}

func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) {
sess := x.NewSession()
defer sess.Close()


+ 10
- 0
models/user_business_struct.go View File

@@ -89,6 +89,7 @@ type UserBusinessAnalysisCurrentYear struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisLast30Day struct {
@@ -157,6 +158,7 @@ type UserBusinessAnalysisLast30Day struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisLastMonth struct {
@@ -225,6 +227,7 @@ type UserBusinessAnalysisLastMonth struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisCurrentMonth struct {
@@ -293,6 +296,7 @@ type UserBusinessAnalysisCurrentMonth struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisCurrentWeek struct {
@@ -362,6 +366,7 @@ type UserBusinessAnalysisCurrentWeek struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisYesterday struct {
@@ -431,6 +436,7 @@ type UserBusinessAnalysisYesterday struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisLastWeek struct {
@@ -500,6 +506,7 @@ type UserBusinessAnalysisLastWeek struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserAnalysisPara struct {
@@ -616,6 +623,8 @@ type UserBusinessAnalysisAll struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`

ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysis struct {
@@ -704,4 +713,5 @@ type UserBusinessAnalysis struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

+ 1
- 0
options/locale/locale_en-US.ini View File

@@ -577,6 +577,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count
static.CloudBrainRunTime=CloudBrain Run Time
static.CommitDatasetNum=Commit Dataset Count
static.CommitModelCount=Commit Model Count
static.ModelConvertCount=Model Convert Count
static.UserIndex=Normalized user index
static.UserIndexPrimitive=User Index
static.countdate=Count Date


+ 1
- 0
options/locale/locale_zh-CN.ini View File

@@ -581,6 +581,7 @@ static.CloudBrainTaskNum=云脑任务数
static.CloudBrainRunTime=云脑运行时间（小时）
static.CommitDatasetNum=上传（提交）数据集文件数
static.CommitModelCount=提交模型数
static.ModelConvertCount=模型转换数
static.UserIndex=归一化用户指数
static.UserIndexPrimitive=用户指数
static.countdate=系统统计时间


+ 5
- 0
routers/repo/user_data_analysis.go View File

@@ -104,6 +104,7 @@ func getExcelHeader(ctx *context.Context) map[string]string {
excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount"))
excelHeader = append(excelHeader, ctx.Tr("user.static.ModelConvertCount"))

excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset"))
@@ -178,6 +179,8 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)
@@ -256,6 +259,8 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord *
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)


Write Preview
Loading…
Cancel
Save