diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 777c2edaa..c28a1bc45 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1073,6 +1073,9 @@ type UserImageConfig struct { CreateVersion bool `json:"create_version"` Flavor Flavor `json:"flavor"` PoolID string `json:"pool_id"` + ShareAddr string `json:"nas_share_addr"` + MountPath string `json:"nas_mount_path"` + NasType string `json:"nas_type"` } type CreateTrainJobParams struct { @@ -1096,13 +1099,18 @@ type Config struct { CreateVersion bool `json:"create_version"` Flavor Flavor `json:"flavor"` PoolID string `json:"pool_id"` + ShareAddr string `json:"nas_share_addr"` + MountPath string `json:"nas_mount_path"` + NasType string `json:"nas_type"` } + type CreateInferenceJobParams struct { JobName string `json:"job_name"` Description string `json:"job_desc"` InfConfig InfConfig `json:"config"` WorkspaceID string `json:"workspace_id"` } + type CreateInfUserImageParams struct { JobName string `json:"job_name"` Description string `json:"job_desc"` @@ -1160,6 +1168,9 @@ type TrainJobVersionConfig struct { Flavor Flavor `json:"flavor"` PoolID string `json:"pool_id"` PreVersionId int64 `json:"pre_version_id"` + ShareAddr string `json:"nas_share_addr"` + MountPath string `json:"nas_mount_path"` + NasType string `json:"nas_type"` } type TrainJobVersionUserImageConfig struct { @@ -1175,6 +1186,9 @@ type TrainJobVersionUserImageConfig struct { PreVersionId int64 `json:"pre_version_id"` UserImageUrl string `json:"user_image_url"` UserCommand string `json:"user_command"` + ShareAddr string `json:"nas_share_addr"` + MountPath string `json:"nas_mount_path"` + NasType string `json:"nas_type"` } type CreateConfigParams struct { @@ -1190,6 +1204,7 @@ type CreateConfigParams struct { LogUrl string `json:"log_url"` Flavor Flavor `json:"flavor"` PoolID string `json:"pool_id"` + Volumes []Volumes `json:"volumes"` } type Parameter struct { diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 40d7a2a2e..beb1ceee5 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -92,6 +92,17 @@ type HourTimeStatistic struct { HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"` HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"` } +type CloudbrainTypeDuration []struct { + Type int `xorm:"type"` + DurationSum int `xorm:"durationSum"` + CardDurationSum int `xorm:"cardDurationSum"` + Count int `xorm:"count"` +} +type CloudbrainAllDuration struct { + DurationSum int `xorm:"durationSum"` + CardDurationSum int `xorm:"cardDurationSum"` + Count int `xorm:"count"` +} func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + @@ -303,7 +314,7 @@ func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, err builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}, builder.Gt{"cloudbrain.start_time": 0}), ) cond = cond.Or( - builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}), + builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}, builder.Lte{"cloudbrain.start_time": beginTime}), ) sess.OrderBy("cloudbrain.id ASC") cloudbrains := make([]*CloudbrainInfo, 0, 10) @@ -425,3 +436,55 @@ func DeleteCloudbrainDurationStatistic(beginTime timeutil.TimeStamp, endTime tim } return nil } + +func GetCloudbrainTypeCardDuration() (CloudbrainTypeDuration, error) { + query := ` + SELECT + cloudbrain.type, + SUM(cloudbrain.duration) as durationSum, + SUM( + COALESCE(cloudbrain.duration * + CASE + WHEN cloudbrain.work_server_number = 0 THEN 1 + ELSE COALESCE(cloudbrain.work_server_number, 1) + END * + COALESCE(cloudbrain_spec.acc_cards_num, 1), 0) + ) as cardDurationSum, + COUNT(*) as count + FROM cloudbrain + LEFT JOIN cloudbrain_spec + ON cloudbrain.id = cloudbrain_spec.cloudbrain_id + GROUP BY cloudbrain.type + ` + // 执行查询 + var results CloudbrainTypeDuration + if err := x.SQL(query).Find(&results); err != nil { + panic(err) + } + return results, nil +} + +func GetCloudbrainAllCardDuration() (CloudbrainAllDuration, error) { + query := ` + SELECT + SUM(cloudbrain.duration) as durationSum, + SUM( + COALESCE(cloudbrain.duration * + CASE + WHEN cloudbrain.work_server_number = 0 THEN 1 + ELSE COALESCE(cloudbrain.work_server_number, 1) + END * + COALESCE(cloudbrain_spec.acc_cards_num, 1), 0) + ) as cardDurationSum, + COUNT(*) as count + FROM cloudbrain + LEFT JOIN cloudbrain_spec + ON cloudbrain.id = cloudbrain_spec.cloudbrain_id + ` + // 执行查询 + var result CloudbrainAllDuration + if _, err := x.SQL(query).Get(&result); err != nil { + panic(err) + } + return result, nil +} diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ccefbf4f6..d5ab871ce 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -355,6 +355,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) @@ -427,6 +428,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) @@ -546,6 +548,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].ModelConvertCount += userRecord.ModelConvertCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount @@ -583,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS startTime := currentTimeNow.AddDate(0, 0, -1) CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5) + CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) @@ -599,29 +602,25 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS //log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson)) } //CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix()) - CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix) + CommitDatasetSizeMap, CommitDatasetNumMap, _ := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) - CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix) + CreateRepoCountMap, _, _ := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) - RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix) + RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) CollectImage, CollectedImage := queryImageStars(start_unix, end_unix) RecommendImage := queryRecommedImage(start_unix, end_unix) InvitationMap := queryUserInvitationCount(start_unix, end_unix) DataDate := currentTimeNow.Format("2006-01-02") + " 00:01" - bonusMap := make(map[string]map[string]int) - if isUserYearData(tableName) { - bonusMap = getBonusMap() - log.Info("truncate all data from table:user_summary_current_year ") - statictisSess.Exec("TRUNCATE TABLE user_summary_current_year") - } + cond := "type != 1 and is_active=true" count, err := sess.Where(cond).Count(new(User)) if err != nil { @@ -687,6 +686,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) + dateRecordAll.ModelConvertCount = getMapValue(dateRecordAll.ID, AiModelConvertMap) dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset) dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset) dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset) @@ -719,37 +719,6 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1 } } - if isUserYearData(tableName) { - //年度数据 - subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC()) - mostActiveDay := "" - if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok { - mostActiveDay = getMostActiveJson(userInfo) - } - scoreMap := make(map[string]float64) - repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap) - dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset) - scoreMap["datasetscore"] = datasetscore - codeInfo, codescore := getCodeInfo(dateRecordAll) - scoreMap["codescore"] = codescore - cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap) - playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap) - re := &UserSummaryCurrentYear{ - ID: dateRecordAll.ID, - Name: dateRecordAll.Name, - Email: dateRecordAll.Email, - Phone: dateRecordAll.Phone, - RegistDate: dateRecordAll.RegistDate, - DateCount: int(subTime.Hours()) / 24, - MostActiveDay: mostActiveDay, - RepoInfo: repoInfo, - DataSetInfo: dataSetInfo, - CodeInfo: codeInfo, - CloudBrainInfo: cloudBrainInfo, - PlayARoll: playARoll, - } - statictisSess.Insert(re) - } } if len(dateRecordBatch) > 0 { err := insertTable(dateRecordBatch, tableName, statictisSess) @@ -779,6 +748,127 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount)) } +func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) { + sess := x.NewSession() + defer sess.Close() + log.Info("RefreshUserYearTable start....") + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + log.Info("UserYear StartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + log.Info("UserYear EndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) + + start_unix := pageStartTime.Unix() + end_unix := pageEndTime.Unix() + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryCreateIssue(start_unix, end_unix) + + CommentCountMap := queryComment(start_unix, end_unix) + + CommitCodeSizeMap, err := GetAllUserKPIStats(pageStartTime, pageEndTime) + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix) + + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + + _, CollectedDataset := queryDatasetStars(start_unix, end_unix) + _, CreatedDataset := queryRecommedDataSet(start_unix, end_unix) + + bonusMap := getBonusMap() + log.Info("truncate all data from table:user_summary_current_year ") + statictisSess.Exec("TRUNCATE TABLE user_summary_current_year") + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") + return + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + for _, userRecord := range userList { + var dateRecordAll UserBusinessAnalysisAll + dateRecordAll.ID = userRecord.ID + dateRecordAll.Email = userRecord.Email + dateRecordAll.Phone = userRecord.PhoneNumber + dateRecordAll.RegistDate = userRecord.CreatedUnix + dateRecordAll.Name = userRecord.Name + + dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap) + dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap) + dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap) + dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap) + + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + //dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap) + dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap) + dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap) + dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap) + dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap) + + dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap) + dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) + + //年度数据 + subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC()) + mostActiveDay := "" + if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok { + mostActiveDay = getMostActiveJson(userInfo) + } + scoreMap := make(map[string]float64) + repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap) + dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset) + scoreMap["datasetscore"] = datasetscore + codeInfo, codescore := getCodeInfo(dateRecordAll) + scoreMap["codescore"] = codescore + cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap) + playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap) + re := &UserSummaryCurrentYear{ + ID: dateRecordAll.ID, + Name: dateRecordAll.Name, + Email: dateRecordAll.Email, + Phone: dateRecordAll.Phone, + RegistDate: dateRecordAll.RegistDate, + DateCount: int(subTime.Hours()) / 24, + MostActiveDay: mostActiveDay, + RepoInfo: repoInfo, + DataSetInfo: dataSetInfo, + CodeInfo: codeInfo, + CloudBrainInfo: cloudBrainInfo, + PlayARoll: playARoll, + } + statictisSess.Insert(re) + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + log.Info("update user year data finished. ") +} + func isUserYearData(tableName string) bool { if tableName == "user_business_analysis_current_year" { currentTimeNow := time.Now() @@ -998,7 +1088,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + - "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num) " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num,model_convert_count) " + "VALUES" for i, record := range dateRecords { @@ -1007,7 +1097,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," + - fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + ")" + fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + "," + fmt.Sprint(record.ModelConvertCount) + ")" if i < (len(dateRecords) - 1) { insertBatchSql += "," } @@ -1098,6 +1188,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) @@ -1179,7 +1270,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) - + dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset) @@ -1368,6 +1459,7 @@ func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) @@ -1393,6 +1485,7 @@ func getUserActivateAll(dateRecord UserBusinessAnalysisAll) int { result += dateRecord.CreateRepoCount result += dateRecord.CloudBrainTaskNum result += dateRecord.CommitModelCount + result += dateRecord.ModelConvertCount result += dateRecord.CommitDatasetNum result += dateRecord.FocusOtherUser result += dateRecord.CollectDataset @@ -1414,6 +1507,7 @@ func getUserActivate(dateRecord UserBusinessAnalysis) int { result += dateRecord.CreateRepoCount result += dateRecord.CloudBrainTaskNum result += dateRecord.CommitModelCount + result += dateRecord.ModelConvertCount result += dateRecord.CommitDatasetNum result += dateRecord.FocusOtherUser result += dateRecord.CollectDataset @@ -1450,6 +1544,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64 result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) @@ -1494,10 +1589,6 @@ func getInt(str string) int { return int(re) } -func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { - CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) -} - func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() @@ -2278,6 +2369,38 @@ func queryUserModel(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserModelConvert(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(AiModelConvert)) + if err != nil { + log.Info("query AiModelConvert error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id").Table("ai_model_convert").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + aiModelList := make([]*AiModelConvert, 0) + sess.Find(&aiModelList) + log.Info("query AiModelConvert size=" + fmt.Sprint(len(aiModelList))) + for _, aiModelRecord := range aiModelList { + if _, ok := resultMap[aiModelRecord.UserId]; !ok { + resultMap[aiModelRecord.UserId] = 1 + } else { + resultMap[aiModelRecord.UserId] += 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) { sess := x.NewSession() defer sess.Close() diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 9dcc12342..00c7f6176 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -89,6 +89,7 @@ type UserBusinessAnalysisCurrentYear struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLast30Day struct { @@ -157,6 +158,7 @@ type UserBusinessAnalysisLast30Day struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLastMonth struct { @@ -225,6 +227,7 @@ type UserBusinessAnalysisLastMonth struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisCurrentMonth struct { @@ -293,6 +296,7 @@ type UserBusinessAnalysisCurrentMonth struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisCurrentWeek struct { @@ -362,6 +366,7 @@ type UserBusinessAnalysisCurrentWeek struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisYesterday struct { @@ -431,6 +436,7 @@ type UserBusinessAnalysisYesterday struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLastWeek struct { @@ -500,6 +506,7 @@ type UserBusinessAnalysisLastWeek struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserAnalysisPara struct { @@ -616,6 +623,7 @@ type UserBusinessAnalysisAll struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysis struct { @@ -704,4 +712,5 @@ type UserBusinessAnalysis struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 6a1fc6e39..5907a3418 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,10 +5,11 @@ package cron import ( - "code.gitea.io/gitea/modules/setting" "context" "time" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/urfs_client/urchin" cloudbrainService "code.gitea.io/gitea/services/cloudbrain" @@ -296,7 +297,7 @@ func registerHandleCloudbrainDurationStatistic() { RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "1 0 * * * ?", + Schedule: "1 1 * * * ?", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.CloudbrainDurationStatisticHour() return nil diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 54be9b394..dcad1eb00 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -283,6 +283,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str Parameter: req.Parameters, UserImageUrl: req.UserImageUrl, UserCommand: req.UserCommand, + ShareAddr: setting.ModelArtsShareAddr, + MountPath: setting.ModelArtsMountPath, + NasType: setting.ModelArtsNasType, }, }) } else { @@ -303,6 +306,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str Code: req.Spec.SourceSpecId, }, Parameter: req.Parameters, + ShareAddr: setting.ModelArtsShareAddr, + MountPath: setting.ModelArtsMountPath, + NasType: setting.ModelArtsNasType, }, }) } @@ -421,6 +427,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job PreVersionId: req.PreVersionId, UserImageUrl: req.UserImageUrl, UserCommand: req.UserCommand, + ShareAddr: setting.ModelArtsShareAddr, + MountPath: setting.ModelArtsMountPath, + NasType: setting.ModelArtsNasType, }, }, jobId) } else { @@ -440,6 +449,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job }, Parameter: req.Parameters, PreVersionId: req.PreVersionId, + ShareAddr: setting.ModelArtsShareAddr, + MountPath: setting.ModelArtsMountPath, + NasType: setting.ModelArtsNasType, }, }, jobId) } diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index 3ccba9011..a8981cf9a 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -497,7 +497,7 @@ sendjob: } req, _ := json.Marshal(createJobParams) - log.Info("%s", req) + log.Info("postapi json: %s", req) if res.StatusCode() == http.StatusUnauthorized && retry < 1 { retry++ @@ -543,6 +543,8 @@ func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.Create var result models.CreateTrainJobResult retry := 0 + req, _ := json.Marshal(createJobParams) + log.Info("postapi json: %s", req) sendjob: res, err := client.R(). diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 68c2c641b..d4e776062 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -585,6 +585,9 @@ var ( TrainJobFLAVORINFOS string ModelArtsSpecialPools string ModelArtsMultiNode string + ModelArtsShareAddr string + ModelArtsMountPath string + ModelArtsNasType string //kanban IsCloudbrainTimingEnabled bool @@ -1557,6 +1560,9 @@ func NewContext() { TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("") ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("") + ModelArtsShareAddr = sec.Key("ModelArts_Share_Addr").MustString("192.168.0.30:/") + ModelArtsMountPath = sec.Key("ModelArts_Mount_Path").MustString("/cache/sfs") + ModelArtsNasType = sec.Key("ModelArts_Nas_Type").MustString("nfs") sec = Cfg.Section("elk") ElkUrl = sec.Key("ELKURL").MustString("") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ad1f3c546..1a33b4d69 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3125,9 +3125,9 @@ reject_pull_request = `suggested changes for %s#%[2]s` upload_dataset=`upload dataset %s` task_gpudebugjob=`created CPU/GPU type debugging task %s` task_npudebugjob=`created NPU type debugging task %s` -task_c2entgpudebugjob=`created CPU/GPU type debugging task %s` -task_c2entnpudebugjob=`created NPU type debugging task %s` -task_c2entgcudebugjob=`created GCU type debugging task %s` +task_c2net_gpudebugjob=`created CPU/GPU type debugging task %s` +task_c2net_npudebugjob=`created NPU type debugging task %s` +task_c2ent_gcudebugjob=`created GCU type debugging task %s` task_nputrainjob=`created NPU training task %s` task_inferencejob=`created reasoning task %s` task_benchmark=`created profiling task %s` diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 14ff9d882..cb8fb682e 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3143,9 +3143,9 @@ reject_pull_request=`建议变更 %s#%[2]s` upload_dataset=`上传了数据集文件 %s` task_gpudebugjob=`创建了CPU/GPU类型调试任务 %s` task_npudebugjob=`创建了NPU类型调试任务 %s` -task_c2entgpudebugjob=`创建了CPU/GPU类型调试任务 %s` -task_c2entnpudebugjob=`创建了NPU类型调试任务 %s` -task_c2entgcudebugjob=`创建了GCU类型调试任务 %s` +task_c2net_gpudebugjob=`创建了CPU/GPU类型调试任务 %s` +task_c2net_npudebugjob=`创建了NPU类型调试任务 %s` +task_c2ent_gcudebugjob=`创建了GCU类型调试任务 %s` task_nputrainjob=`创建了NPU类型训练任务 %s` task_inferencejob=`创建了推理任务 %s` task_benchmark=`创建了评测任务 %s` diff --git a/public/home/home.js b/public/home/home.js index 8fb7b08b1..2c70ca8ee 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -244,11 +244,11 @@ document.onreadystatechange = function () { html += " " + getRepotext(record) + "" } else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" - || record.OpType == "31" || record.OpType == "32" || record.OpType == "33" || record.OpType == "39" || record.OpType == "40" || record.OpType == "41"){ + || record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){ html += recordPrefix + actionName; html += " " + record.RefName + "" } - else if(record.OpType == "25" || record.OpType == "29"){ + else if(record.OpType == "25" || record.OpType == "29" || record.OpType == "39" || record.OpType == "40" || record.OpType == "41"){ html += recordPrefix + actionName; html += " " + record.RefName + "" } @@ -298,6 +298,7 @@ function getTaskLink(record){ }else if(record.OpType == 39 || record.OpType == 40 || record.OpType == 41){ re = re + "/grampus/notebook/" + record.Content; } + re = encodeURI(re); return re; } diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 4a982f799..805443788 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -9,6 +9,7 @@ import ( "bufio" "encoding/json" "io" + "io/ioutil" "net/http" "os" "path" @@ -647,6 +648,19 @@ func CloudbrainDownloadLogFile(ctx *context.Context) { } } + existStr := "" + if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) { + if job.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(job.JobID) + if err == nil && result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + existStr = taskRes.TaskStatuses[0].ExitDiagnostics + } + } + } + logDir := "/model" if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { logDir = cloudbrain.ResultPath @@ -664,17 +678,30 @@ func CloudbrainDownloadLogFile(ctx *context.Context) { } } if fileName != "" { - prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir - url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName) + prefix := "/" + setting.CBCodePathPrefix + job.JobName + "/model" + filePath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + prefix + "/" + fileName + // Read the file contents into a byte slice + data, err := ioutil.ReadFile(filePath) if err != nil { - log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("ReadFile", err) + return + } + + // Set the appropriate response headers + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + + // Write the file contents to the response + if _, err := ctx.Resp.Write(data); err != nil { + ctx.ServerError("Write", err) + return + } + if _, err := ctx.Resp.Write([]byte(existStr)); err != nil { + log.Error("Write failed: %v", err.Error(), ctx.Data["msgID"]) return } - log.Info("fileName=" + fileName) - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) } else { log.Info("fileName is null.") - } } @@ -760,8 +787,28 @@ func CloudbrainGetLog(ctx *context.APIContext) { content = result["Content"].(string) } - if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 { - content = content + ctx.Data["existStr"].(string) + if (job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference)) && job.Type == models.TypeCloudBrainOne && job.Status == string(models.JobFailed) { + if ctx.Data["existStr"] != nil { + if baseLine == "" && order == "desc" && result["Lines"].(int) == 0 { + result["Lines"] = 1 + result["EndLine"] = 1 + content = content + ctx.Data["existStr"].(string) + } + + if result["Lines"].(int) == 0 && result["StartLine"] == result["EndLine"] && result["StartLine"].(int) != 0 { + content = content + ctx.Data["existStr"].(string) + result["Lines"] = 1 + result["StartLine"] = result["StartLine"].(int) - 1 + } + if result["Lines"].(int) == 1 && result["StartLine"] == result["EndLine"] { + result["Lines"] = 0 + result["StartLine"] = result["StartLine"].(int) + 1 + } + } + } else { + if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 { + content = content + ctx.Data["existStr"].(string) + } } logFileName := result["FileName"] diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0d68fff30..bb04038b9 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -103,86 +103,62 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { }) } func GetOverviewDuration(ctx *context.Context) { - recordCloudbrain, err := models.GetRecordBeginTime() - if err != nil { - log.Error("Can not get recordCloudbrain", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return - } - recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix - now := time.Now() - endTime := now - var workServerNumber int64 - var cardNum int64 + durationSum := 0 + cardDurationSum := 0 - durationAllSum := int64(0) - cardDuSum := int64(0) + cloudBrainOneCardDuSum := 0 + cloudBrainTwoCardDuSum := 0 + c2NetCardDuSum := 0 + cDNetCardDuSum := 0 - cloudBrainOneCardDuSum := int64(0) - cloudBrainTwoCardDuSum := int64(0) - c2NetCardDuSum := int64(0) - cDNetCardDuSum := int64(0) + cloudBrainOneDuration := 0 + cloudBrainTwoDuration := 0 + c2NetDuration := 0 + cDCenterDuration := 0 - cloudBrainOneDuration := int64(0) - cloudBrainTwoDuration := int64(0) - c2NetDuration := int64(0) - cDCenterDuration := int64(0) - - cloudbrains, _, err := models.CloudbrainAllKanBan(&models.CloudbrainsOptions{ - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) + cloudbrainTypeDuration, err := models.GetCloudbrainTypeCardDuration() if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) + log.Error("GetCloudbrainTypeCardDuration err!", err) return } - models.LoadSpecs4CloudbrainInfo(cloudbrains) - - for _, cloudbrain := range cloudbrains { - cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) - if cloudbrain.Cloudbrain.Spec != nil { - cardNum = int64(cloudbrain.Cloudbrain.Spec.AccCardsNum) - } else { - cardNum = 1 + for _, result := range cloudbrainTypeDuration { + if result.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration = result.DurationSum + cloudBrainOneCardDuSum = result.CardDurationSum } - if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { - workServerNumber = int64(cloudbrain.Cloudbrain.WorkServerNumber) - } else { - workServerNumber = 1 + if result.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration = result.DurationSum + cloudBrainTwoCardDuSum = result.CardDurationSum } - duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration) - CardDuration := workServerNumber * int64(cardNum) * duration - - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration += duration - cloudBrainOneCardDuSum += CardDuration - } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration += duration - cloudBrainTwoCardDuSum += CardDuration - } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - c2NetDuration += duration - c2NetCardDuSum += CardDuration - } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { - cDCenterDuration += duration - cDNetCardDuSum += CardDuration + if result.Type == models.TypeC2Net { + c2NetDuration = result.DurationSum + c2NetCardDuSum = result.CardDurationSum } - - durationAllSum += duration - cardDuSum += CardDuration + if result.Type == models.TypeCDCenter { + cDCenterDuration = result.DurationSum + cDNetCardDuSum = result.CardDurationSum + } + } + cloudbrainAllDuration, err := models.GetCloudbrainAllCardDuration() + if err != nil { + log.Error("GetCloudbrainAllCardDuration err!", err) + return } + durationSum = cloudbrainAllDuration.DurationSum + cardDurationSum = cloudbrainAllDuration.CardDurationSum + ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, "cloudBrainTwoCardDuSum": cloudBrainTwoCardDuSum, "c2NetCardDuSum": c2NetCardDuSum, "cDNetCardDuSum": cDNetCardDuSum, - "cardDuSum": cardDuSum, + "cardDuSum": cardDurationSum, "cloudBrainOneDuration": cloudBrainOneDuration, "cloudBrainTwoDuration": cloudBrainTwoDuration, "c2NetDuration": c2NetDuration, "cDCenterDuration": cDCenterDuration, - "durationSum": durationAllSum, + "durationSum": durationSum, }) } diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 9839a5041..36664458e 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -603,7 +603,7 @@ func stopModelConvert(id string) error { } func StopModelConvertApi(ctx *context.Context) { - id := ctx.Params(":id") + id := ctx.Query("id") log.Info("stop model convert start.id=" + id) err := stopModelConvert(id) if err == nil { diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 6176fcda5..55f25dba6 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -847,6 +847,9 @@ func createForGPU(ctx *context.Context, jobName string) error { codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath os.RemoveAll(codePath) + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(cloudbrain.DefaultBranchName) + if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) return errors.New("system error") @@ -891,7 +894,7 @@ func createForGPU(ctx *context.Context, jobName string) error { BranchName: cloudbrain.DefaultBranchName, BootFile: BootFile, Params: Params, - CommitID: "", + CommitID: commitID, ModelName: modelName, ModelVersion: modelVersion, CkptName: CkptName, diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index de95babe9..43c1ab1a4 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -14,13 +14,7 @@ import ( ) func CloudbrainDurationStatisticHour() { - defer func() { - err := recover() - if err == nil { - return - } - }() - if setting.IsCloudbrainTimingEnabled { + if setting.IsCloudbrainTimingEnabled { var statisticTime time.Time var count int64 recordDurationUpdateTime, err := models.GetDurationRecordUpdateTime() @@ -35,17 +29,16 @@ func CloudbrainDurationStatisticHour() { statisticTime = currentTime } - err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(statisticTime.Add(-1*time.Hour).Unix()), timeutil.TimeStamp(currentTime.Unix())) + err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(statisticTime.Unix()), timeutil.TimeStamp(currentTime.Unix())) if err != nil { log.Error("DeleteCloudbrainDurationStatistic failed", err) } - + statisticTime = statisticTime.Add(+1 * time.Hour) for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) { countEach := summaryDurationStat(statisticTime) count += countEach statisticTime = statisticTime.Add(+1 * time.Hour) } - log.Info("summaryDurationStat count: %v", count) } } func UpdateDurationStatisticHistoryData(beginTime time.Time, endTime time.Time) int64 { @@ -71,15 +64,18 @@ func summaryDurationStat(statisticTime time.Time) int64 { ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime) if err != nil { - log.Info("GetCloudbrainByTime err: %v", err) + log.Error("GetCloudbrainByTime err: %v", err) return 0 } - models.LoadSpecs4CloudbrainInfo(ciTasks) - cloudBrainCenterCodeAndCardTypeInfo, cloudbrainMap := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime) + err = models.LoadSpecs4CloudbrainInfo(ciTasks) + if err != nil { + log.Error("LoadSpecs4CloudbrainInfo err: %v", err) + } + cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, int(beginTime), int(endTime)) resourceQueues, err := models.GetCanUseCardInfo() if err != nil { - log.Info("GetCanUseCardInfo err: %v", err) + log.Error("GetCanUseCardInfo err: %v", err) return 0 } @@ -91,56 +87,45 @@ func summaryDurationStat(statisticTime time.Time) int64 { cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType] += resourceQueue.CardsTotalNum * 1 * 60 * 60 } } - - for centerCode, CardTypes := range cloudBrainCenterCodeAndCardTypeInfo { - for cardType, cardDuration := range CardTypes { - cloudbrainTable := cloudbrainMap[centerCode+"/"+cardType] - if cloudbrainTable != nil { - if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType]; !ok { - cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType] = 0 - } - cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - DateTimeUnix: dateTimeUnix, - DayTime: dayTime, - HourTime: hourTime, - Cluster: cloudbrainTable.Cluster, - AiCenterName: GetAiCenterNameByCode(centerCode, "zh-CN"), - AiCenterCode: centerCode, - AccCardType: cardType, - CardsUseDuration: cardDuration, - CardsTotalDuration: cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType], - CreatedUnix: timeutil.TimeStampNow(), - } - if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { - log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) - } - count++ - delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType) - } - } - } - for key, cardsTotalDuration := range cardsTotalDurationMap { - cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - DateTimeUnix: dateTimeUnix, - DayTime: dayTime, - HourTime: hourTime, - Cluster: strings.Split(key, "/")[0], - AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"), - AiCenterCode: strings.Split(key, "/")[1], - AccCardType: strings.Split(key, "/")[2], - CardsUseDuration: 0, - CardsTotalDuration: cardsTotalDuration, - CardsTotalNum: cardsTotalDuration / 1 / 60 / 60, - CreatedUnix: timeutil.TimeStampNow(), - } - if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { - log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + if _, ok := cloudBrainCenterCodeAndCardTypeInfo[strings.Split(key, "/")[0]+"/"+strings.Split(key, "/")[1]][strings.Split(key, "/")[2]]; ok { + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTimeUnix: dateTimeUnix, + DayTime: dayTime, + HourTime: hourTime, + Cluster: strings.Split(key, "/")[0], + AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"), + AiCenterCode: strings.Split(key, "/")[1], + AccCardType: strings.Split(key, "/")[2], + CardsUseDuration: cloudBrainCenterCodeAndCardTypeInfo[strings.Split(key, "/")[0]+"/"+strings.Split(key, "/")[1]][strings.Split(key, "/")[2]], + CardsTotalDuration: cardsTotalDuration, + CardsTotalNum: cardsTotalDuration / 1 / 60 / 60, + CreatedUnix: timeutil.TimeStampNow(), + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + count++ + } else { + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTimeUnix: dateTimeUnix, + DayTime: dayTime, + HourTime: hourTime, + Cluster: strings.Split(key, "/")[0], + AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"), + AiCenterCode: strings.Split(key, "/")[1], + AccCardType: strings.Split(key, "/")[2], + CardsUseDuration: 0, + CardsTotalDuration: cardsTotalDuration, + CardsTotalNum: cardsTotalDuration / 1 / 60 / 60, + CreatedUnix: timeutil.TimeStampNow(), + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + count++ } - count++ } - - log.Info("finish summary cloudbrainDurationStat") return count } @@ -159,33 +144,21 @@ func GetAiCenterNameByCode(centerCode string, language string) string { return aiCenterName } -func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) (map[string]map[string]int, map[string]*models.Cloudbrain) { +func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, hourBeginTime int, hourEndTime int) map[string]map[string]int { var WorkServerNumber int var AccCardsNum int - cloudbrainMap := make(map[string]*models.Cloudbrain) cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) for _, cloudbrain := range ciTasks { - if cloudbrain.Cloudbrain.StartTime == 0 { - cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix - } - if cloudbrain.Cloudbrain.EndTime == 0 { - cloudbrain.Cloudbrain.EndTime = timeutil.TimeStamp(time.Now().Unix()) - } - cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) - if cloudbrain.Cloudbrain.Spec != nil { - if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if cloudbrain.Cloudbrain.Spec != nil { - cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = &cloudbrain.Cloudbrain - } - } - } - cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) if cloudbrain.Cloudbrain.StartTime == 0 { cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix } if cloudbrain.Cloudbrain.EndTime == 0 { - cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix + if cloudbrain.Cloudbrain.Status == string(models.JobRunning) { + cloudbrain.Cloudbrain.EndTime = timeutil.TimeStamp(time.Now().Unix()) + } else { + cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.StartTime + timeutil.TimeStamp(cloudbrain.Cloudbrain.Duration) + } } if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber @@ -197,55 +170,36 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be } else { AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter]; !ok { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter] = make(map[string]int) + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter]; !ok { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter] = make(map[string]int) } + taskStartTime := int(cloudbrain.Cloudbrain.StartTime) + taskEndTime := int(cloudbrain.Cloudbrain.EndTime) if cloudbrain.Cloudbrain.Spec != nil { - if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) && cloudbrain.Cloudbrain.DeletedAt.IsZero() { - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = 0 - } - } else { - if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += 0 - } + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if taskStartTime < hourBeginTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (taskEndTime - hourBeginTime) + } else if taskStartTime < hourBeginTime && taskEndTime > hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (hourEndTime - hourBeginTime) + } else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (taskEndTime - taskStartTime) + } else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime > hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (hourEndTime - taskStartTime) } } else { - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - } - } else { - if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - } + if taskStartTime < hourBeginTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (taskEndTime - hourBeginTime) + } else if taskStartTime < hourBeginTime && taskEndTime > hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (hourEndTime - hourBeginTime) + } else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (taskEndTime - taskStartTime) + } else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime > hourEndTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (hourEndTime - taskStartTime) } } } } - - return cloudBrainCenterCodeAndCardType, cloudbrainMap + return cloudBrainCenterCodeAndCardType } func CloudbrainUpdateHistoryData(ctx *context.Context) { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index ec6c5d671..bc47d5730 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -974,7 +974,7 @@ func GrampusStopJob(ctx *context.Context) { if task.Status == models.GrampusStatusStopped || task.Status == models.GrampusStatusFailed || task.Status == models.GrampusStatusSucceeded { log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) resultCode = "-1" - errorMsg = "System error" + errorMsg = ctx.Tr("cloudbrain.Already_stopped") break } @@ -982,7 +982,7 @@ func GrampusStopJob(ctx *context.Context) { if err != nil { log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) resultCode = strconv.Itoa(res.ErrorCode) - errorMsg = res.ErrorMsg + errorMsg = ctx.Tr("cloudbrain.Stopped_failed") break } oldStatus := task.Status diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 8ae3528f2..3dbe101a8 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -346,9 +346,12 @@ func NotebookShow(ctx *context.Context) { } func GetModelDownload(task *models.Cloudbrain) models.ModelDownload { + index := strings.Index(task.PreTrainModelUrl, "/") + key := task.PreTrainModelUrl[index+1:] + task.CkptName + url, _ := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key) modelDownload := models.ModelDownload{ Name: task.CkptName, - DownloadLink: "s3://" + task.PreTrainModelUrl + task.CkptName, + DownloadLink: url, IsDelete: false, } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index a6de283a4..8121f167c 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -21,6 +21,7 @@ import ( const ( PAGE_SIZE = 2000 Excel_File_Path = "/useranalysis/" + USER_YEAR = 2022 ) func getUserMetricsExcelHeader(ctx *context.Context) map[string]string { @@ -104,6 +105,7 @@ func getExcelHeader(ctx *context.Context) map[string]string { excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.ModelConvertCount")) excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser")) excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset")) @@ -178,6 +180,8 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount) + tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) @@ -256,6 +260,8 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord * tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount) + tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) @@ -714,6 +720,12 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Info("startTime time:" + startTime.Format("2006-01-02 15:04:05")) log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05")) warnEmailMessage := "用户统计信息入库失败，请尽快定位。" + + startYear := time.Date(USER_YEAR, 1, 1, 0, 0, 0, 1, t.Location()) + endYear := startYear.AddDate(1, 0, 0) + + models.RefreshUserYearTable(startYear, endYear) + //query wiki data log.Info("start to time count data") wikiMap, err := queryWikiCountMap(startTime, endTime) diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 5ebbba798..b02cd5caf 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -16,7 +16,7 @@ type StatusInfo struct { var CloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)} var CloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)} -var GrampusNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning, models.GrampusStatusStopping} +var GrampusNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning} var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeDebug}, diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go index 0c3d9d1a2..2a752acf5 100755 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 41} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 39, 40, 41} type ClientsManager struct { Clients *orderedmap.OrderedMap diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index a2c714de5..50225ef65 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -96,16 +96,9 @@ {{end}}
- {{if and (eq .JobType "DEBUG") (eq .Cloudbrain.Type 0)}} + {{if eq .JobType "DEBUG"}} - {{.DisplayJobName}} - - {{else if eq .JobType "DEBUG"}} - {{.DisplayJobName}} @@ -142,13 +135,13 @@
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} + {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -240,7 +233,7 @@ {{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}} {{$.i18n.Tr "repo.debug"}} @@ -276,7 +269,7 @@ + data-jobid="{{$JobID}}" data-bootfile="{{.BootFile}}"> {{$.i18n.Tr "repo.stop"}} @@ -311,17 +304,17 @@ {{else}} -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.delete"}} - -
+
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
{{end}}
diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index eb287cc4d..1c56c1465 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -125,7 +125,7 @@
-