diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 44b669181..bb1241247 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -87,7 +87,7 @@ type Cloudbrain struct { LogUrl string //日志输出的obs路径 PreVersionId int64 //父版本的版本id FlavorCode string //modelarts上的规格id - Description string //描述 + Description string `xorm:"varchar(256)"` //描述 WorkServerNumber int //节点数 FlavorName string //规格名称 EngineName string //引擎名称 diff --git a/models/models.go b/models/models.go index 4d39e2f14..809f9b0ca 100755 --- a/models/models.go +++ b/models/models.go @@ -139,6 +139,7 @@ func init() { new(RepoStatistic), new(SummaryStatistic), new(UserBusinessAnalysis), + new(UserBusinessAnalysisAll), new(UserLoginLog), ) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 08be81241..f81683242 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -4,13 +4,80 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) +type UserBusinessAnalysisAll struct { + ID int64 `xorm:"pk"` + + CountDate int64 `xorm:"pk"` + + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCommitRepo // 5 + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + + //action :ActionCreateIssue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + + //user + Email string `xorm:"NOT NULL"` + + //user + Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NULL"` +} + type UserBusinessAnalysis struct { ID int64 `xorm:"pk"` @@ -93,46 +160,12 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool { return ulist[i].ID > ulist[j].ID } -func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { - log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") - - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - statictisSess.Find(&userBusinessAnalysisList) - - resultMap := make(map[int64]*UserBusinessAnalysis) - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } - } +type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList +func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisAllList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool { + return ulist[i].ID > ulist[j].ID } func getLastCountDate() int64 { @@ -153,71 +186,36 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } -func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { +func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) statictisSess := xStatistic.NewSession() defer statictisSess.Close() - resultMap := make(map[int64]*UserBusinessAnalysis) - - var newAndCond = builder.NewCond() - if len(opts.UserName) > 0 { - newAndCond = newAndCond.And( - builder.Like{"name", opts.UserName}, - ) - } - if !opts.IsAll { - newAndCond = newAndCond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - newAndCond = newAndCond.And( - builder.Lte{"count_date": opts.EndTime}, - ) - } - - allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll)) if err != nil { log.Info("query error." + err.Error()) return nil, 0 } log.Info("query return total:" + fmt.Sprint(allCount)) - pageSize := 200 + if allCount == 0 { + RefreshUserStaticAllTabel() + } + pageSize := 1000 totalPage := int(allCount) / pageSize - + userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} for i := 0; i <= int(totalPage); i++ { - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). - Find(&userBusinessAnalysisList); err != nil { + userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) + if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisAllList); err != nil { return nil, 0 } - log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } + log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList))) + for _, userRecord := range userBusinessAnalysisAllList { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord) } } - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, allCount @@ -338,6 +336,50 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } +func RefreshUserStaticAllTabel() { + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + log.Info("delete all data from table: user_business_analysis_all") + statictisSess.Exec("delete from user_business_analysis_all") + + currentTimeNow := time.Now() + pageStartTime := getLastCountDate() + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() + + var cond = builder.NewCond() + cond = cond.And( + builder.Gte{"count_date": pageStartTime}, + ) + cond = cond.And( + builder.Lte{"count_date": pageEndTime}, + ) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). + Find(&userBusinessAnalysisList); err != nil { + return + } + log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) + allData := getAllData(userRecord.ID, statictisSess) + allData.ID = userRecord.ID + allData.CountDate = 0 + allData.DataDate = userRecord.DataDate + allData.Email = userRecord.Email + allData.OpenIIndex = userRecord.OpenIIndex + allData.GiteaAgeMonth = userRecord.GiteaAgeMonth + allData.Name = userRecord.Name + allData.RegistDate = userRecord.RegistDate + + _, err := statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + log.Info("refresh all data finished.") +} + func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { log.Info("start to count other user info data") @@ -491,10 +533,99 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("insert daterecord failed." + err.Error()) return err } + + if isExistUserInAllTable(dateRecord.ID, statictisSess) { + updateCurrentData(dateRecord.ID, statictisSess, dateRecord) + } else { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) + allData := getAllData(dateRecord.ID, statictisSess) + allData.ID = dateRecord.ID + allData.CountDate = 0 + allData.DataDate = dateRecord.DataDate + allData.Email = dateRecord.Email + allData.OpenIIndex = dateRecord.OpenIIndex + allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth + allData.Name = dateRecord.Name + allData.RegistDate = dateRecord.RegistDate + + _, err = statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + return err + } + } + } return nil } +func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { + + _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + + ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + + ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + + ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + + ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + + ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + + ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + + ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + + ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + + ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + + ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + + ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + + ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + + ",login_count+=" + fmt.Sprint(currentData.LoginCount) + + " where id=" + fmt.Sprint(userId)) + + if err != nil { + log.Info("update table failed." + err.Error()) + } + +} + +func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { + + allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) + if err != nil { + return false + } + return allCount > 0 +} + +func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { + var dateRecord UserBusinessAnalysisAll + + rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") + if err == nil { + for i, row := range rows { + log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) + dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) + dateRecord.CommitCount = getInt(string(row["commit_count"])) + dateRecord.IssueCount = getInt(string(row["issue_count"])) + dateRecord.CommentCount = getInt(string(row["comment_count"])) + dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) + dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) + dateRecord.WatchedCount = getInt(string(row["watched_count"])) + dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) + dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) + dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) + dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) + dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) + dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) + dateRecord.LoginCount = getInt(string(row["login_count"])) + } + } + return dateRecord +} + +func getInt(str string) int { + re, err := strconv.ParseInt(str, 10, 32) + if err != nil { + return 0 + } + return int(re) +} + func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) } diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 39312bfde..9ca26b357 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -176,10 +176,10 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } -func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { +func GetObsListObject(jobName, parentDir string, versionOutputPath string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, versionOutputPath, parentDir), "/") strPrefix := strings.Split(input.Prefix, "/") output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index c53e62efa..9637bae99 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -303,8 +303,7 @@ func ModelList(ctx *context.APIContext) { return } VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) - parentDir = VersionOutputPath + "/" + parentDir - models, err := storage.GetObsListObject(task.JobName, parentDir) + models, err := storage.GetObsListObject(task.JobName, parentDir, VersionOutputPath) if err != nil { log.Info("get TrainJobListModel failed:", err) ctx.ServerError("GetObsListObject:", err) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index a1c754951..83498b70e 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -380,6 +380,85 @@ func trainJobNewDataPrepare(ctx *context.Context) error { return nil } +func ErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { + ctx.Data["PageIsCloudBrain"] = true + + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //if !can { + // log.Error("the user can not create train-job") + // ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) + // return + //} + + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return err + } + ctx.Data["attachments"] = attachs + + var resourcePools modelarts.ResourcePool + if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engine_versions"] = versionInfos.Version + + var flavorInfos modelarts.Flavor + if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["flavor_infos"] = flavorInfos.Info + + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + ctx.Data["train_url"] = outputObsPath + + Branches, err := ctx.Repo.GitRepo.GetBranches() + if err != nil { + ctx.ServerError("GetBranches error:", err) + return err + } + ctx.Data["Branches"] = Branches + ctx.Data["BranchesCount"] = len(Branches) + + configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) + if err != nil { + ctx.ServerError("getConfigList failed:", err) + return err + } + ctx.Data["config_list"] = configList.ParaConfigs + ctx.Data["bootFile"] = form.BootFile + ctx.Data["uuid"] = form.Attachment + ctx.Data["params"] = form.Params + ctx.Data["branch_name"] = form.BranchName + + return nil +} + func TrainJobNewVersion(ctx *context.Context) { err := trainJobNewVersionDataPrepare(ctx) if err != nil { @@ -477,6 +556,94 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { return nil } +func VersionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { + ctx.Data["PageIsCloudBrain"] = true + var jobID = ctx.Params(":jobid") + // var versionName = ctx.Params(":version-name") + var versionName = ctx.Query("version_name") + + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) + return err + } + + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = task.JobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return err + } + ctx.Data["attachments"] = attachs + + var resourcePools modelarts.ResourcePool + if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engine_versions"] = versionInfos.Version + + var flavorInfos modelarts.Flavor + if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["flavor_infos"] = flavorInfos.Info + + var Parameters modelarts.Parameters + if err = json.Unmarshal([]byte(form.Params), &Parameters); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["params"] = Parameters.Parameter + + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + ctx.Data["train_url"] = outputObsPath + + Branches, err := ctx.Repo.GitRepo.GetBranches() + if err != nil { + ctx.ServerError("GetBranches error:", err) + return err + } + ctx.Data["branches"] = Branches + ctx.Data["description"] = form.Description + ctx.Data["dataset_name"] = task.DatasetName + ctx.Data["work_server_number"] = form.WorkServerNumber + ctx.Data["flavor_name"] = form.FlavorName + ctx.Data["engine_name"] = form.EngineName + ctx.Data["flavor_code"] = task.FlavorCode + ctx.Data["engine_id"] = task.EngineID + + ctx.Data["bootFile"] = form.BootFile + ctx.Data["uuid"] = form.Attachment + ctx.Data["branch_name"] = form.BranchName + configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) + if err != nil { + ctx.ServerError("getConfigList failed:", err) + return err + } + ctx.Data["config_list"] = configList.ParaConfigs + + return nil +} + func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) { ctx.Data["PageIsTrainJob"] = true VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(modelarts.TotalVersionCount) @@ -504,7 +671,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - trainJobNewDataPrepare(ctx) + ErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) return } @@ -529,13 +696,6 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) }); err != nil { log.Error("创建任务失败，服务器超时！: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) - - ctx.Data["bootFile"] = form.BootFile - ctx.Data["uuid"] = form.Attachment - ctx.Data["datasetName"] = attach.Name - ctx.Data["params"] = form.Params - ctx.Data["branch_name"] = branch_name - trainJobNewDataPrepare(ctx) ctx.RenderWithErr("创建任务失败，服务器超时！", tplModelArtsTrainJobNew, &form) return } @@ -555,7 +715,9 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } + // parentDir := VersionOutputPath + "/" if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobNew, &form) @@ -651,6 +813,13 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) TotalVersionCount: modelarts.TotalVersionCount, } + //将params转换Parameters.Parameter，出错时返回给前端 + var Parameters modelarts.Parameters + if err := json.Unmarshal([]byte(params), &Parameters); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return + } + err = modelarts.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) @@ -658,7 +827,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) ctx.Data["bootFile"] = form.BootFile ctx.Data["uuid"] = form.Attachment ctx.Data["datasetName"] = attach.Name - ctx.Data["params"] = form.Params + ctx.Data["params"] = Parameters.Parameter ctx.Data["branch_name"] = branch_name ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) return @@ -689,7 +858,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath - codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" @@ -701,16 +870,16 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } - attach, err := models.GetAttachmentByUUID(uuid) - if err != nil { - log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error()) - return - } + // attach, err := models.GetAttachmentByUUID(uuid) + // if err != nil { + // log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error()) + // return + // } //todo: del the codeLocalPath _, err = ioutil.ReadDir(codeLocalPath) @@ -724,13 +893,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ Branch: branch_name, }); err != nil { log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err) - trainJobNewVersionDataPrepare(ctx) - - ctx.Data["bootFile"] = form.BootFile - ctx.Data["uuid"] = form.Attachment - ctx.Data["datasetName"] = attach.Name - ctx.Data["params"] = form.Params - ctx.Data["branch_name"] = branch_name + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobVersionNew, &form) return } @@ -738,21 +901,23 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ //todo: upload code (send to file_server todo this work?) if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobVersionNew, &form) return } if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobVersionNew, &form) return } - if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + parentDir := VersionOutputPath + "/" + // if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobVersionNew, &form) return } @@ -772,7 +937,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ err := json.Unmarshal([]byte(params), ¶meters) if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", params, err) - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobVersionNew, &form) return } @@ -791,7 +956,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if isSaveParam == "on" { if form.ParameterTemplateName == "" { log.Error("ParameterTemplateName is empty") - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数时，作业参数名称不能为空", tplModelArtsTrainJobVersionNew, &form) return } @@ -815,7 +980,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if err != nil { log.Error("Failed to CreateTrainJobConfig: %v", err) - trainJobNewVersionDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobVersionNew, &form) return } @@ -862,15 +1027,11 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ err = modelarts.GenerateTrainJobVersion(ctx, req, jobID) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) - trainJobNewVersionDataPrepare(ctx) - ctx.Data["bootFile"] = form.BootFile - ctx.Data["uuid"] = form.Attachment - ctx.Data["datasetName"] = attach.Name - ctx.Data["params"] = form.Params + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job/" + jobID) // ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) } @@ -963,7 +1124,6 @@ func paramCheckCreateTrainJob(form auth.CreateModelArtsTrainJobForm) error { func TrainJobShow(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true - var jobID = ctx.Params(":jobid") repo := ctx.Repo.Repository diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 5a6b3d724..a141df086 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -16,17 +16,6 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) -func QueryUserStaticData(ctx *context.Context) { - startDate := ctx.Query("startDate") - endDate := ctx.Query("endDate") - log.Info("startDate=" + startDate + " endDate=" + endDate) - startTime, _ := time.Parse("2006-01-02", startDate) - endTime, _ := time.Parse("2006-01-02", endDate) - endTime = endTime.AddDate(0, 0, 1) - log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) - ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) -} - func QueryUserStaticDataPage(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") @@ -131,8 +120,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) - formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") - xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) + formatTime = userRecord.DataDate + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01") } //设置默认打开的表单 @@ -171,6 +160,12 @@ func QueryUserStaticDataPage(ctx *context.Context) { } func TimingCountDataByDateAndReCount(date string, isReCount bool) { + + if date == "refreshAll" { + models.RefreshUserStaticAllTabel() + return + } + t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -223,6 +218,10 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } + + if isReCount { + models.RefreshUserStaticAllTabel() + } } func TimingCountDataByDate(date string) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index fe2588b25..d3212690c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,7 +792,6 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData) m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() {