package repo import ( "bufio" "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/services/reward/point/account" "encoding/json" "errors" "fmt" "io" "net/http" "os" "regexp" "sort" "strconv" "strings" "time" "unicode/utf8" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/modules/timeutil" "github.com/unknwon/i18n" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" ) const ( tplCloudBrainNew base.TplName = "repo/cloudbrain/new" tplCloudBrainShow base.TplName = "repo/cloudbrain/show" tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index" tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index" tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new" tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show" tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit" tplCloudBrainImageEdit base.TplName = "repo/cloudbrain/image/edit" tplCloudBrainTrainJobNew base.TplName = "repo/cloudbrain/trainjob/new" tplCloudBrainTrainJobShow base.TplName = "repo/cloudbrain/trainjob/show" tplCloudBrainInferenceJobNew base.TplName = "repo/cloudbrain/inference/new" tplCloudBrainInferenceJobShow base.TplName = "repo/cloudbrain/inference/show" ) var ( gpuInfos *models.GpuInfos categories *models.Categories benchmarkTypes *models.BenchmarkTypes benchmarkGpuInfos *models.GpuInfos benchmarkResourceSpecs *models.ResourceSpecs trainGpuInfos *models.GpuInfos inferenceGpuInfos *models.GpuInfos ) const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types" const CLONE_FILE_PREFIX = "file:///" var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0) var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) // MustEnableDataset check if repository enable internal cb func MustEnableCloudbrain(ctx *context.Context) { if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { ctx.NotFound("MustEnableCloudbrain", nil) return } } func cutString(str string, lens int) string { if len(str) < lens { return str } return str[:lens] } func jobNamePrefixValid(s string) string { lowStr := strings.ToLower(s) re := regexp.MustCompile(`[^a-z0-9_\\-]+`) removeSpecial := re.ReplaceAllString(lowStr, "") re = regexp.MustCompile(`^[_\\-]+`) return re.ReplaceAllString(removeSpecial, "") } func cloudBrainNewDataPrepare(ctx *context.Context) error { ctx.Data["PageIsCloudBrain"] = true t := time.Now() var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["display_job_name"] = displayJobName ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand() ctx.Data["code_path"] = cloudbrain.CodeMountPath ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath ctx.Data["model_path"] = cloudbrain.ModelMountPath ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled if categories == nil { json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) } ctx.Data["benchmark_categories"] = categories.Category ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType queuesDetail, _ := cloudbrain.GetQueuesDetail() if queuesDetail != nil { ctx.Data["QueuesDetail"] = queuesDetail } prepareCloudbrainOneSpecs(ctx) ctx.Data["params"] = "" ctx.Data["branchName"] = ctx.Repo.BranchName ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled ctx.Data["datasetType"] = models.TypeCloudBrainOne ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode") return nil } func prepareCloudbrainOneSpecs(ctx *context.Context) { debugSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{ JobType: models.JobTypeDebug, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne, }) ctx.Data["debug_specs"] = debugSpecs trainSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{ JobType: models.JobTypeTrain, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne, }) ctx.Data["train_specs"] = trainSpecs inferenceSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{ JobType: models.JobTypeInference, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne, }) ctx.Data["inference_specs"] = inferenceSpecs benchmarkSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{ JobType: models.JobTypeBenchmark, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne, }) ctx.Data["benchmark_specs"] = benchmarkSpecs } func CloudBrainNew(ctx *context.Context) { err := cloudBrainNewDataPrepare(ctx) if err != nil { ctx.ServerError("get new cloudbrain info failed", err) return } ctx.Data["PageIsGPUDebug"] = true ctx.HTML(200, tplCloudBrainNew) } func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.Data["PageIsCloudBrain"] = true displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := strings.TrimSpace(form.Image) uuids := form.Attachment jobType := form.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath branchName := form.BranchName bootFile := strings.TrimSpace(form.BootFile) repo := ctx.Repo.Repository tpl := tplCloudBrainNew if jobType == string(models.JobTypeTrain) { tpl = tplCloudBrainTrainJobNew } tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tpl, &form) return } count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } } var datasetInfos map[string]models.DatasetInfo var datasetNames string //var if uuids != "" { datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } } command := cloudbrain.GetCloudbrainDebugCommand() if jobType == string(models.JobTypeTrain) { bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) return } tpl = tplCloudBrainTrainJobNew commandTrain, err := getTrainJobCommand(form) if err != nil { log.Error("getTrainJobCommand failed: %v", err) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tpl, &form) return } command = commandTrain } if branchName == "" { branchName = cloudbrain.DefaultBranchName } errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath) if errStr != "" { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form) return } commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName) spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobType(jobType), ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("Resource specification not available", tpl, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tpl, &form) return } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, DisplayJobName: displayJobName, JobName: jobName, Image: image, Command: command, Uuids: uuids, DatasetNames: datasetNames, DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), JobType: jobType, Description: form.Description, BranchName: branchName, BootFile: form.BootFile, Params: form.Params, CommitID: commitID, BenchmarkTypeID: 0, BenchmarkChildTypeID: 0, ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), Spec: spec, } err = cloudbrain.GenerateTask(req) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tpl, &form) return } if jobType == string(models.JobTypeTrain) { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all") } else { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") } } func loadCodeAndMakeModelPath(repo *models.Repository, codePath string, branchName string, jobName string, resultPath string) string { err := downloadCode(repo, codePath, branchName) if err != nil { return "cloudbrain.load_code_failed" } err = uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") if err != nil { return "cloudbrain.load_code_failed" } modelPath := setting.JobPath + jobName + resultPath + "/" err = mkModelPath(modelPath) if err != nil { return "cloudbrain.load_code_failed" } err = uploadCodeToMinio(modelPath, jobName, resultPath+"/") if err != nil { return "cloudbrain.load_code_failed" } return "" } func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBrainInferencForm) { ctx.Data["PageIsCloudBrain"] = true displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := strings.TrimSpace(form.Image) uuid := form.Attachment jobType := string(models.JobTypeInference) codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath branchName := form.BranchName bootFile := strings.TrimSpace(form.BootFile) labelName := form.LabelName repo := ctx.Repo.Repository ckptUrl := setting.Attachment.Minio.RealPath + form.TrainUrl + form.CkptName log.Info("ckpt url:" + ckptUrl) tpl := tplCloudBrainInferenceJobNew command, err := getInferenceJobCommand(form) if err != nil { log.Error("getTrainJobCommand failed: %v", err) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tpl, &form) return } tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) return } count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } } if branchName == "" { branchName = cloudbrain.DefaultBranchName } errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ResultPath) if errStr != "" { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form) return } commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName) datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeInference, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("Resource specification not available", tpl, &form) return } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, DisplayJobName: displayJobName, JobName: jobName, Image: image, Command: command, Uuids: uuid, DatasetNames: datasetNames, DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: setting.Attachment.Minio.RealPath + form.TrainUrl, BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), JobType: jobType, Description: form.Description, BranchName: branchName, BootFile: form.BootFile, Params: form.Params, CommitID: commitID, ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), ModelName: form.ModelName, ModelVersion: form.ModelVersion, CkptName: form.CkptName, TrainUrl: form.TrainUrl, LabelName: labelName, Spec: spec, } err = cloudbrain.GenerateTask(req) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tpl, &form) return } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/inference-job") } /** 检查用户传输的参数是否符合专属资源池 */ func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string { if cloudbrain.SpecialPools != nil { var isInPoolOrg = false var matchSpecialPool = false for _, specialPool := range cloudbrain.SpecialPools.Pools { if cloudbrain.IsElementExist(specialPool.JobType, jobType) && cloudbrain.IsQueueInSpecialtPool(specialPool.Pool, queue) { if cloudbrain.IsResourceSpecInSpecialPool(specialPool.ResourceSpec, resourceSpecId) { matchSpecialPool = true org, _ := models.GetOrgByName(specialPool.Org) if org != nil { isInPoolOrg, _ = models.IsOrganizationMember(org.ID, ctx.User.ID) if isInPoolOrg { break //传入参数，和专属资源池匹配上了，检查通过 } } } } } //资源池有匹配上，但是用户不在相应的组织中，返回错误信息。界面已经过滤了选择，界面操作不会到这个逻辑 if matchSpecialPool && !isInPoolOrg { return ctx.Tr("repo.grampus.no_operate_right") } } //没有匹配到资源池或者没有设置专属资源池，检查通过; 获取和资源池完全匹配检查通过 return "" } func CloudBrainRestart(ctx *context.Context) { var ID = ctx.Params(":id") var resultCode = "0" var errorMsg = "" var status = string(models.JobWaiting) task := ctx.Cloudbrain for { if task.Status != string(models.JobStopped) && task.Status != string(models.JobSucceeded) && task.Status != string(models.JobFailed) { log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" errorMsg = "the job is not stopped" break } if task.Image == "" || task.GpuQueue == "" || task.Type != models.TypeCloudBrainOne { log.Error("the job(%s) version is too old", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" errorMsg = "the job's version is too old and can not be restarted" break } if !ctx.IsSigned || (ctx.User.ID != task.UserID && !ctx.IsUserSiteAdmin()) { log.Error("the user has no right ro restart the job", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" errorMsg = "you have no right to restart the job" break } specOld, err := resource.GetCloudbrainSpec(task.ID) if err != nil || specOld == nil { log.Error("CloudBrainRestart GetCloudbrainSpec error.task.id = %d", task.ID) resultCode = "-1" errorMsg = "Resource specification not support any more" break } spec, err := resource.GetAndCheckSpec(ctx.User.ID, specOld.ID, models.FindSpecsOptions{ JobType: models.JobType(task.JobType), ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { log.Error("CloudBrainRestart GetAndCheckSpec error.task.id = %d", task.ID) resultCode = "-1" errorMsg = "Resource specification not support any more" break } task.Spec = spec if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) resultCode = "-1" errorMsg = ctx.Tr("points.insufficient_points_balance") break } count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) resultCode = "-1" errorMsg = "system error" break } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) resultCode = "-1" errorMsg = ctx.Tr("repo.cloudbrain.morethanonejob") break } } err = cloudbrain.RestartTask(ctx, task, &ID) if err != nil { log.Error("RestartTask failed:%v", err.Error(), ctx.Data["MsgID"]) resultCode = "-1" errorMsg = "system error" break } break } ctx.JSON(200, map[string]string{ "result_code": resultCode, "error_msg": errorMsg, "status": status, "id": ID, }) } func CloudBrainBenchMarkShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark) } func CloudBrainShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainShow, models.JobTypeDebug) } func CloudBrainTrainJobShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainTrainJobShow, models.JobTypeTrain) } func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.JobType) { ctx.Data["PageIsCloudBrain"] = true debugListType := ctx.Query("debugListType") cloudbrain.InitSpecialPool() var task *models.Cloudbrain var err error if jobType == models.JobTypeTrain || jobType == models.JobTypeInference { task, err = models.GetCloudbrainByJobID(ctx.Params(":jobid")) } else { task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id")) } if err != nil { log.Info("error:" + err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } result, err := cloudbrain.GetJob(task.JobID) if err != nil { log.Info("error:" + err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } prepareSpec4Show(ctx, task) if ctx.Written() { return } if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) taskRoles := jobRes.TaskRoles taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) ctx.Data["taskRes"] = taskRes ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State task.ContainerID = taskRes.TaskStatuses[0].ContainerID task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) if task.DeletedAt.IsZero() { //normal record if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) } err = models.UpdateJob(task) if err != nil { ctx.Data["error"] = err.Error() return } } else { //deleted record } ctx.Data["result"] = jobRes } else { log.Info("error:" + err.Error()) return } user, err := models.GetUserByID(task.UserID) if err == nil { task.User = user } if task.BenchmarkTypeID > 0 { for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { if task.BenchmarkTypeID == benchmarkType.Id { ctx.Data["BenchmarkTypeName"] = benchmarkType.First task.BenchmarkTypeName = benchmarkType.First for _, benchmarkChildType := range benchmarkType.Second { if task.BenchmarkChildTypeID == benchmarkChildType.Id { ctx.Data["BenchmarkChildTypeName"] = benchmarkChildType.Value break } } break } } } if task.JobType == string(models.JobTypeBenchmark) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkTypeName = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType if task.JobType == string(models.JobTypeBrainScore) { ctx.Data["BenchmarkChildTypeName"] = getBrainRegion(task.BenchmarkChildTypeID) } } if task.TrainJobDuration == "" { if task.Duration == 0 { var duration int64 if task.Status == string(models.JobWaiting) { duration = 0 } else if task.Status == string(models.JobRunning) { duration = time.Now().Unix() - int64(task.CreatedUnix) } else { duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) } task.Duration = duration } task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) } ctx.Data["duration"] = task.TrainJobDuration if len(task.Parameters) > 0 { var parameters models.Parameters err := json.Unmarshal([]byte(task.Parameters), ¶meters) if err != nil { log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err) task.Parameters = "" } else { if len(parameters.Parameter) > 0 { paramTemp := "" for _, Parameter := range parameters.Parameter { param := Parameter.Label + " = " + Parameter.Value + "; " paramTemp = paramTemp + param } task.Parameters = paramTemp[:len(paramTemp)-2] } else { task.Parameters = "" } } } ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, false) ctx.Data["task"] = task labelName := strings.Fields(task.LabelName) ctx.Data["LabelName"] = labelName ctx.Data["jobName"] = task.JobName ctx.Data["displayJobName"] = task.DisplayJobName version_list_task := make([]*models.Cloudbrain, 0) version_list_task = append(version_list_task, task) ctx.Data["version_list_task"] = version_list_task ctx.Data["debugListType"] = debugListType ctx.Data["code_path"] = cloudbrain.CodeMountPath ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath ctx.Data["model_path"] = cloudbrain.ModelMountPath ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) ctx.Data["branchName"] = task.BranchName ctx.HTML(200, tpName) } func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName ctx.Redirect(debugUrl) } func prepareSpec4Show(ctx *context.Context, task *models.Cloudbrain) { s, err := resource.GetCloudbrainSpec(task.ID) if err != nil { log.Info("error:" + err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } ctx.Data["Spec"] = s } func oldPrepareSpec4Show(ctx *context.Context, task *models.Cloudbrain) { hasSpec := false if task.JobType == string(models.JobTypeTrain) { if cloudbrain.TrainResourceSpecs == nil { json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) } for _, tmp := range cloudbrain.TrainResourceSpecs.ResourceSpec { if tmp.Id == task.ResourceSpecId { hasSpec = true ctx.Data["GpuNum"] = tmp.GpuNum ctx.Data["CpuNum"] = tmp.CpuNum ctx.Data["MemMiB"] = tmp.MemMiB ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB break } } } else if task.JobType == string(models.JobTypeInference) { if cloudbrain.InferenceResourceSpecs == nil { json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs) } for _, tmp := range cloudbrain.InferenceResourceSpecs.ResourceSpec { if tmp.Id == task.ResourceSpecId { hasSpec = true ctx.Data["GpuNum"] = tmp.GpuNum ctx.Data["CpuNum"] = tmp.CpuNum ctx.Data["MemMiB"] = tmp.MemMiB ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB break } } } else { if cloudbrain.ResourceSpecs == nil { json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) } for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec { if tmp.Id == task.ResourceSpecId { hasSpec = true ctx.Data["GpuNum"] = tmp.GpuNum ctx.Data["CpuNum"] = tmp.CpuNum ctx.Data["MemMiB"] = tmp.MemMiB ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB break } } } if !hasSpec && cloudbrain.SpecialPools != nil { for _, specialPool := range cloudbrain.SpecialPools.Pools { if specialPool.ResourceSpec != nil { for _, spec := range specialPool.ResourceSpec { if task.ResourceSpecId == spec.Id { ctx.Data["GpuNum"] = spec.GpuNum ctx.Data["CpuNum"] = spec.CpuNum ctx.Data["MemMiB"] = spec.MemMiB ctx.Data["ShareMemMiB"] = spec.ShareMemMiB break } } } } } } func CloudBrainCommitImageShow(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true ctx.Data["Type"] = ctx.Cloudbrain.Type ctx.HTML(200, tplCloudBrainImageSubmit) } func GetImage(ctx *context.Context) { var ID = ctx.Params(":id") id, _ := strconv.ParseInt(ID, 10, 64) image, err := models.GetImageByID(id) if err != nil { log.Error("GetImageByID failed:%v", err.Error()) ctx.JSON(http.StatusNotFound, nil) } ctx.JSON(http.StatusOK, image) } func CloudBrainImageEdit(ctx *context.Context) { ctx.Data["PageIsImageEdit"] = true ctx.Data["PageFrom"] = ctx.Params(":from") var ID = ctx.Params(":id") id, err := strconv.ParseInt(ID, 10, 64) if err != nil { log.Error("GetImageByID failed:%v", err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) } image, err := models.GetImageByID(id) if err != nil { log.Error("GetImageByID failed:%v", err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) } ctx.Data["Image"] = image ctx.HTML(http.StatusOK, tplCloudBrainImageEdit) } func CloudBrainImageEditPost(ctx *context.Context, form auth.EditImageCloudBrainForm) { if utf8.RuneCountInString(form.Description) > 255 { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255))) return } validTopics, errMessage := checkTopics(form.Topics) if errMessage != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage))) return } image, err := models.GetImageByID(form.ID) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist"))) } image.IsPrivate = form.IsPrivate image.Description = form.Description err = models.WithTx(func(ctx models.DBContext) error { if err := models.UpdateLocalImage(image); err != nil { return err } if err := models.SaveImageTopics(image.ID, validTopics...); err != nil { return err } return nil }) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist"))) } else { ctx.JSON(http.StatusOK, models.BaseOKMessage) } } func CloudBrainImageDelete(ctx *context.Context) { var ID = ctx.Params(":id") id, err := strconv.ParseInt(ID, 10, 64) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist"))) return } err = models.DeleteLocalImage(id) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_delete_fail"))) } else { ctx.JSON(http.StatusOK, models.BaseOKMessage) } } func CloudBrainCommitImageCheck(ctx *context.Context, form auth.CommitImageCloudBrainForm) { isExist, _ := models.IsImageExistByUser(form.Tag, ctx.User.ID) if isExist { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_overwrite"))) } else { ctx.JSON(http.StatusOK, models.BaseOKMessage) } } func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImageCloudBrainForm) { if !NamePattern.MatchString(form.Tag) { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) return } if utf8.RuneCountInString(form.Description) > 255 { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255))) return } validTopics, errMessage := checkTopics(form.Topics) if errMessage != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage))) return } err := cloudbrain.CommitAdminImage(models.CommitImageParams{ CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{ ImageDescription: form.Description, ImageTag: form.Tag, }, IsPrivate: form.IsPrivate, CloudBrainType: form.Type, Topics: validTopics, UID: ctx.User.ID, Type: models.GetRecommondType(form.IsRecommend), Place: form.Place, }, ctx.User) if err != nil { log.Error("CommitImagefailed") if models.IsErrImageTagExist(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist"))) } else if models.IsErrorImageCommitting(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing"))) } else { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail"))) } return } ctx.JSON(200, models.BaseOKMessage) } func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) { if !NamePattern.MatchString(form.Tag) { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) return } if utf8.RuneCountInString(form.Description) > 255 { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255))) return } validTopics, errMessage := checkTopics(form.Topics) if errMessage != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage))) return } err := cloudbrain.CommitImage(ctx.Cloudbrain.JobID, models.CommitImageParams{ CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{ Ip: ctx.Cloudbrain.ContainerIp, TaskContainerId: ctx.Cloudbrain.ContainerID, ImageDescription: form.Description, ImageTag: form.Tag, }, IsPrivate: form.IsPrivate, CloudBrainType: form.Type, Topics: validTopics, UID: ctx.User.ID, }, ctx.User) if err != nil { log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"]) if models.IsErrImageTagExist(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist"))) } else if models.IsErrorImageCommitting(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing"))) } else { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail"))) } return } ctx.JSON(200, models.BaseOKMessage) } func checkTopics(Topics string) ([]string, string) { var topics = make([]string, 0) var topicsStr = strings.TrimSpace(Topics) if len(topicsStr) > 0 { topics = strings.Split(topicsStr, ",") } validTopics, invalidTopics := models.SanitizeAndValidateImageTopics(topics) if len(validTopics) > 25 { return nil, "repo.topic.count_prompt" } if len(invalidTopics) > 0 { return nil, "repo.imagetopic.format_prompt" } return validTopics, "" } func CloudBrainStop(ctx *context.Context) { var ID = ctx.Params(":id") var resultCode = "0" var errorMsg = "" var status = "" task := ctx.Cloudbrain for { if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) { log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) resultCode = "-1" errorMsg = "cloudbrain.Already_stopped" resultCode = task.Status break } err := cloudbrain.StopJob(task.JobID) if err != nil { log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) resultCode = "-1" errorMsg = "cloudbrain.Stopped_failed" break } oldStatus := task.Status task.Status = string(models.JobStopped) if task.EndTime == 0 { task.EndTime = timeutil.TimeStampNow() } task.ComputeAndSetDuration() if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) resultCode = "-1" errorMsg = "cloudbrain.Stopped_success_update_status_fail" break } status = task.Status break } ctx.JSON(200, map[string]interface{}{ "result_code": resultCode, "error_msg": ctx.Tr(errorMsg), "status": status, "id": ID, "StatusOK": 0, }) } func StopJobsByUserID(userID int64) { cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID) if err != nil { log.Warn("Failed to get cloudBrain info", err) return } StopJobs(cloudBrains) } func StopJobsByRepoID(repoID int64) { cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID) if err != nil { log.Warn("Failed to get cloudBrain info", err) return } StopJobs(cloudBrains) } func DeleteJobsByRepoID(repoID int64) { cloudBrains, err := models.GetCloudbrainsNeededDeleteByRepoID(repoID) if err != nil { log.Warn("Failed to get cloudBrain info", err) return } DeleteJobs(cloudBrains) } /** */ func StopJobs(cloudBrains []*models.Cloudbrain) { for _, taskInfo := range cloudBrains { if taskInfo.Type == models.TypeCloudBrainOne { err := retry(3, time.Second*30, func() error { return cloudbrain.StopJob(taskInfo.JobID) }) logErrorAndUpdateJobStatus(err, taskInfo) } else { if taskInfo.JobType == string(models.JobTypeTrain) { err := retry(3, time.Second*30, func() error { _, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10)) return err }) logErrorAndUpdateJobStatus(err, taskInfo) } else { param := models.NotebookAction{ Action: models.ActionStop, } err := retry(3, time.Second*30, func() error { _, err := modelarts.ManageNotebook(taskInfo.JobID, param) return err }) logErrorAndUpdateJobStatus(err, taskInfo) } } } } func DeleteJobs(cloudBrains []*models.Cloudbrain) { for _, taskInfo := range cloudBrains { err := models.DeleteJob(taskInfo) if err != nil { log.Warn("Failed to DeleteJob:", err) return } if taskInfo.Type == models.TypeCloudBrainOne { cloudbrain.DelCloudBrainJob(taskInfo.JobName) DeleteCloudbrainJobStorage(taskInfo.JobName, models.TypeCloudBrainOne) } if taskInfo.Type == models.TypeCloudBrainTwo { if taskInfo.JobType == string(models.JobTypeTrain) || taskInfo.JobType == string(models.JobTypeInference) { modelarts.DelTrainJob(taskInfo.JobID) DeleteJobStorage(taskInfo.JobName) } if taskInfo.JobType == string(models.JobTypeDebug) { modelarts.DelNotebook2(taskInfo.JobID) } } if taskInfo.Type == models.TypeC2Net { if taskInfo.JobType == string(models.JobTypeTrain) { cloudbrain.DelCloudBrainJob(taskInfo.JobName) DeleteCloudbrainJobStorage(taskInfo.JobName, models.TypeCloudBrainOne) } } } } func retry(attempts int, sleep time.Duration, f func() error) (err error) { for i := 0; i < attempts; i++ { if i > 0 { log.Warn("retrying after error:", err) time.Sleep(sleep) } err = f() if err == nil { return nil } } return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { if err != nil { log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err) } else { oldStatus := taskInfo.Status taskInfo.Status = string(models.JobStopped) if taskInfo.EndTime == 0 { taskInfo.EndTime = timeutil.TimeStampNow() } taskInfo.ComputeAndSetDuration() if oldStatus != taskInfo.Status { notification.NotifyChangeCloudbrainStatus(taskInfo, oldStatus) } err = models.UpdateJob(taskInfo) if err != nil { log.Warn("UpdateJob failed", err) } } } func CloudBrainDel(ctx *context.Context) { var listType = ctx.Query("debugListType") if err := deleteCloudbrainJob(ctx); err != nil { log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"]) ctx.ServerError(err.Error(), err) return } var isAdminPage = ctx.Query("isadminpage") var isHomePage = ctx.Query("ishomepage") if ctx.IsUserSiteAdmin() && isAdminPage == "true" { ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains") } else if isHomePage == "true" { ctx.Redirect(setting.AppSubURL + "/cloudbrains") } else { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=" + listType) } } func deleteCloudbrainJob(ctx *context.Context) error { task := ctx.Cloudbrain if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) { log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) return errors.New("the job has not been stopped") } err := models.DeleteJob(task) if err != nil { log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"]) return err } DeleteCloudbrainJobStorage(task.JobName, models.TypeCloudBrainOne) return nil } func CloudBrainShowModels(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true ID := ctx.Params(":id") parentDir := ctx.Query("parentDir") dirArray := strings.Split(parentDir, "/") task, err := models.GetCloudbrainByID(ID) if err != nil { log.Error("no such job!", ctx.Data["msgID"]) ctx.ServerError("no such job:", err) return } //get dirs dirs, err := GetModelDirs(task.JobName, parentDir) if err != nil { log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("GetModelDirs failed:", err) return } var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("json.Unmarshal failed:", err) return } for i, fileInfo := range fileInfos { temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime) fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05") } sort.Slice(fileInfos, func(i, j int) bool { return fileInfos[i].ModTime > fileInfos[j].ModTime }) ctx.Data["Path"] = dirArray ctx.Data["Dirs"] = fileInfos ctx.Data["task"] = task ctx.Data["ID"] = ID ctx.HTML(200, tplCloudBrainShowModels) } func GetPublicImages(ctx *context.Context) { uid := getUID(ctx) opts := models.SearchImageOptions{ IncludePublicOnly: true, UID: uid, Keyword: ctx.Query("q"), Topics: ctx.Query("topic"), IncludeOfficialOnly: ctx.QueryBool("recommend"), SearchOrderBy: "type desc, num_stars desc,id desc", Status: models.IMAGE_STATUS_SUCCESS, CloudbrainType: ctx.QueryInt("cloudbrainType"), } getImages(ctx, &opts) } func GetCustomImages(ctx *context.Context) { uid := getUID(ctx) opts := models.SearchImageOptions{ UID: uid, IncludeOwnerOnly: true, Keyword: ctx.Query("q"), Topics: ctx.Query("topic"), Status: -1, SearchOrderBy: "id desc", } getImages(ctx, &opts) } func GetStarImages(ctx *context.Context) { uid := getUID(ctx) opts := models.SearchImageOptions{ UID: uid, IncludeStarByMe: true, Keyword: ctx.Query("q"), Topics: ctx.Query("topic"), Status: models.IMAGE_STATUS_SUCCESS, SearchOrderBy: "id desc", } getImages(ctx, &opts) } func getUID(ctx *context.Context) int64 { var uid int64 = -1 if ctx.IsSigned { uid = ctx.User.ID } return uid } func GetAllImages(ctx *context.Context) { uid := getUID(ctx) opts := models.SearchImageOptions{ UID: uid, Keyword: ctx.Query("q"), Topics: ctx.Query("topic"), IncludeOfficialOnly: ctx.QueryBool("recommend"), SearchOrderBy: "id desc", Status: -1, } if ctx.Query("private") != "" { if ctx.QueryBool("private") { opts.IncludePrivateOnly = true } else { opts.IncludePublicOnly = true } } getImages(ctx, &opts) } func getImages(ctx *context.Context, opts *models.SearchImageOptions) { page := ctx.QueryInt("page") if page <= 0 { page = 1 } pageSize := ctx.QueryInt("pageSize") if pageSize <= 0 { pageSize = 15 } opts.ListOptions = models.ListOptions{ Page: page, PageSize: pageSize, } imageList, total, err := models.SearchImage(opts) if err != nil { log.Error("Can not get images:%v", err) ctx.JSON(http.StatusOK, models.ImagesPageResult{ Count: 0, Images: []*models.Image{}, }) } else { ctx.JSON(http.StatusOK, models.ImagesPageResult{ Count: total, Images: imageList, }) } } func GetModelDirs(jobName string, parentDir string) (string, error) { var req string modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/") if parentDir == "" { req = "baseDir=" + modelActualPath } else { req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir } return getDirs(req) } func GetResultDirs(jobName string, parentDir string) (string, error) { var req string modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/") if parentDir == "" { req = "baseDir=" + modelActualPath } else { req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir } return getDirs(req) } func CloudBrainDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") jobName := ctx.Query("jobName") filePath := "jobs/" + jobName + "/model/" + parentDir url, err := storage.Attachments.PresignedGetURL(filePath, fileName) if err != nil { log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("PresignedGetURL", err) return } ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } func CloudBrainDownloadInferenceResult(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") jobName := ctx.Query("jobName") filePath := "jobs/" + jobName + "/result/" + parentDir url, err := storage.Attachments.PresignedGetURL(filePath, fileName) if err != nil { log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("PresignedGetURL", err) return } ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } func GetRate(ctx *context.Context) { isObjectDetcionAll := ctx.QueryBool("isObjectDetcionAll") if isObjectDetcionAll { ctx.Redirect(setting.BenchmarkServerHost + "?username=admin") return } var ID = ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) if err != nil { ctx.ServerError("GetCloudbrainByJobID failed", err) return } if job.JobType == string(models.JobTypeBenchmark) { log.Info("url=" + setting.BenchmarkServerHost + "?username=" + ctx.User.Name) ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name) } else if job.JobType == string(models.JobTypeSnn4imagenet) { ctx.Redirect(setting.Snn4imagenetServerHost) } else if job.JobType == string(models.JobTypeBrainScore) { ctx.Redirect(setting.BrainScoreServerHost) } else { log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) } } func downloadCode(repo *models.Repository, codePath, branchName string) error { //add "file:///" prefix to make the depth valid if err := git.Clone(CLONE_FILE_PREFIX+repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil { log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) return err } configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666) if err != nil { log.Error("open file(%s) failed:%v", codePath+"/,git/config", err) return err } defer configFile.Close() pos := int64(0) reader := bufio.NewReader(configFile) for { line, err := reader.ReadString('\n') if err != nil { if err == io.EOF { log.Error("not find the remote-url") return nil } else { log.Error("read error: %v", err) return err } } if strings.Contains(line, "url") && strings.Contains(line, ".git") { originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n" if len(line) > len(originUrl) { originUrl += strings.Repeat(" ", len(line)-len(originUrl)) } bytes := []byte(originUrl) _, err := configFile.WriteAt(bytes, pos) if err != nil { log.Error("WriteAt failed:%v", err) return err } break } pos += int64(len(line)) } return nil } func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType, userName string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) return err } repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName) if err != nil { log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error()) return err } if err := git.Clone(CLONE_FILE_PREFIX+repoExt.RepoPath(), codePath, git.CloneRepoOptions{Depth: 1}); err != nil { log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } fileName := codePath + cloudbrain.TaskInfoName f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) if err != nil { log.Error("OpenFile failed", err.Error()) return err } defer f.Close() data, err := json.Marshal(models.TaskInfo{ Username: userName, TaskName: taskName, CodeName: repo.Name, BenchmarkCategory: strings.Split(benchmarkCategory, ","), CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"), GpuType: gpuType, }) if err != nil { log.Error("json.Marshal failed", err.Error()) return err } _, err = f.Write(data) if err != nil { log.Error("WriteString failed", err.Error()) return err } return nil } func uploadCodeToMinio(codePath, jobName, parentDir string) error { files, err := readDir(codePath) if err != nil { log.Error("readDir(%s) failed: %s", codePath, err.Error()) return err } for _, file := range files { if file.IsDir() { if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil { log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error()) return err } } else { destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name() sourceFile := codePath + file.Name() err = storage.Attachments.UploadObject(destObject, sourceFile) if err != nil { log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error()) return err } } } return nil } func mkModelPath(modelPath string) error { return mkPathAndReadMeFile(modelPath, "You can put the files into this directory and download the files by the web page.") } func mkPathAndReadMeFile(path string, text string) error { err := os.MkdirAll(path, os.ModePerm) if err != nil { log.Error("MkdirAll(%s) failed:%v", path, err) return err } fileName := path + "README" f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) if err != nil { log.Error("OpenFile failed", err.Error()) return err } defer f.Close() _, err = f.WriteString(text) if err != nil { log.Error("WriteString failed", err.Error()) return err } return nil } func DeleteCloudbrainJobStorage(jobName string, cloudbrainType int) error { //delete local localJobPath := setting.JobPath + jobName err := os.RemoveAll(localJobPath) if err != nil { log.Error("RemoveAll(%s) failed:%v", localJobPath, err) } //delete oss if cloudbrainType == models.TypeCloudBrainOne { dirPath := setting.CBCodePathPrefix + jobName + "/" err = storage.Attachments.DeleteDir(dirPath) if err != nil { log.Error("DeleteDir(%s) failed:%v", localJobPath, err) } } else if cloudbrainType == models.TypeCloudBrainTwo { //dirPath := setting.CodePathPrefix + jobName + "/" //err = storage.ObsRemoveObject(setting.Bucket, dirPath) //if err != nil { // log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err) //} log.Info("no need to delete") } else { log.Error("cloudbrainType(%d) error", cloudbrainType) } return nil } func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil { log.Error("GetCloudBrainUnStoppedJob failed:", err.Error()) return } for _, task := range cloudBrains { if task.Type == models.TypeCloudBrainOne { result, err := cloudbrain.GetJob(task.JobID) if err != nil { log.Error("GetJob(%s) failed:%v", task.JobName, err) continue } if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) taskRoles := jobRes.TaskRoles taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State if task.Status != string(models.JobWaiting) { models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) } var maxDuration int64 if task.JobType == string(models.JobTypeBenchmark) { maxDuration = setting.BenchmarkMaxDuration } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { maxDuration = setting.ModelBenchmarkMaxDuration } else { maxDuration = setting.MaxDuration } if task.Duration >= maxDuration && task.JobType != string(models.JobTypeTrain) { log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName) err = cloudbrain.StopJob(task.JobID) if err != nil { log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err) continue } task.Status = string(models.JobStopped) if task.EndTime == 0 { task.EndTime = timeutil.TimeStampNow() } task.ComputeAndSetDuration() if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) continue } } } } } else if task.Type == models.TypeCloudBrainTwo { if task.JobType == string(models.JobTypeDebug) { err := modelarts.HandleNotebookInfo(task) if err != nil { log.Error("HandleNotebookInfo(%s) failed:%v", task.DisplayJobName, err) continue } } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) { err := modelarts.HandleTrainJobInfo(task) if err != nil { log.Error("HandleTrainJobInfo(%s) failed:%v", task.DisplayJobName, err) continue } } else { log.Error("task.JobType(%s) is error:%s", task.DisplayJobName, task.JobType) } } else if task.Type == models.TypeC2Net { result, err := grampus.GetJob(task.JobID) if err != nil { log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err) continue } if result != nil { if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] } oldStatus := task.Status task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) task.Duration = result.JobInfo.RunSec task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) if task.StartTime == 0 && result.JobInfo.StartedAt > 0 { task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt) } if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { task.EndTime = task.StartTime.Add(task.Duration) } task.CorrectCreateUnix() if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) continue } } } else { log.Error("task.Type(%s) is error:%d", task.JobName, task.Type) } } return } func HandleTaskWithNoDuration(ctx *context.Context) { mode := ctx.Query("mode") log.Info("HandleTaskWithNoDuration start") count := 0 start := time.Now().Unix() for { var cloudBrains []*models.Cloudbrain var err error if mode == "1" { cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime() } else { cloudBrains, err = models.GetStoppedJobWithNoDurationJob() } if err != nil { log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error()) break } if len(cloudBrains) == 0 { log.Info("HandleTaskWithNoTrainJobDuration:no task need handle") break } handleNoDurationTask(cloudBrains) count += len(cloudBrains) if len(cloudBrains) < 100 { log.Info("HandleTaskWithNoTrainJobDuration:task less than 100") break } if time.Now().Unix()-start > 600 { log.Info("HandleTaskWithNoDuration : time out") ctx.JSON(200, fmt.Sprintf("task stop for time out,count=%d", count)) return } } log.Info("HandleTaskWithNoTrainJobDuration:count=%d", count) ctx.JSON(200, fmt.Sprintf("success,count=%d", count)) } func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { for _, task := range cloudBrains { time.Sleep(time.Millisecond * 100) log.Info("Handle job ,%+v", task) if task.Type == models.TypeCloudBrainOne { result, err := cloudbrain.GetJob(task.JobID) if err != nil { log.Error("GetJob(%s) failed:%v", task.JobName, err) updateDefaultDuration(task) continue } if result != nil { if result.Msg != "success" { updateDefaultDuration(task) continue } jobRes, err := models.ConvertToJobResultPayload(result.Payload) if err != nil || len(jobRes.TaskRoles) == 0 { updateDefaultDuration(task) continue } taskRoles := jobRes.TaskRoles taskRes, err := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) if err != nil || len(taskRes.TaskStatuses) == 0 { updateDefaultDuration(task) continue } task.Status = taskRes.TaskStatuses[0].State log.Info("task startTime = %v endTime= %v ,jobId=%d", jobRes.JobStatus.StartTime, jobRes.JobStatus.EndTime, task.ID) if jobRes.JobStatus.CreatedTime > 0 { task.StartTime = timeutil.TimeStamp(jobRes.JobStatus.CreatedTime / 1000) if jobRes.JobStatus.CompletedTime > 0 { task.EndTime = timeutil.TimeStamp(jobRes.JobStatus.CompletedTime / 1000) } else { task.EndTime = task.UpdatedUnix } } else { task.StartTime = 0 task.EndTime = 0 } if task.EndTime < task.StartTime { log.Info("endTime[%v] is less than starTime[%v],jobId=%d", task.EndTime, task.StartTime, task.ID) st := task.StartTime task.StartTime = task.EndTime task.EndTime = st } task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) err = models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) } } } else if task.Type == models.TypeCloudBrainTwo { if task.JobType == string(models.JobTypeDebug) { //result, err := modelarts.GetJob(task.JobID) result, err := modelarts.GetNotebook2(task.JobID) if err != nil { log.Error("GetJob(%s) failed:%v", task.JobName, err) updateDefaultDuration(task) continue } if result != nil { task.Status = result.Status startTime := result.Lease.CreateTime duration := result.Lease.Duration / 1000 if startTime > 0 { task.StartTime = timeutil.TimeStamp(startTime / 1000) task.EndTime = task.StartTime.Add(duration) } task.CorrectCreateUnix() task.ComputeAndSetDuration() err = models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) continue } } } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) { result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) if err != nil { log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) updateDefaultDuration(task) continue } if result != nil { startTime := result.StartTime / 1000 if startTime > 0 { task.StartTime = timeutil.TimeStamp(startTime) task.EndTime = task.StartTime.Add(result.Duration / 1000) } task.ComputeAndSetDuration() err = models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) continue } } } else { log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) } } else { log.Error("task.Type(%s) is error:%d", task.JobName, task.Type) } } } func updateDefaultDuration(task *models.Cloudbrain) { log.Info("updateDefaultDuration: taskId=%d", task.ID) task.StartTime = task.CreatedUnix task.EndTime = task.UpdatedUnix task.ComputeAndSetDuration() err := models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) } } func CloudBrainBenchmarkIndex(ctx *context.Context) { MustEnableCloudbrain(ctx) repo := ctx.Repo.Repository page := ctx.QueryInt("page") if page <= 0 { page = 1 } var jobTypes []string jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, Type: models.TypeCloudBrainOne, JobTypes: jobTypes, }) if err != nil { ctx.ServerError("Get debugjob faild:", err) return } for i, task := range ciTasks { ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource if ciTasks[i].TrainJobDuration == "" { if ciTasks[i].Duration == 0 { var duration int64 if task.Status == string(models.JobRunning) { duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) } else { duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) } ciTasks[i].Duration = duration } ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration) } ciTasks[i].BenchmarkTypeName = "" if ciTasks[i].JobType == string(models.JobTypeBenchmark) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost } else { ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost } } if task.BenchmarkTypeID > 0 { for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { if task.BenchmarkTypeID == benchmarkType.Id { ciTasks[i].BenchmarkTypeRankLink = benchmarkType.RankLink ciTasks[i].BenchmarkTypeName = benchmarkType.First break } } } } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = ciTasks ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) ctx.Data["RepoIsEmpty"] = repo.IsEmpty ctx.HTML(200, tplCloudBrainBenchmarkIndex) } func GetChildTypes(ctx *context.Context) { benchmarkTypeID := ctx.QueryInt("benchmark_type_id") re := make(map[string]interface{}) for { var isExist bool for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { if benchmarkTypeID == benchmarkType.Id { isExist = true re["child_types"] = benchmarkType.Second re["result_code"] = "0" break } } if !isExist { re["result_code"] = "1" log.Error("no such benchmark_type_id", ctx.Data["MsgID"]) re["errMsg"] = "system error" break } break } ctx.JSON(200, re) } func CloudBrainBenchmarkNew(ctx *context.Context) { ctx.Data["description"] = "" ctx.Data["benchmarkTypeID"] = -1 ctx.Data["benchmark_child_types_id_hidden"] = -1 err := cloudBrainNewDataPrepare(ctx) if err != nil { ctx.ServerError("get new cloudbrain info failed", err) return } ctx.HTML(200, tplCloudBrainBenchmarkNew) } func getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID int, ctx *context.Context) (*models.BenchmarkDataset, error) { var childInfo *models.BenchmarkDataset var isExist bool for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { if benchmarkType.Id == benchmarkTypeID { for _, childType := range benchmarkType.Second { if childType.Id == benchmarkChildTypeID { childInfo = childType isExist = true break } } break } } if !isExist { log.Error("no such benchmark_type_id&benchmark_child_type_id") return childInfo, errors.New("no such benchmark_type_id&benchmark_child_type_id") } return childInfo, nil } func getBenchmarkGpuQueue(gpuQueue string) (string, error) { queue := "" if benchmarkGpuInfos == nil { if err := json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos); err != nil { log.Error("json.Unmarshal BenchmarkGpuTypes(%s) failed:%v", setting.BenchmarkGpuTypes, err) return queue, err } } var isExist bool for _, gpuInfo := range benchmarkGpuInfos.GpuInfo { if gpuQueue == gpuInfo.Queue { isExist = true queue = gpuQueue break } } if !isExist { log.Error("no such gpuQueue, %s", gpuQueue) return queue, errors.New("no such gpuQueue") } return queue, nil } func getBenchmarkResourceSpec(resourceSpecID int) (int, error) { var id int if benchmarkResourceSpecs == nil { if err := json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs); err != nil { log.Error("json.Unmarshal BenchmarkResourceSpecs(%s) failed:%v", setting.BenchmarkResourceSpecs, err) return id, err } } var isExist bool for _, resourceSpec := range benchmarkResourceSpecs.ResourceSpec { if resourceSpecID == resourceSpec.Id { isExist = true id = resourceSpecID break } } if !isExist { log.Error("no such resourceSpecID, %d", resourceSpecID) return id, errors.New("no such resourceSpec") } return id, nil } func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { jobType := form.JobType if jobType == string(models.JobTypeBenchmark) { BenchMarkAlgorithmCreate(ctx, form) } else { ModelBenchmarkCreate(ctx, form) } } func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.Data["PageIsCloudBrain"] = true displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := strings.TrimSpace(form.Image) command := cloudbrain.CommandBenchmark codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath benchmarkTypeID := form.BenchmarkTypeID benchmarkChildTypeID := form.BenchmarkChildTypeID ctx.Data["description"] = form.Description ctx.Data["benchmarkTypeID"] = benchmarkTypeID ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID repo := ctx.Repo.Repository tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } } if !jobNamePattern.MatchString(jobName) { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form) return } childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID, ctx) if err != nil { log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form) return } spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeBenchmark, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("Resource specification not available", tplCloudBrainBenchmarkNew, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplCloudBrainBenchmarkNew, &form) return } count, err := models.GetBenchmarkCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainBenchmarkNew, &form) return } } os.RemoveAll(codePath) if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } if _, err := os.Stat(codePath + "/train.py"); err != nil { if os.IsNotExist(err) { // file does not exist log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form) } else { log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) } return } else if _, err := os.Stat(codePath + "/test.py"); err != nil { if os.IsNotExist(err) { // file does not exist log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form) } else { log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) } return } if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if err := downloadRateCode(repo, jobName, childInfo.Owner, childInfo.RepoName, benchmarkPath, form.BenchmarkCategory, spec.AccCardType, ctx.User.Name); err != nil { log.Error("downloadRateCode failed, %v", err, ctx.Data["MsgID"]) //cloudBrainNewDataPrepare(ctx) //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) //return } if err := uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/"); err != nil { log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) //cloudBrainNewDataPrepare(ctx) //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) //return } uuid := childInfo.Attachment datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, &form) return } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, DisplayJobName: displayJobName, JobName: jobName, Image: image, Command: command, Uuids: uuid, DatasetNames: datasetNames, DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), JobType: string(models.JobTypeBenchmark), Description: form.Description, BranchName: cloudbrain.DefaultBranchName, BootFile: "", Params: "", CommitID: "", BenchmarkTypeID: benchmarkTypeID, BenchmarkChildTypeID: benchmarkChildTypeID, ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), Spec: spec, } err = cloudbrain.GenerateTask(req) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form) return } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.Data["PageIsCloudBrain"] = true displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := form.Image uuid := form.Attachment jobType := form.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath branchName := cloudbrain.DefaultBranchName repo := ctx.Repo.Repository tpl := tplCloudBrainBenchmarkNew command := cloudbrain.GetCloudbrainDebugCommand() tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tpl, &form) return } count, err := models.GetBenchmarkCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } } downloadCode(repo, codePath, branchName) uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" mkModelPath(modelPath) uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name) uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/") command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description)) } benchmarkChildTypeID := 0 brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name) uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/") benchmarkChildTypeID = form.BenchmarkChildTypeID command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) } datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeBenchmark, ComputeResource: models.GPU, Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("Resource specification not available", tpl, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tpl, &form) return } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, DisplayJobName: displayJobName, JobName: jobName, Image: image, Command: command, Uuids: uuid, DatasetNames: datasetNames, DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), JobType: jobType, Description: form.Description, BranchName: branchName, BootFile: form.BootFile, Params: form.Params, CommitID: "", BenchmarkTypeID: 0, BenchmarkChildTypeID: benchmarkChildTypeID, ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), Spec: spec, } err = cloudbrain.GenerateTask(req) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tpl, &form) return } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } func getBrainRegion(benchmarkChildTypeID int) string { values := []string{"V1", "V2", "V4", "IT"} return values[benchmarkChildTypeID] } func trimSpaceNewlineInString(s string) string { re := regexp.MustCompile(`\r?\n`) return re.ReplaceAllString(s, " ") } func BenchmarkDel(ctx *context.Context) { if err := deleteCloudbrainJob(ctx); err != nil { log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"]) ctx.ServerError(err.Error(), err) return } var isAdminPage = ctx.Query("isadminpage") var isHomePage = ctx.Query("ishomepage") if ctx.IsUserSiteAdmin() && isAdminPage == "true" { ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains") } else if isHomePage == "true" { ctx.Redirect(setting.AppSubURL + "/cloudbrains") } else { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } } func CloudBrainTrainJobNew(ctx *context.Context) { err := cloudBrainNewDataPrepare(ctx) if err != nil { ctx.ServerError("get new train-job info failed", err) return } ctx.HTML(http.StatusOK, tplCloudBrainTrainJobNew) } func InferenceCloudBrainJobNew(ctx *context.Context) { err := cloudBrainNewDataPrepare(ctx) if err != nil { ctx.ServerError("get new train-job info failed", err) return } ctx.HTML(http.StatusOK, tplCloudBrainInferenceJobNew) } func InferenceCloudBrainJobShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference) } func DownloadInferenceResultFile(ctx *context.Context) { var jobID = ctx.Params(":jobid") var versionName = ctx.Query("version_name") task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) if err != nil { log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) return } allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl) returnFileName := task.DisplayJobName + ".zip" MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile) } func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) { var command string bootFile := strings.TrimSpace(form.BootFile) params := form.Params if !strings.HasSuffix(bootFile, ".py") { log.Error("bootFile(%s) format error", bootFile) return command, errors.New("bootFile format error") } var parameters models.Parameters var param string if len(params) != 0 { err := json.Unmarshal([]byte(params), ¶meters) if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", params, err) return command, err } for _, parameter := range parameters.Parameter { param += " --" + parameter.Label + "=" + parameter.Value } } param += " --modelname" + "=" + form.CkptName command += "python /code/" + bootFile + param + " > " + cloudbrain.ResultPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile return command, nil } func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) { var command string bootFile := strings.TrimSpace(form.BootFile) params := form.Params if !strings.HasSuffix(bootFile, ".py") { log.Error("bootFile(%s) format error", bootFile) return command, errors.New("bootFile format error") } var parameters models.Parameters var param string if len(params) != 0 { err := json.Unmarshal([]byte(params), ¶meters) if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", params, err) return command, err } for _, parameter := range parameters.Parameter { param += " --" + parameter.Label + "=" + parameter.Value } } command += "python /code/" + bootFile + param + " | tee " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile return command, nil } func CloudBrainTrainJobDel(ctx *context.Context) { var listType = ctx.Query("listType") if err := deleteCloudbrainJob(ctx); err != nil { log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"]) ctx.ServerError(err.Error(), err) return } var isAdminPage = ctx.Query("isadminpage") var isHomePage = ctx.Query("ishomepage") if ctx.IsUserSiteAdmin() && isAdminPage == "true" { ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains") } else if isHomePage == "true" { ctx.Redirect(setting.AppSubURL + "/cloudbrains") } else { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType) } } func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { var lang = ctx.Locale.Language() if benchmarkTypesMap[lang] == nil { var val = i18n.Tr(lang, BENCHMARK_TYPE_CODE) //use config val = setting.BenchmarkTypes var tempType *models.BenchmarkTypes if err := json.Unmarshal([]byte(val), &tempType); err != nil { log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", val, err, ctx.Data["MsgID"]) return &models.BenchmarkTypes{} } benchmarkTypesMap[lang] = tempType } return benchmarkTypesMap[lang] } func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { if task.Type == models.TypeCloudBrainOne { return ctx.Tr("repo.cloudbrain1") } else if task.Type == models.TypeCloudBrainTwo { return ctx.Tr("repo.cloudbrain2") } else if task.Type == models.TypeC2Net { return getCutStringAiCenterByAiCenter(task.AiCenter) } return "" } func getCutStringAiCenterByAiCenter(aiCenter string) string { if aiCenter == "" { return "" } index := strings.LastIndex(aiCenter, "+") return aiCenter[index+1:] } func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { if task.Type == models.TypeCloudBrainOne || task.Type == models.TypeCloudBrainTwo { return ctx.Tr("cloudbrain.resource_cluster_openi") } else if task.Type == models.TypeC2Net { return ctx.Tr("cloudbrain.resource_cluster_c2net") } return "" } func GetCloudbrainCardDuration(task models.Cloudbrain) string { cardNum, _, _ := GetCloudbrainCardNumAndType(task) cardDuration := models.ConvertDurationToStr(int64(cardNum) * task.Duration) return cardDuration } func GetCloudbrainWaitTime(task models.Cloudbrain) string { var waitTime string if task.Status == string(models.JobWaiting) { waitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix() waitTime = models.ConvertDurationToStr(waitTimeInt) if waitTimeInt < 0 { waitTime = "00:00:00" } } else if task.Status == string(models.JobStopped) && task.StartTime.AsTime().Unix() == 0 { waitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() waitTime = models.ConvertDurationToStr(waitTimeInt) if waitTimeInt < 0 { waitTime = "00:00:00" } } else { waitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() waitTime = models.ConvertDurationToStr(waitTimeInt) if waitTimeInt < 0 { waitTime = "00:00:00" } } return waitTime } func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) { if !models.SpecsMapInitFlag { models.InitCloudbrainOneResourceSpecMap() } if !models.GpuInfosMapInitFlag { models.InitCloudbrainOneGpuInfoMap() } flavorName, err := GetCloudbrainFlavorName(task) if err != nil { return 0, "", nil } return getCardNumAndTypeByFlavorname(flavorName) } func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { if FlavorName == "" { return 0, "", nil } else { var beginIndex = strings.Index(FlavorName, ":") var lastIndex = strings.LastIndex(FlavorName, ":") var endIndex = strings.Index(FlavorName, "*") if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) { cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) if err != nil { log.Error("strconv.Atoi failed: %v", err) return 0, "", err } cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) return cardNum, cardType, err } return 0, "", nil } } func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { if task.Type == models.TypeCloudBrainOne { resourceSpec, gpuInfo, err := getCloudBrainOneResourceSpec(task) if err != nil { log.Info("getCloudBrainOneResourceSpec err:", err) return "", err } else { if resourceSpec == nil || gpuInfo == nil { err := errors.New("resourceSpec or gpuInfo is nil") return "", err } else { CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(resourceSpec.GpuNum) + "*Nvidia-" + gpuInfo.Value + " | CPU:" + strconv.Itoa(resourceSpec.CpuNum) + "核" + strconv.Itoa(resourceSpec.MemMiB) + "MB" return CloudbrainOneFlavorName, nil } } } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net) && task.FlavorName != "" { replaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") return replaceFlavorName, nil } else if task.Type == models.TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { cloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode) return cloudbrainTwoFlavorName, nil } else if task.Type == models.TypeCloudBrainTwo && task.JobType == string(models.JobTypeDebug) && task.FlavorName == "" && task.FlavorCode == "" { tasks, err := models.GetModelartsReDebugTaskByJobId(task.JobID) if err != nil { return "", err } if len(tasks) >= 1 { return getFlavorNameByFlavorCode(tasks[0].FlavorCode), nil } return "", nil } return "", nil } func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, *models.GpuInfo, error) { gpuQueueDefault := "openidebug" if task.GpuQueue != "" { gpuQueueDefault = task.GpuQueue } if task.ResourceSpecId >= 0 { if task.JobType == string(models.JobTypeTrain) { if models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId] != nil { return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[gpuQueueDefault], nil } else { return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil } } else if task.JobType == string(models.JobTypeDebug) { if models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId] != nil { return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[gpuQueueDefault], nil } else { return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil } } else if task.JobType == string(models.JobTypeInference) { return models.CloudbrainInferenceResourceSpecsMap[task.ResourceSpecId], models.CloudbrainInferenceGpuInfosMap[gpuQueueDefault], nil } else if task.JobType == string(models.JobTypeBenchmark) || task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { return models.CloudbrainBenchmarkResourceSpecsMap[task.ResourceSpecId], models.CloudbrainBenchmarkGpuInfosMap[gpuQueueDefault], nil } } else { err := errors.New("ResourceSpecId is null") return nil, nil, err } return nil, nil, nil } func getFlavorNameByFlavorCode(flavorCode string) string { index := strings.LastIndex(flavorCode, ".") cardNum, err := strconv.Atoi(strings.TrimSpace(flavorCode[index+1 : len(flavorCode)])) if err != nil { log.Error("strconv.Atoi failed: %v", err) return "" } cloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) + "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + "核" + strconv.Itoa(cardNum*256) + "GB" return cloudbrainTwoFlavorName }