From f912a130ee012d830375ba750c317093dbf68e07 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Fri, 18 Feb 2022 11:57:39 +0800 Subject: [PATCH 1/3] notebook opt --- models/cloudbrain.go | 10 ++++++ modules/auth/modelarts.go | 3 +- modules/modelarts/modelarts.go | 53 +++++++++++++++++++++++------ modules/setting/setting.go | 5 ++- routers/repo/modelarts.go | 34 +++++++----------- templates/repo/modelarts/notebook/new.tmpl | 13 +++++-- templates/repo/modelarts/notebook/show.tmpl | 13 ++++++- 7 files changed, 95 insertions(+), 36 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 0a14ea7b4..4c8763c80 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -449,6 +449,16 @@ type FlavorInfo struct { Desc string `json:"desc"` } +type ImageInfosModelArts struct { + ImageInfo []*ImageInfoModelArts `json:"image_info"` +} + +type ImageInfoModelArts struct { + Id string `json:"id"` + Value string `json:"value"` + Desc string `json:"desc"` +} + type PoolInfos struct { PoolInfo []*PoolInfo `json:"pool_info"` } diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index 821cd72f8..2d30de6ed 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -19,7 +19,8 @@ type CreateModelArtsNotebookForm struct { JobName string `form:"job_name" binding:"Required"` Attachment string `form:"attachment"` Description string `form:"description"` - Flavor string `form:"flavor"` + Flavor string `form:"flavor" binding:"Required"` + ImageId string `form:"image_id" binding:"Required"` } func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 301c4cb0e..62b99abb5 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -2,6 +2,7 @@ package modelarts import ( "encoding/json" + "errors" "fmt" "path" "strconv" @@ -15,8 +16,8 @@ import ( const ( //notebook - storageTypeOBS = "obs" - autoStopDuration = 4 * 60 * 60 + storageTypeOBS = "obs" + autoStopDuration = 4 * 60 * 60 autoStopDurationMs = 4 * 60 * 60 * 1000 DataSetMountPath = "/home/ma-user/work" @@ -63,6 +64,7 @@ const ( var ( poolInfos *models.PoolInfos FlavorInfos *models.FlavorInfos + ImageInfos *models.ImageInfosModelArts ) type GenerateTrainJobReq struct { @@ -263,31 +265,38 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin return nil } -func GenerateNotebook2(ctx *context.Context, jobName, uuid, description, flavor string) error { +func GenerateNotebook2(ctx *context.Context, jobName, uuid, description, flavor, imageId string) error { if poolInfos == nil { json.Unmarshal([]byte(setting.PoolInfos), &poolInfos) } + + imageName, err := GetNotebookImageName(imageId) + if err != nil { + log.Error("GetNotebookImageName failed: %v", err.Error()) + return err + } + jobResult, err := createNotebook2(models.CreateNotebook2Params{ JobName: jobName, Description: description, Flavor: flavor, Duration: autoStopDurationMs, - ImageID: "59a6e9f5-93c0-44dd-85b0-82f390c5d53a", + ImageID: imageId, PoolID: poolInfos.PoolInfo[0].PoolId, Feature: models.NotebookFeature, - Volume: models.VolumeReq{ - Capacity: 100, - Category: models.EVSCategory, - Ownership: models.ManagedOwnership, + Volume: models.VolumeReq{ + Capacity: setting.Capacity, + Category: models.EVSCategory, + Ownership: models.ManagedOwnership, }, - WorkspaceID: "0", + WorkspaceID: "0", }) if err != nil { log.Error("createNotebook2 failed: %v", err.Error()) return err } err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: string(models.JobWaiting), + Status: jobResult.Status, UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, JobID: jobResult.ID, @@ -296,6 +305,7 @@ func GenerateNotebook2(ctx *context.Context, jobName, uuid, description, flavor Type: models.TypeCloudBrainTwo, Uuid: uuid, ComputeResource: models.NPUResource, + Image: imageName, }) if err != nil { @@ -598,3 +608,26 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e return nil } + +func GetNotebookImageName(imageId string) (string, error) { + var validImage = false + var imageName = "" + + if ImageInfos == nil { + json.Unmarshal([]byte(setting.ImageInfos), &ImageInfos) + } + + for _, imageInfo := range ImageInfos.ImageInfo { + if imageInfo.Id == imageId { + validImage = true + imageName = imageInfo.Value + } + } + + if !validImage { + log.Error("the image id(%s) is invalid", imageId) + return imageName, errors.New("the image id is invalid") + } + + return imageName, nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c6828f9f7..4e063fe05 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -513,6 +513,8 @@ var ( PoolInfos string Flavor string DebugHost string + ImageInfos string + Capacity int //train-job ResourcePools string Engines string @@ -1326,7 +1328,8 @@ func NewContext() { ProfileID = sec.Key("PROFILE_ID").MustString("") PoolInfos = sec.Key("POOL_INFOS").MustString("") Flavor = sec.Key("FLAVOR").MustString("") - DebugHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73") + ImageInfos = sec.Key("IMAGE_INFOS").MustString("") + Capacity = sec.Key("IMAGE_INFOS").MustInt(100) ResourcePools = sec.Key("Resource_Pools").MustString("") Engines = sec.Key("Engines").MustString("") EngineVersions = sec.Key("Engine_Versions").MustString("") diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 647afb0d0..0f0985b47 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -103,8 +103,13 @@ func MustEnableModelArts(ctx *context.Context) { } func NotebookNew(ctx *context.Context) { - ctx.Data["PageIsCloudBrain"] = true + notebookNewDataPrepare(ctx) + + ctx.HTML(200, tplModelArtsNotebookNew) +} +func notebookNewDataPrepare(ctx *context.Context) error { + ctx.Data["PageIsCloudBrain"] = true t := time.Now() var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["job_name"] = jobName @@ -112,26 +117,14 @@ func NotebookNew(ctx *context.Context) { attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) if err != nil { ctx.ServerError("GetAllUserAttachments failed:", err) - return + return err } - ctx.Data["attachments"] = attachs - ctx.Data["dataset_path"] = modelarts.DataSetMountPath - ctx.Data["env"] = modelarts.NotebookEnv - ctx.Data["notebook_type"] = modelarts.NotebookType - if modelarts.FlavorInfos == nil { - json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) - } - ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo - - ctx.HTML(200, tplModelArtsNotebookNew) -} -func notebookNewDataPrepare(ctx *context.Context) error { - ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] - ctx.Data["job_name"] = jobName + if modelarts.ImageInfos == nil { + json.Unmarshal([]byte(setting.ImageInfos), &modelarts.ImageInfos) + } + ctx.Data["images"] = modelarts.ImageInfos.ImageInfo if modelarts.FlavorInfos == nil { json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) @@ -191,8 +184,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm uuid := form.Attachment description := form.Description flavor := form.Flavor - - flavor = "modelarts.bm.910.arm.public.1" + imageId := form.ImageId count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) if err != nil { @@ -223,7 +215,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm } } - err = modelarts.GenerateNotebook2(ctx, jobName, uuid, description, flavor) + err = modelarts.GenerateNotebook2(ctx, jobName, uuid, description, flavor, imageId) if err != nil { log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) notebookNewDataPrepare(ctx) diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index babd65e58..4bf63ac81 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -51,7 +51,16 @@ -