From e7613c40842e42a4e5b183edb13b1f0b0204d858 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 11 Jul 2022 17:43:26 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 11 ++++++----- modules/cloudbrain/cloudbrain.go | 7 +++++++ modules/setting/setting.go | 3 +++ routers/repo/cloudbrain.go | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 8e1b94a97..2c23bf82d 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -570,11 +570,12 @@ type SpecialPools struct { Pools []*SpecialPool `json:"pools"` } type SpecialPool struct { - Org string `json:"org"` - Type string `json:"type"` - IsExclusive bool `json:"isExclusive"` - Pool []*GpuInfo `json:"pool"` - JobType []string `json:"jobType"` + Org string `json:"org"` + Type string `json:"type"` + IsExclusive bool `json:"isExclusive"` + Pool []*GpuInfo `json:"pool"` + JobType []string `json:"support_type"` + ResourceSpec []*ResourceSpec `json:"resource_specs"` } type ImageInfosModelArts struct { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 430304dd5..09d6b9c0e 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -42,6 +42,7 @@ const ( var ( ResourceSpecs *models.ResourceSpecs TrainResourceSpecs *models.ResourceSpecs + SpecialPools *models.SpecialPools ) type GenerateCloudBrainTaskReq struct { @@ -538,3 +539,9 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e return nil } + +func InitSpecialPool() { + if SpecialPools == nil && setting.SpecialPools != "" { + json.Unmarshal([]byte(setting.SpecialPools), &SpecialPools) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 506e31ba3..d206ed9a3 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -460,6 +460,7 @@ var ( CBCodePathPrefix string JobType string GpuTypes string + SpecialPools string DebugServerHost string ResourceSpecs string MaxDuration int64 @@ -1311,6 +1312,8 @@ func NewContext() { MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") + SpecialPools = sec.Key("SPECIAL_POOL").MustString("") + MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) sec = Cfg.Section("benchmark") diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5a3d0a6f8..55776a2b4 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -149,6 +149,8 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType + cloudbrain.InitSpecialPool() + if gpuInfos == nil { json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) } @@ -178,6 +180,44 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) } ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec + + if cloudbrain.SpecialPools != nil { + var debugGpuTypes []*models.GpuInfo + var trainGpuTypes []*models.GpuInfo + + for _, pool := range cloudbrain.SpecialPools.Pools { + org, _ := models.GetOrgByName(pool.Org) + if org != nil { + isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID) + if isOrgMember { + for _, jobType := range pool.JobType { + if jobType == string(models.JobTypeDebug) { + debugGpuTypes = append(debugGpuTypes, pool.Pool...) + if pool.ResourceSpec != nil { + ctx.Data["resource_specs"] = pool.ResourceSpec + } + } else if jobType == string(models.JobTypeTrain) { + trainGpuTypes = append(trainGpuTypes, pool.Pool...) + if pool.ResourceSpec != nil { + ctx.Data["train_resource_specs"] = pool.ResourceSpec + } + } + } + } + } + + } + + if len(debugGpuTypes) > 0 { + ctx.Data["gpu_types"] = debugGpuTypes + } + + if len(trainGpuTypes) > 0 { + ctx.Data["train_gpu_types"] = trainGpuInfos + } + + } + ctx.Data["params"] = "" ctx.Data["branchName"] = ctx.Repo.BranchName From 42ef86cb9ed6b0bd3c023780b1bfce2285d7b072 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 12 Jul 2022 10:49:26 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BA=91=E8=84=91=E4=B8=80=E4=B8=93?= =?UTF-8?q?=E5=B1=9E=E8=B5=84=E6=BA=90=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 ++-- modules/cloudbrain/cloudbrain.go | 47 ++++++++++++++++++++++++++++++++++++++ routers/repo/cloudbrain.go | 49 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 2c23bf82d..ea1826de0 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -574,8 +574,8 @@ type SpecialPool struct { Type string `json:"type"` IsExclusive bool `json:"isExclusive"` Pool []*GpuInfo `json:"pool"` - JobType []string `json:"support_type"` - ResourceSpec []*ResourceSpec `json:"resource_specs"` + JobType []string `json:"jobType"` + ResourceSpec []*ResourceSpec `json:"resourceSpecs"` } type ImageInfosModelArts struct { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 09d6b9c0e..f48665cb0 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -223,6 +223,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { for _, spec := range TrainResourceSpecs.ResourceSpec { if req.ResourceSpecId == spec.Id { resourceSpec = spec + break } } } else { @@ -232,10 +233,26 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { for _, spec := range ResourceSpecs.ResourceSpec { if req.ResourceSpecId == spec.Id { resourceSpec = spec + break } } } + //如果没有匹配到spec信息，尝试从专属资源池获取 + if resourceSpec == nil && SpecialPools != nil { + for _, specialPool := range SpecialPools.Pools { + if specialPool.ResourceSpec != nil { + if IsElementExist(specialPool.JobType, req.JobType) && IsQueueInSpecialtPool(specialPool.Pool, req.GpuQueue) { + for _, spec := range specialPool.ResourceSpec { + if req.ResourceSpecId == spec.Id { + resourceSpec = spec + break + } + } + } + } + } + } if resourceSpec == nil { log.Error("no such resourceSpecId(%d)", req.ResourceSpecId, req.Ctx.Data["MsgID"]) @@ -545,3 +562,33 @@ func InitSpecialPool() { json.Unmarshal([]byte(setting.SpecialPools), &SpecialPools) } } + +func IsResourceSpecInSpecialPool(resourceSpecs []*models.ResourceSpec, resourceSpecId int) bool { + if resourceSpecs == nil || len(resourceSpecs) == 0 { + return true + } + for _, v := range resourceSpecs { + if v.Id == resourceSpecId { + return true + } + } + return false +} + +func IsQueueInSpecialtPool(pool []*models.GpuInfo, queue string) bool { + for _, v := range pool { + if v.Queue == queue { + return true + } + } + return false +} + +func IsElementExist(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + return false +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 55776a2b4..17a7b0214 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -213,7 +213,7 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } if len(trainGpuTypes) > 0 { - ctx.Data["train_gpu_types"] = trainGpuInfos + ctx.Data["train_gpu_types"] = trainGpuTypes } } @@ -257,6 +257,10 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { repo := ctx.Repo.Repository tpl := tplCloudBrainNew + if jobType == string(models.JobTypeTrain) { + tpl = tplCloudBrainTrainJobNew + } + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { @@ -322,6 +326,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command = commandTrain } + errStr := checkCloudBrainSpecialPool(ctx, jobType, gpuQueue, resourceSpecId) + + if errStr != "" { + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr(errStr, tpl, &form) + } + if branchName == "" { branchName = cloudbrain.DefaultBranchName } @@ -374,6 +385,42 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } } +/** + 检查用户传输的参数是否符合专属资源池 +*/ +func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string { + if cloudbrain.SpecialPools != nil { + + var isInPoolOrg = false + var matchSpecialPool = false + + for _, specialPool := range cloudbrain.SpecialPools.Pools { + + if cloudbrain.IsElementExist(specialPool.JobType, jobType) && cloudbrain.IsQueueInSpecialtPool(specialPool.Pool, queue) { + if cloudbrain.IsResourceSpecInSpecialPool(specialPool.ResourceSpec, resourceSpecId) { + matchSpecialPool = true + org, _ := models.GetOrgByName(specialPool.Org) + if org != nil { + isInPoolOrg, _ = models.IsOrganizationMember(org.ID, ctx.User.ID) + if isInPoolOrg { + break //传入参数，和专属资源池匹配上了，检查通过 + } + } + } + + } + + } + //资源池有匹配上，但是用户不在相应的组织中，返回错误信息。界面已经过滤了选择，界面操作不会到这个逻辑 + if matchSpecialPool && !isInPoolOrg { + return ctx.Tr("repo.grampus.no_operate_right") + } + + } + //没有匹配到资源池或者没有设置专属资源池，检查通过; 获取和资源池完全匹配检查通过 + return "" +} + func CloudBrainRestart(ctx *context.Context) { var ID = ctx.Params(":id") var resultCode = "0" From b1107fb8f7f1d5674d5fb318aff6603226d706d9 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 12 Jul 2022 11:16:25 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 3 +++ routers/repo/cloudbrain.go | 1 + 2 files changed, 4 insertions(+) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index f48665cb0..6c0681aba 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -241,6 +241,9 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { //如果没有匹配到spec信息，尝试从专属资源池获取 if resourceSpec == nil && SpecialPools != nil { for _, specialPool := range SpecialPools.Pools { + if resourceSpec != nil { + break + } if specialPool.ResourceSpec != nil { if IsElementExist(specialPool.JobType, req.JobType) && IsQueueInSpecialtPool(specialPool.Pool, req.GpuQueue) { for _, spec := range specialPool.ResourceSpec { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 17a7b0214..3aa26e0ed 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -203,6 +203,7 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } } } + break } } From e31e058e632e358ec7fe0604154eeed2e4b475db Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 12 Jul 2022 11:43:50 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 3aa26e0ed..9cc0b5ebe 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -332,6 +332,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if errStr != "" { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(errStr, tpl, &form) + return } if branchName == "" {