From 91455626a1a0ce18c5bd7823b6a9ecb3a7bf4455 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 21 Jun 2022 17:57:10 +0800 Subject: [PATCH 1/6] test --- routers/repo/modelarts.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 12a5a0623..f2fa4e370 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -985,6 +985,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) FlavorName := form.FlavorName VersionCount := modelarts.VersionCount EngineName := form.EngineName + dataPath = "/test-opendata/attachment/1/2/12974bf4-3865-4995-a14f-e9e2af343f2f12974bf4-3865-4995-a14f-e9e2af343f2f/;s3://test-opendata/attachment/1/2/12974bf4-3865-4995-a14f-e9e2af343f2f12974bf4-3865-4995-a14f-e9e2af343f2f/" count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { From b017248de66ebd45e62155a672272b87c40896de Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 22 Jun 2022 17:03:22 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E4=BA=91=E8=84=91=E4=BA=8C=E5=A4=9A?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=80=89=E6=8B=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/modelarts.go | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f2fa4e370..269c7425e 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -979,14 +979,19 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" - dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + // dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName isLatestVersion := modelarts.IsLatestVersion FlavorName := form.FlavorName VersionCount := modelarts.VersionCount EngineName := form.EngineName - dataPath = "/test-opendata/attachment/1/2/12974bf4-3865-4995-a14f-e9e2af343f2f12974bf4-3865-4995-a14f-e9e2af343f2f/;s3://test-opendata/attachment/1/2/12974bf4-3865-4995-a14f-e9e2af343f2f12974bf4-3865-4995-a14f-e9e2af343f2f/" - + dataPath, err := GetObsDataPath(uuid) + if err != nil { + log.Error("GetObsDataPath failed:%v", err, ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("GetObsDataPath error", tplModelArtsTrainJobNew, &form) + return + } count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -2421,3 +2426,24 @@ func TrainJobDownloadLogFile(ctx *context.Context) { ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } +func GetObsDataPath(uuid string) (string, error) { + var obsDataPath string + list := strings.Split(uuid, ";") + for k, uuidStr := range list { + _, err := models.GetAttachmentByUUID(uuidStr) + if err != nil { + log.Error("GetAttachmentByUUID failed:%v", err) + return "", err + } else { + if k <= 0 { + obsDataPath = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + } + if k > 0 { + obsDataPathNext := ";" + "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + obsDataPath = obsDataPath + obsDataPathNext + } + + } + } + return obsDataPath, nil +} From 674de083a12a5573ffca9a8c852b6e83139e9f9e Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 22 Jun 2022 18:05:01 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E4=BA=91=E8=84=91=E4=BA=8C=E5=A4=9A?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E9=80=89=E6=8B=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 19 +++--------- modules/setting/setting.go | 2 ++ routers/repo/modelarts.go | 70 +++++++++++++++++++++++++++++++----------- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 78b40fd56..5988e62bb 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,14 @@ package modelarts import ( - "code.gitea.io/gitea/modules/timeutil" "encoding/json" "errors" "fmt" "path" "strconv" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -96,6 +97,7 @@ type GenerateTrainJobReq struct { VersionCount int EngineName string TotalVersionCount int + DatasetName string } type GenerateInferenceJobReq struct { @@ -335,11 +337,6 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error return err } - attach, err := models.GetAttachmentByUUID(req.Uuid) - if err != nil { - log.Error("GetAttachmentByUUID(%s) failed:%v", strconv.FormatInt(jobResult.JobID, 10), err.Error()) - return err - } jobId := strconv.FormatInt(jobResult.JobID, 10) err = models.CreateCloudbrain(&models.Cloudbrain{ Status: TransTrainJobStatus(jobResult.Status), @@ -353,7 +350,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error VersionID: jobResult.VersionID, VersionName: jobResult.VersionName, Uuid: req.Uuid, - DatasetName: attach.Name, + DatasetName: req.DatasetName, CommitID: req.CommitID, IsLatestVersion: req.IsLatestVersion, ComputeResource: models.NPUResource, @@ -408,12 +405,6 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job return err } - attach, err := models.GetAttachmentByUUID(req.Uuid) - if err != nil { - log.Error("GetAttachmentByUUID(%s) failed:%v", strconv.FormatInt(jobResult.JobID, 10), err.Error()) - return err - } - var jobTypes []string jobTypes = append(jobTypes, string(models.JobTypeTrain)) repo := ctx.Repo.Repository @@ -441,7 +432,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job VersionID: jobResult.VersionID, VersionName: jobResult.VersionName, Uuid: req.Uuid, - DatasetName: attach.Name, + DatasetName: req.DatasetName, CommitID: req.CommitID, IsLatestVersion: req.IsLatestVersion, PreVersionName: req.PreVersionName, diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5c87b68c5..8c26c7b9e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -465,6 +465,7 @@ var ( MaxDuration int64 TrainGpuTypes string TrainResourceSpecs string + MaxDatasetNum int //benchmark config IsBenchmarkEnabled bool @@ -1294,6 +1295,7 @@ func NewContext() { MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") + MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 269c7425e..b1864136a 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -985,13 +985,21 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) FlavorName := form.FlavorName VersionCount := modelarts.VersionCount EngineName := form.EngineName - dataPath, err := GetObsDataPath(uuid) + if IsDatasetUseCountExceed(uuid) { + log.Error("DatasetUseCount is Exceed:%v") + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobNew, &form) + return + } + datasetName, err := GetDatasetNameByUUID(uuid) if err != nil { - log.Error("GetObsDataPath failed:%v", err, ctx.Data["MsgID"]) + log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"]) trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("GetObsDataPath error", tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobNew, &form) return } + dataPath := GetObsDataPathByUUID(uuid) + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -1167,6 +1175,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) EngineName: EngineName, VersionCount: VersionCount, TotalVersionCount: modelarts.TotalVersionCount, + DatasetName: datasetName, } //将params转换Parameters.Parameter，出错时返回给前端 @@ -1228,13 +1237,22 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" - dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + // dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName PreVersionName := form.VersionName FlavorName := form.FlavorName EngineName := form.EngineName isLatestVersion := modelarts.IsLatestVersion + datasetName, err := GetDatasetNameByUUID(uuid) + if err != nil { + log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobNew, &form) + return + } + dataPath := GetObsDataPathByUUID(uuid) + canNewJob, _ := canUserCreateTrainJobVersion(ctx, latestTask.UserID) if !canNewJob { ctx.RenderWithErr("user cann't new trainjob", tplModelArtsTrainJobVersionNew, &form) @@ -1392,6 +1410,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ EngineName: EngineName, PreVersionName: PreVersionName, TotalVersionCount: latestTask.TotalVersionCount + 1, + DatasetName: datasetName, } err = modelarts.GenerateTrainJobVersion(ctx, req, jobID) @@ -2426,24 +2445,39 @@ func TrainJobDownloadLogFile(ctx *context.Context) { ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } -func GetObsDataPath(uuid string) (string, error) { +func GetObsDataPathByUUID(uuid string) string { var obsDataPath string - list := strings.Split(uuid, ";") - for k, uuidStr := range list { - _, err := models.GetAttachmentByUUID(uuidStr) + uuidList := strings.Split(uuid, ";") + for k, _ := range uuidList { + if k <= 0 { + obsDataPath = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + } + if k > 0 { + obsDataPathNext := ";" + "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + obsDataPath = obsDataPath + obsDataPathNext + } + + } + return obsDataPath +} +func GetDatasetNameByUUID(uuid string) (string, error) { + uuidList := strings.Split(uuid, ";") + var datasetName string + for _, uuidStr := range uuidList { + attach, err := models.GetAttachmentByUUID(uuidStr) if err != nil { log.Error("GetAttachmentByUUID failed:%v", err) return "", err - } else { - if k <= 0 { - obsDataPath = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - } - if k > 0 { - obsDataPathNext := ";" + "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - obsDataPath = obsDataPath + obsDataPathNext - } - } + datasetName = datasetName + attach.Name + ";" + } + return datasetName, nil +} +func IsDatasetUseCountExceed(uuid string) bool { + uuidList := strings.Split(uuid, ";") + if len(uuidList) > setting.MaxDatasetNum { + return true + } else { + return false } - return obsDataPath, nil } From e9ec5e63880fbac0e9d4e4e57ea9059c1c0927c5 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 22 Jun 2022 18:08:23 +0800 Subject: [PATCH 4/6] fix-bug --- routers/repo/modelarts.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index b1864136a..0a4a9bbc1 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1244,11 +1244,17 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ EngineName := form.EngineName isLatestVersion := modelarts.IsLatestVersion + if IsDatasetUseCountExceed(uuid) { + log.Error("DatasetUseCount is Exceed:%v") + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobVersionNew, &form) + return + } datasetName, err := GetDatasetNameByUUID(uuid) if err != nil { log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"]) - trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobNew, &form) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobVersionNew, &form) return } dataPath := GetObsDataPathByUUID(uuid) From 123d9399b3c4ca5253bb771e29333e2e02738ea5 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 23 Jun 2022 14:30:30 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=BC=95=E7=94=A8=E6=AC=A1=E6=95=B0?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=9A=E6=95=B0=E6=8D=AE=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 4 +++- models/dataset.go | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 31aefb3d1..e1af2e079 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -110,8 +110,10 @@ func (a *Attachment) IncreaseDownloadCount() error { } func IncreaseAttachmentUseNumber(uuid string) error { + + uuidInCondition := "(" + strings.ReplaceAll(uuid, ";", ",") + ")" // Update use number. - if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid=?", uuid); err != nil { + if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid in", uuidInCondition); err != nil { return fmt.Errorf("increase attachment use count: %v", err) } diff --git a/models/dataset.go b/models/dataset.go index 4122b63b2..2194aa42b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -445,10 +445,22 @@ func UpdateDataset(ctx DBContext, rel *Dataset) error { func IncreaseDatasetUseCount(uuid string) { IncreaseAttachmentUseNumber(uuid) + attachments, _ := GetAttachmentsByUUIDs(strings.Split(uuid, ";")) - attachment, _ := GetAttachmentByUUID(uuid) - if attachment != nil { - x.Exec("UPDATE `dataset` SET use_count=use_count+1 WHERE id=?", attachment.DatasetID) + countMap := make(map[int64]int) + + for _, attachment := range attachments { + value, ok := countMap[attachment.DatasetID] + if ok { + countMap[attachment.DatasetID] = value + 1 + } else { + countMap[attachment.DatasetID] = 1 + } + + } + + for key, value := range countMap { + x.Exec("UPDATE `dataset` SET use_count=use_count+? WHERE id=?", value, key) } } From b9cb90d85e00f9c365619859a43bc35dd20f6eed Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 23 Jun 2022 15:46:37 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index e1af2e079..9b0083aaa 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -111,9 +111,14 @@ func (a *Attachment) IncreaseDownloadCount() error { func IncreaseAttachmentUseNumber(uuid string) error { - uuidInCondition := "(" + strings.ReplaceAll(uuid, ";", ",") + ")" + uuidArray := strings.Split(uuid, ";") + for i := range uuidArray { + uuidArray[i] = "'" + uuidArray[i] + "'" + } + + uuidInCondition := "(" + strings.Join(uuidArray, ",") + ")" // Update use number. - if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid in", uuidInCondition); err != nil { + if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid in " + uuidInCondition); err != nil { return fmt.Errorf("increase attachment use count: %v", err) }