From 3a26a12a5382bf4b1d51f746b2ef66156e337e80 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 14:50:44 +0800 Subject: [PATCH 01/18] fix-3262 --- models/cloudbrain.go | 2 ++ modules/setting/setting.go | 1 + routers/repo/cloudbrain.go | 13 ++++++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1e7c702ab..b552ecde8 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2424,6 +2424,7 @@ type DatasetInfo struct { DataLocalPath string Name string FullName string + Size int64 } func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { @@ -2479,6 +2480,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, + Size: attach.Size, } if i == 0 { datasetNames = attach.Name diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a473dad6a..ed09de790 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,6 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string + DebugDatasetSize int64 //benchmark config IsBenchmarkEnabled bool diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d3d76f440..05376f138 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -293,6 +293,13 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } + + if jobType == string(models.JobTypeDebug) { + for _, infos := range datasetInfos { + log.Info("the size of dataset", infos.Size) + } + + } } command := cloudbrain.GetCloudbrainDebugCommand() @@ -842,10 +849,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile!=""{ - ctx.Redirect(getFileUrl(debugUrl,task.BootFile)) + if task.BootFile != "" { + ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) - }else{ + } else { ctx.Redirect(debugUrl) } From 90a0ef946a4d5d634fb5ac9bed618f6b4e300078 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 15:53:40 +0800 Subject: [PATCH 02/18] fix-3262 --- models/cloudbrain.go | 4 ++-- modules/setting/setting.go | 3 ++- routers/repo/cloudbrain.go | 11 +++++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index b552ecde8..6652647dc 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2424,7 +2424,7 @@ type DatasetInfo struct { DataLocalPath string Name string FullName string - Size int64 + Size int } func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { @@ -2480,7 +2480,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, - Size: attach.Size, + Size: int(attach.Size / (1024 * 1024 * 1024)), //GB } if i == 0 { datasetNames = attach.Name diff --git a/modules/setting/setting.go b/modules/setting/setting.go index ed09de790..5f4eeb53e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,7 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - DebugDatasetSize int64 + DebugDatasetSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,6 +1480,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") + DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(15) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 05376f138..a01ef20e7 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -296,9 +296,16 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType == string(models.JobTypeDebug) { for _, infos := range datasetInfos { - log.Info("the size of dataset", infos.Size) + log.Info("the size of dataset(GB):", infos.Size) + + if infos.Size > int(setting.DebugDatasetSize) { + log.Info("datasize exceed limit:", infos.Size) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) + return + } } - } } From 3280eb2c126b5a6fe50a6597ffb3f0c625802b92 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 16:05:06 +0800 Subject: [PATCH 03/18] fix-3262 --- options/locale/locale_zh-CN.ini | 2 +- routers/repo/cloudbrain.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8f9e6b664..0c15605ff 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3304,7 +3304,7 @@ Stopped_failed=任务停止失败，请稍后再试。 Stopped_success_update_status_fail=任务停止成功，状态及运行时间更新失败。 load_code_failed=代码加载失败，请确认选择了正确的分支。 - +error.debug_datasetsize = 数据集大小超过限制（15GB） error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a01ef20e7..f91f224ab 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -296,10 +296,8 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType == string(models.JobTypeDebug) { for _, infos := range datasetInfos { - log.Info("the size of dataset(GB):", infos.Size) - if infos.Size > int(setting.DebugDatasetSize) { - log.Info("datasize exceed limit:", infos.Size) + log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) From 2a77370d0216ca4361787b622038d5e7f955c714 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 16:32:31 +0800 Subject: [PATCH 04/18] fix-3262 --- routers/repo/cloudbrain.go | 1 - routers/repo/modelarts.go | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f91f224ab..64111ee04 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -284,7 +284,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } var datasetInfos map[string]models.DatasetInfo var datasetNames string - //var if uuids != "" { datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) if err != nil { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 01d2e2fa4..52a4ea566 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -251,6 +251,21 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return } + + var datasetInfos map[string]models.DatasetInfo + if uuid != "" { + datasetInfos, _, err = models.GetDatasetInfo(uuid) + for _, infos := range datasetInfos { + if infos.Size > int(setting.DebugDatasetSize) { + log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + notebookNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) + return + } + } + + } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") } From 4a4a02cc9004d12a91a91c1d6141dbad14b6103d Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 16:42:02 +0800 Subject: [PATCH 05/18] fix-3262 --- routers/repo/modelarts.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 52a4ea566..fed31618f 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -218,6 +218,22 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm return } } + + var datasetInfos map[string]models.DatasetInfo + if uuid != "" { + datasetInfos, _, err = models.GetDatasetInfo(uuid) + for _, infos := range datasetInfos { + if infos.Size > int(setting.DebugDatasetSize) { + log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + notebookNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) + return + } + } + + } + var aiCenterCode = models.AICenterOfCloudBrainTwo if setting.ModelartsCD.Enabled { aiCenterCode = models.AICenterOfChengdu @@ -251,21 +267,6 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return } - - var datasetInfos map[string]models.DatasetInfo - if uuid != "" { - datasetInfos, _, err = models.GetDatasetInfo(uuid) - for _, infos := range datasetInfos { - if infos.Size > int(setting.DebugDatasetSize) { - log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) - notebookNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) - return - } - } - - } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") } From 49e821d89e0f38c4a6e36f0f9143e55d0c8921db Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 6 Dec 2022 15:48:15 +0800 Subject: [PATCH 06/18] fix-3262 --- models/cloudbrain.go | 2 +- modules/setting/setting.go | 2 +- options/locale/locale_zh-CN.ini | 2 +- routers/repo/cloudbrain.go | 2 +- routers/repo/dataset.go | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6652647dc..3dc8d9694 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2480,7 +2480,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, - Size: int(attach.Size / (1024 * 1024 * 1024)), //GB + Size: int(attach.Size), } if i == 0 { datasetNames = attach.Name diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5f4eeb53e..05a2ba300 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1480,7 +1480,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(15) + DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(20 * 1024 * 1024 * 1024) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 0c15605ff..e7adeea18 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3304,7 +3304,7 @@ Stopped_failed=任务停止失败，请稍后再试。 Stopped_success_update_status_fail=任务停止成功，状态及运行时间更新失败。 load_code_failed=代码加载失败，请确认选择了正确的分支。 -error.debug_datasetsize = 数据集大小超过限制（15GB） +error.debug_datasetsize = 数据集大小超过限制（20GB） error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 64111ee04..eba6d79c9 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -297,7 +297,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { for _, infos := range datasetInfos { if infos.Size > int(setting.DebugDatasetSize) { log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize / (1024*1024*1024))) // GB cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) return diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 1f445492d..38f5c7f23 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -373,6 +373,7 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { } data, err := json.Marshal(datasets) + log.Info("datakey", string(data)) if err != nil { log.Error("json.Marshal failed:", err.Error()) ctx.JSON(200, map[string]string{ @@ -521,6 +522,8 @@ func GetDatasetStatus(ctx *context.Context) { return } + log.Info(fmt.Sprint(attachment.DecompressState)) + ctx.JSON(200, map[string]string{ "result_code": "0", "UUID": UUID, From bbf4d0623f88ca61691c159a3d89852152e1d9a9 Mon Sep 17 00:00:00 2001 From: Gitea Date: Wed, 7 Dec 2022 09:45:36 +0800 Subject: [PATCH 07/18] =?UTF-8?q?=E9=99=90=E5=88=B6=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 4 ++-- modules/templates/helper.go | 5 ++++- routers/repo/cloudbrain.go | 4 ++-- routers/repo/dataset.go | 2 -- routers/repo/modelarts.go | 6 +++--- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 05a2ba300..b9b0f756e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,7 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - DebugDatasetSize int + DebugAttachSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,7 +1480,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(20 * 1024 * 1024 * 1024) + DebugAttachSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(20 * 1024 * 1024 * 1024) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index c314127f1..3ae028499 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -97,6 +97,9 @@ func NewFuncMap() []template.FuncMap { "AllowedReactions": func() []string { return setting.UI.Reactions }, + "DebugAttachSize": func() int { + return setting.DebugAttachSize + }, "AvatarLink": models.AvatarLink, "Safe": Safe, "SafeJS": SafeJS, @@ -151,7 +154,7 @@ func NewFuncMap() []template.FuncMap { "EscapePound": func(str string) string { return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str) }, - "IpynbBool":func(str string) bool{ + "IpynbBool": func(str string) bool { return strings.Contains(str, ".ipynb") }, "nl2br": func(text string) template.HTML { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index eba6d79c9..2cd6f3ee0 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -295,9 +295,9 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType == string(models.JobTypeDebug) { for _, infos := range datasetInfos { - if infos.Size > int(setting.DebugDatasetSize) { + if infos.Size > int(setting.DebugAttachSize) { log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize / (1024*1024*1024))) // GB + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize/(1024*1024*1024))) // GB cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) return diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 38f5c7f23..32c884b5e 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -522,8 +522,6 @@ func GetDatasetStatus(ctx *context.Context) { return } - log.Info(fmt.Sprint(attachment.DecompressState)) - ctx.JSON(200, map[string]string{ "result_code": "0", "UUID": UUID, diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index fed31618f..96e408a76 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -218,14 +218,14 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm return } } - + var datasetInfos map[string]models.DatasetInfo if uuid != "" { datasetInfos, _, err = models.GetDatasetInfo(uuid) for _, infos := range datasetInfos { - if infos.Size > int(setting.DebugDatasetSize) { + if infos.Size > int(setting.DebugAttachSize) { log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize)) notebookNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) return From 8de31e8bf0c88b4ba85b8e1f41820b80f7e12a2e Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 7 Dec 2022 16:01:12 +0800 Subject: [PATCH 08/18] fix issue --- templates/repo/cloudbrain/new.tmpl | 4 +++- templates/repo/modelarts/notebook/new.tmpl | 2 +- web_src/js/components/dataset/selectDataset.vue | 30 +++++++++++++++++++++++++ web_src/js/features/i18nVue.js | 2 ++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index b75e75cdf..cb18517b3 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -23,7 +23,8 @@ {{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -216,6 +217,7 @@ {{template "base/footer" .}}