From 3a26a12a5382bf4b1d51f746b2ef66156e337e80 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 14:50:44 +0800 Subject: [PATCH 01/85] fix-3262 --- models/cloudbrain.go | 2 ++ modules/setting/setting.go | 1 + routers/repo/cloudbrain.go | 13 ++++++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1e7c702ab..b552ecde8 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2424,6 +2424,7 @@ type DatasetInfo struct { DataLocalPath string Name string FullName string + Size int64 } func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { @@ -2479,6 +2480,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, + Size: attach.Size, } if i == 0 { datasetNames = attach.Name diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a473dad6a..ed09de790 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,6 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string + DebugDatasetSize int64 //benchmark config IsBenchmarkEnabled bool diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d3d76f440..05376f138 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -293,6 +293,13 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } + + if jobType == string(models.JobTypeDebug) { + for _, infos := range datasetInfos { + log.Info("the size of dataset", infos.Size) + } + + } } command := cloudbrain.GetCloudbrainDebugCommand() @@ -842,10 +849,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile!=""{ - ctx.Redirect(getFileUrl(debugUrl,task.BootFile)) + if task.BootFile != "" { + ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) - }else{ + } else { ctx.Redirect(debugUrl) } From 90a0ef946a4d5d634fb5ac9bed618f6b4e300078 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 15:53:40 +0800 Subject: [PATCH 02/85] fix-3262 --- models/cloudbrain.go | 4 ++-- modules/setting/setting.go | 3 ++- routers/repo/cloudbrain.go | 11 +++++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index b552ecde8..6652647dc 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2424,7 +2424,7 @@ type DatasetInfo struct { DataLocalPath string Name string FullName string - Size int64 + Size int } func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { @@ -2480,7 +2480,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, - Size: attach.Size, + Size: int(attach.Size / (1024 * 1024 * 1024)), //GB } if i == 0 { datasetNames = attach.Name diff --git a/modules/setting/setting.go b/modules/setting/setting.go index ed09de790..5f4eeb53e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,7 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - DebugDatasetSize int64 + DebugDatasetSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,6 +1480,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") + DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(15) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 05376f138..a01ef20e7 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -296,9 +296,16 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType == string(models.JobTypeDebug) { for _, infos := range datasetInfos { - log.Info("the size of dataset", infos.Size) + log.Info("the size of dataset(GB):", infos.Size) + + if infos.Size > int(setting.DebugDatasetSize) { + log.Info("datasize exceed limit:", infos.Size) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) + return + } } - } } From 3280eb2c126b5a6fe50a6597ffb3f0c625802b92 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 16:05:06 +0800 Subject: [PATCH 03/85] fix-3262 --- options/locale/locale_zh-CN.ini | 2 +- routers/repo/cloudbrain.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8f9e6b664..0c15605ff 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3304,7 +3304,7 @@ Stopped_failed=任务停止失败，请稍后再试。 Stopped_success_update_status_fail=任务停止成功，状态及运行时间更新失败。 load_code_failed=代码加载失败，请确认选择了正确的分支。 - +error.debug_datasetsize = 数据集大小超过限制（15GB） error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a01ef20e7..f91f224ab 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -296,10 +296,8 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType == string(models.JobTypeDebug) { for _, infos := range datasetInfos { - log.Info("the size of dataset(GB):", infos.Size) - if infos.Size > int(setting.DebugDatasetSize) { - log.Info("datasize exceed limit:", infos.Size) + log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) From 2a77370d0216ca4361787b622038d5e7f955c714 Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 16:32:31 +0800 Subject: [PATCH 04/85] fix-3262 --- routers/repo/cloudbrain.go | 1 - routers/repo/modelarts.go | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f91f224ab..64111ee04 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -284,7 +284,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } var datasetInfos map[string]models.DatasetInfo var datasetNames string - //var if uuids != "" { datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) if err != nil { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 01d2e2fa4..52a4ea566 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -251,6 +251,21 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return } + + var datasetInfos map[string]models.DatasetInfo + if uuid != "" { + datasetInfos, _, err = models.GetDatasetInfo(uuid) + for _, infos := range datasetInfos { + if infos.Size > int(setting.DebugDatasetSize) { + log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + notebookNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) + return + } + } + + } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") } From 4a4a02cc9004d12a91a91c1d6141dbad14b6103d Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 5 Dec 2022 16:42:02 +0800 Subject: [PATCH 05/85] fix-3262 --- routers/repo/modelarts.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 52a4ea566..fed31618f 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -218,6 +218,22 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm return } } + + var datasetInfos map[string]models.DatasetInfo + if uuid != "" { + datasetInfos, _, err = models.GetDatasetInfo(uuid) + for _, infos := range datasetInfos { + if infos.Size > int(setting.DebugDatasetSize) { + log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + notebookNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) + return + } + } + + } + var aiCenterCode = models.AICenterOfCloudBrainTwo if setting.ModelartsCD.Enabled { aiCenterCode = models.AICenterOfChengdu @@ -251,21 +267,6 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return } - - var datasetInfos map[string]models.DatasetInfo - if uuid != "" { - datasetInfos, _, err = models.GetDatasetInfo(uuid) - for _, infos := range datasetInfos { - if infos.Size > int(setting.DebugDatasetSize) { - log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) - notebookNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) - return - } - } - - } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") } From 49e821d89e0f38c4a6e36f0f9143e55d0c8921db Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 6 Dec 2022 15:48:15 +0800 Subject: [PATCH 06/85] fix-3262 --- models/cloudbrain.go | 2 +- modules/setting/setting.go | 2 +- options/locale/locale_zh-CN.ini | 2 +- routers/repo/cloudbrain.go | 2 +- routers/repo/dataset.go | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6652647dc..3dc8d9694 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2480,7 +2480,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, - Size: int(attach.Size / (1024 * 1024 * 1024)), //GB + Size: int(attach.Size), } if i == 0 { datasetNames = attach.Name diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5f4eeb53e..05a2ba300 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1480,7 +1480,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(15) + DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(20 * 1024 * 1024 * 1024) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 0c15605ff..e7adeea18 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3304,7 +3304,7 @@ Stopped_failed=任务停止失败，请稍后再试。 Stopped_success_update_status_fail=任务停止成功，状态及运行时间更新失败。 load_code_failed=代码加载失败，请确认选择了正确的分支。 -error.debug_datasetsize = 数据集大小超过限制（15GB） +error.debug_datasetsize = 数据集大小超过限制（20GB） error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 64111ee04..eba6d79c9 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -297,7 +297,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { for _, infos := range datasetInfos { if infos.Size > int(setting.DebugDatasetSize) { log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize / (1024*1024*1024))) // GB cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) return diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 1f445492d..38f5c7f23 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -373,6 +373,7 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { } data, err := json.Marshal(datasets) + log.Info("datakey", string(data)) if err != nil { log.Error("json.Marshal failed:", err.Error()) ctx.JSON(200, map[string]string{ @@ -521,6 +522,8 @@ func GetDatasetStatus(ctx *context.Context) { return } + log.Info(fmt.Sprint(attachment.DecompressState)) + ctx.JSON(200, map[string]string{ "result_code": "0", "UUID": UUID, From bbf4d0623f88ca61691c159a3d89852152e1d9a9 Mon Sep 17 00:00:00 2001 From: Gitea Date: Wed, 7 Dec 2022 09:45:36 +0800 Subject: [PATCH 07/85] =?UTF-8?q?=E9=99=90=E5=88=B6=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 4 ++-- modules/templates/helper.go | 5 ++++- routers/repo/cloudbrain.go | 4 ++-- routers/repo/dataset.go | 2 -- routers/repo/modelarts.go | 6 +++--- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 05a2ba300..b9b0f756e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,7 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - DebugDatasetSize int + DebugAttachSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,7 +1480,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - DebugDatasetSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(20 * 1024 * 1024 * 1024) + DebugAttachSize = sec.Key("DEBUG_DATASET_SIZE").MustInt(20 * 1024 * 1024 * 1024) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index c314127f1..3ae028499 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -97,6 +97,9 @@ func NewFuncMap() []template.FuncMap { "AllowedReactions": func() []string { return setting.UI.Reactions }, + "DebugAttachSize": func() int { + return setting.DebugAttachSize + }, "AvatarLink": models.AvatarLink, "Safe": Safe, "SafeJS": SafeJS, @@ -151,7 +154,7 @@ func NewFuncMap() []template.FuncMap { "EscapePound": func(str string) string { return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str) }, - "IpynbBool":func(str string) bool{ + "IpynbBool": func(str string) bool { return strings.Contains(str, ".ipynb") }, "nl2br": func(text string) template.HTML { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index eba6d79c9..2cd6f3ee0 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -295,9 +295,9 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType == string(models.JobTypeDebug) { for _, infos := range datasetInfos { - if infos.Size > int(setting.DebugDatasetSize) { + if infos.Size > int(setting.DebugAttachSize) { log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize / (1024*1024*1024))) // GB + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize/(1024*1024*1024))) // GB cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tpl, &form) return diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 38f5c7f23..32c884b5e 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -522,8 +522,6 @@ func GetDatasetStatus(ctx *context.Context) { return } - log.Info(fmt.Sprint(attachment.DecompressState)) - ctx.JSON(200, map[string]string{ "result_code": "0", "UUID": UUID, diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index fed31618f..96e408a76 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -218,14 +218,14 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm return } } - + var datasetInfos map[string]models.DatasetInfo if uuid != "" { datasetInfos, _, err = models.GetDatasetInfo(uuid) for _, infos := range datasetInfos { - if infos.Size > int(setting.DebugDatasetSize) { + if infos.Size > int(setting.DebugAttachSize) { log.Info("datasize exceed limit, the size of dataset(GB)::", infos.Size) - log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugDatasetSize)) + log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize)) notebookNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize"), tplModelArtsNotebookNew, &form) return From 8de31e8bf0c88b4ba85b8e1f41820b80f7e12a2e Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 7 Dec 2022 16:01:12 +0800 Subject: [PATCH 08/85] fix issue --- templates/repo/cloudbrain/new.tmpl | 4 +++- templates/repo/modelarts/notebook/new.tmpl | 2 +- web_src/js/components/dataset/selectDataset.vue | 30 +++++++++++++++++++++++++ web_src/js/features/i18nVue.js | 2 ++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index b75e75cdf..cb18517b3 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -23,7 +23,8 @@ {{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -216,6 +217,7 @@ {{template "base/footer" .}} + diff --git a/web_src/js/standalone/cloudbrainNew.js b/web_src/js/standalone/cloudbrainNew.js index bc54adc65..1823312db 100644 --- a/web_src/js/standalone/cloudbrainNew.js +++ b/web_src/js/standalone/cloudbrainNew.js @@ -6,7 +6,7 @@ if (createFlag) return false; createFlag = true; }; - $("select.dropdown").dropdown(); + // $("select.dropdown").dropdown(); $(document).keydown(function (event) { switch (event.keyCode) { case 13: @@ -182,4 +182,50 @@ } validate(); }); + + // 评测任务相关创建func + let repoLink = $(".cloudbrain-type").data("repo-link"); + function setChildType(type_id=1) { + if (type_id == 3) { + $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark'); + $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark'); + } else { + $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark'); + $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark'); + } + let child_selected_id = $('#benchmark_child_types_id_hidden').val(); + $.get(`${repoLink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => { + const n_length = data['child_types'].length + let html = '' + for (let i = 0; i < n_length; i++) { + if (child_selected_id == data['child_types'][i].id) { + html += ``; + } else { + html += ``; + } + } + document.getElementById("benchmark_child_types_id").innerHTML = html; + }) + } + $(document).ready(function () { + if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') { + setChildType(); + } + $(".ui.selection.dropdown.benchmark_types_id").dropdown({ + onChange:function (value, text, $selectedItem){ + setChildType(value) + } + }) + $('.ui.search.dropdown.job_type').dropdown({ + onChange: function (value, text, $selectedItem) { + if (value === "BRAINSCORE") { + $('#brainscore_child_type').css('display', 'block') + $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann') + } else { + $('#brainscore_child_type').css('display', 'none') + $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet') + } + } + }) + }) })(); From e6b3253add4985da944c968d4b98d2445de942cb Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 11:24:40 +0800 Subject: [PATCH 23/85] fix issue --- templates/repo/cloudbrain/benchmark/new.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index 669fd42ab..3772d99f5 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -3,7 +3,7 @@
{{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} From ab729b4f28a34f31f0b36a3ac3eac383d812a44c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 21 Dec 2022 12:42:23 +0800 Subject: [PATCH 24/85] notebook multifile --- models/cloudbrain.go | 8 + modules/cloudbrain/cloudbrain.go | 1 + modules/modelarts/modelarts.go | 84 +-------- modules/modelarts_cd/modelarts.go | 1 + modules/notebook/contentManager.go | 198 ++++++++++++++++++++ modules/structs/cloudbrain.go | 1 + options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 2 +- routers/api/v1/api.go | 3 +- routers/api/v1/repo/cloudbrain.go | 3 + routers/repo/modelarts.go | 5 +- services/cloudbrain/cloudbrainTask/notebook.go | 243 ++++++++++++++++++++++--- 12 files changed, 435 insertions(+), 116 deletions(-) create mode 100644 modules/notebook/contentManager.go diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 37965c73a..099f07c3a 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2126,6 +2126,14 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobName: jobName} return getRepoCloudBrain(cb) } +func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) { + cloudBrain := new(Cloudbrain) + has, err := x.In("status", JobWaiting, JobRunning).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) + if has { + return cloudBrain, err + } + return nil, err +} func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index c85f4b8cd..827e2b736 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -709,6 +709,7 @@ type GenerateModelArtsNotebookReq struct { ImageId string AutoStopDurationMs int64 + BranchName string Spec *models.Specification ModelName string diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index dcad1eb00..bd7f848fc 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,9 @@ package modelarts import ( - "encoding/base64" "encoding/json" "errors" "fmt" - "io/ioutil" - "net/http" - "path" "strconv" "strings" @@ -239,6 +235,7 @@ func GenerateNotebook2(ctx *context.Context, req cloudbrain.GenerateModelArtsNot ComputeResource: models.NPUResource, Image: imageName, BootFile: req.BootFile, + BranchName: req.BranchName, Description: req.Description, CreatedUnix: createTime, UpdatedUnix: createTime, @@ -830,10 +827,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { task.FlavorCode = result.Flavor } - if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" { - uploadNoteBookFile(task, result) - - } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -844,81 +837,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { return nil } -func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { - jupyterUrl := result.Url + "?token=" + result.Token - - cookies, xsrf := getCookiesAndCsrf(jupyterUrl) - if xsrf == "" { - log.Error("browser jupyterUrl failed:%v", task.DisplayJobName) - } else { - - codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath - fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) - if err != nil { - log.Error("read jupyter file failed:%v", task.DisplayJobName, err) - } - - base64Content := base64.StdEncoding.EncodeToString(fileContents) - client := getRestyClient() - uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile) - res, err := client.R(). - SetCookies(cookies). - SetHeader("X-XSRFToken", xsrf). - SetBody(map[string]interface{}{ - "type": "file", - "format": "base64", - "name": path.Base(task.BootFile), - "path": path.Base(task.BootFile), - "content": base64Content}). - Put(uploadUrl) - if err != nil { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } else if res.StatusCode() != http.StatusCreated { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } - - } - -} - -func getJupyterBaseUrl(url string) string { - jupyterUrlLength := len(url) - baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))] - return baseUrl -} - -func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) { - log.Info("jupyter url:" + jupyterUrl) - var cookies []*http.Cookie - const retryTimes = 10 - for i := 0; i < retryTimes; i++ { - res, err := http.Get(jupyterUrl) - if err != nil { - log.Error("browser jupyterUrl failed.", err) - if i == retryTimes-1 { - return cookies, "" - } - - } else { - cookies = res.Cookies() - xsrf := "" - for _, cookie := range cookies { - if cookie.Name == "_xsrf" { - xsrf = cookie.Value - break - } - - } - if xsrf != "" { - return cookies, xsrf - } - - } - } - return cookies, "" - -} - func SyncTempStatusJob() { jobs, err := models.GetCloudBrainTempJobs() if err != nil { diff --git a/modules/modelarts_cd/modelarts.go b/modules/modelarts_cd/modelarts.go index bdc42002a..9d74c0919 100755 --- a/modules/modelarts_cd/modelarts.go +++ b/modules/modelarts_cd/modelarts.go @@ -148,6 +148,7 @@ func GenerateNotebook(ctx *context.Context, req cloudbrain.GenerateModelArtsNote UpdatedUnix: createTime, Spec: req.Spec, BootFile: req.BootFile, + BranchName: req.BranchName, ModelName: req.ModelName, ModelVersion: req.ModelVersion, LabelName: req.LabelName, diff --git a/modules/notebook/contentManager.go b/modules/notebook/contentManager.go new file mode 100644 index 000000000..823e59665 --- /dev/null +++ b/modules/notebook/contentManager.go @@ -0,0 +1,198 @@ +package notebook + +import ( + "crypto/tls" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "path" + "strings" + + "github.com/go-resty/resty/v2" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/log" +) + +var restyClient *resty.Client + +type NotebookApiResponse struct { + Name string `json:"name"` + Path string `json:"path"` +} + +type NotebookContent struct { + Url string + Path string + Cookies []*http.Cookie + Xsrf string + PathType string //file directory + Token string +} + +func (c *NotebookContent) IsNotebookFileCanBrowser() bool { + if c.Xsrf == "" { + c.SetCookiesAndCsrf() + } + if c.Xsrf == "" { + log.Warn("xsrf is empty, can not broswer url:" + c.Url) + return false + } + return c.IsNoteBookContentsExist() + +} + +func (c *NotebookContent) SetCookiesAndCsrf() { + log.Info("jupyter url:" + c.Url) + var cookies []*http.Cookie + const retryTimes = 10 + url := c.Url + if c.Token != "" { + url = c.Url + "?token=" + c.Token + } + for i := 0; i < retryTimes; i++ { + res, err := http.Get(url) + if err != nil { + log.Error("browser jupyterUrl failed.", err) + if i == retryTimes-1 { + c.Cookies = cookies + } + + } else { + cookies = res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + if len(cookies) > 1 { + break + } + + } + + } + if xsrf != "" { + c.Cookies = cookies + c.Xsrf = xsrf + } + + } + } + c.Cookies = cookies + +} + +func (c *NotebookContent) IsNoteBookContentsExist() bool { + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + "?type=" + c.PathType + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + Get(uploadUrl) + if err != nil { + log.Warn("browser url error:"+uploadUrl, err) + return false + } + return res.StatusCode() == http.StatusOK +} + +func (c *NotebookContent) UploadNoteBookFile(task *models.Cloudbrain) error { + + err := c.MakeNoteBookDir() + if err != nil { + return err + } + + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath + fileContents, err := ioutil.ReadFile(codePath + "/" + c.Path) + if err != nil { + log.Error("read jupyter file failed:%v", task.DisplayJobName, err) + } + + base64Content := base64.StdEncoding.EncodeToString(fileContents) + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": "file", + "format": "base64", + "name": path.Base(c.Path), + "path": c.Path, + "content": base64Content}). + Put(uploadUrl) + if err != nil { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + return err + } else if res.StatusCode() != http.StatusCreated { + log.Error("upload jupyter file failed:%v, status is %s", task.DisplayJobName, res.Status(), err) + return fmt.Errorf("status:", res.StatusCode()) + } + return nil +} + +/** + if c.Path is a/b/c.txt + makedir a/b + if c.Path is a/b/c + makedir a/b +*/ +func (c *NotebookContent) MakeNoteBookDir() error { + filePaths := strings.Split(c.Path, "/") + + for i := 0; i < len(filePaths)-1; i++ { + cTemp := &NotebookContent{ + Url: c.Url, + Cookies: c.Cookies, + Path: path.Join(filePaths[0 : i+1]...), + PathType: "directory", + Xsrf: c.Xsrf, + } + if !cTemp.IsNoteBookContentsExist() { + + createTempDirUrl := getJupyterBaseUrl(cTemp.Url) + "api/contents/" + cTemp.Path + client := getRestyClient() + var jobResult NotebookApiResponse + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": cTemp.PathType, + "path": cTemp.Path, + }).SetResult(&jobResult). + Put(createTempDirUrl) + if err != nil { + return err + } + if res.StatusCode() != http.StatusCreated { + return fmt.Errorf("status code:" + res.Status()) + } + + } + + } + return nil +} + +func getJupyterBaseUrl(url string) string { + jupyterUrlLength := len(url) + baseUrl := url + if strings.HasSuffix(url, "lab") { + baseUrl = url[0 : jupyterUrlLength-len(path.Base(url))] + } + + return baseUrl +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + return restyClient +} diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 9ea5601c9..e4403ed04 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -47,6 +47,7 @@ type CreateFileNotebookJobOption struct { BranchName string `json:"branch_name" binding:"Required"` OwnerName string `json:"owner_name" binding:"Required"` ProjectName string `json:"project_name" binding:"Required"` + JobId string `json:"job_id"` } type Cloudbrain struct { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 702c17bce..f86879b06 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1061,7 +1061,7 @@ model_rename=Duplicate model name, please modify model name. notebook_file_not_exist=Notebook file does not exist. notebook_select_wrong=Please select a Notebook(.ipynb) file first. notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. -debug_again_fail=Fail to restart debug task, please try again later. +notebook_repo_conflict=The files in different branches of the same repository can not run together. date=Date repo_add=Project Increment diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index bf609ba14..cfd58d6b9 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1060,7 +1060,7 @@ model_rename=模型名称重复，请修改模型名称 notebook_file_not_exist=Notebook文件不存在。 notebook_select_wrong=请先选择Notebook（.ipynb）文件。 notebook_file_no_right=您没有这个Notebook文件的读权限。 -debug_again_fail=再次调试失败，请稍后再试。 +notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。 date=日期 repo_add=新增项目 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 4936c2362..6a8090cd4 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -745,7 +745,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/file_notebook", func() { m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) - + //m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) + m.Post("/status", bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) }) m.Group("/repos", func() { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 805443788..250a08423 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -110,6 +110,9 @@ func GeneralCloudBrainJobStop(ctx *context.APIContext) { func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { cloudbrainTask.FileNotebookCreate(ctx.Context, option) } +func FileNoteBookStatus(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { + cloudbrainTask.FileNotebookStatus(ctx.Context, option) +} func GetFileNoteBookInfo(ctx *context.APIContext) { //image description spec description waiting count diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 3dbe101a8..76de80f71 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -440,10 +440,9 @@ func NotebookDebug2(ctx *context.Context) { return } if task.BootFile != "" { - ctx.Redirect(getFileUrl(result.Url, task.BootFile) + "?token=" + result.Token) - } else { - ctx.Redirect(result.Url + "?token=" + result.Token) + go cloudbrainTask.UploadNotebookFiles(task) } + ctx.Redirect(result.Url + "?token=" + result.Token) } diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index cc9563520..766966843 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -4,6 +4,9 @@ import ( "fmt" "net/http" "path" + "strings" + + "code.gitea.io/gitea/modules/notebook" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts_cd" @@ -29,6 +32,9 @@ import ( ) const NoteBookExtension = ".ipynb" +const CPUType = 0 +const GPUType = 1 +const NPUType = 2 func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { @@ -66,7 +72,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } //create repo if not exist - repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) + repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ Name: setting.FileNoteBook.ProjectName, @@ -80,17 +86,200 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp AutoInit: true, DefaultBranch: "master", }) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + return + } + } else { + + noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type)) + if noteBook != nil { + + if isRepoConfilcts(option, noteBook) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict"))) + return + } + + if isNotebookSpecMath(option, noteBook) { + + err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) + noteBook.BranchName += ";" + option.BranchName + err := models.UpdateJob(noteBook) + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: noteBook.JobID, + }) + return + } + + } + } + + if option.Type <= GPUType { + cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + } else { + modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) } + +} +func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) { + if ctx.Written() { + return + } + + if path.Ext(option.File) != NoteBookExtension { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) + return + } + + isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) + if !isNotebookFileExist { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + task, err := models.GetCloudbrainByJobID(option.JobId) if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + log.Error("job not found:"+option.JobId, err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job.")) return } - if option.Type <= 1 { - cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) { + log.Warn("Boot file is empty or status is running. ") + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running.")) + return + } + if !isRepoFileMatch(option, task) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file.")) + return + } + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) { + ctx.JSON(http.StatusOK, models.BaseOKMessageApi) } else { - modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed.")) + + } + +} + +func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) { + var debugBaseUrl string + var token string + if task.Type == models.TypeCloudBrainOne { + debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab" + + } else { + var result *models.GetNotebook2Result + var err error + if task.Type == models.TypeCloudBrainTwo { + result, err = modelarts.GetNotebook2(task.JobID) + } else if task.Type == models.TypeCDCenter { + result, err = modelarts_cd.GetNotebook(task.JobID) + } + if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" { + log.Error("notebook job not found:"+task.JobID, err) + return "", "", fmt.Errorf("can not get job or job is invalid.") + } + + debugBaseUrl = result.Url + token = result.Token + + } + return debugBaseUrl, token, nil +} + +func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool { + c := ¬ebook.NotebookContent{ + Url: debugBaseUrl, + Path: bootFile, + PathType: "file", + Token: token, + } + if c.IsNotebookFileCanBrowser() { + return true + } else { + c.SetCookiesAndCsrf() + c.UploadNoteBookFile(task) + return c.IsNotebookFileCanBrowser() + } + +} + +func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + if option.Type == NPUType || option.Type == CPUType { + return true + } + spec, err := models.GetCloudbrainSpecByID(book.ID) + if err != nil { + log.Warn("can not get spec ", err) + return false + } + return spec.AccCardsNum > 0 +} + +func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName { + return true + } + } } + return false + +} + +func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile { + return true + } + } + + return false + +} +func UploadNotebookFiles(task *models.Cloudbrain) { + if task.Status == string(models.JobRunning) && task.BootFile != "" { + + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + log.Error("can not get base url:", err) + return + } + bootFiles := strings.Split(task.BootFile, ";") + + for _, bootFile := range bootFiles { + uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token) + } + + } } func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { @@ -131,17 +320,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return } } - errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) - if errStr != "" { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return } command := cloudbrain.GetCloudbrainDebugCommand() @@ -185,7 +375,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot JobType: jobType, Description: getDescription(option), BranchName: option.BranchName, - BootFile: option.File, + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), Params: "{\"parameter\":[]}", CommitID: "", BenchmarkTypeID: 0, @@ -206,8 +396,15 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } -func getCodePath(jobName string) string { - return setting.JobPath + jobName + cloudbrain.CodeMountPath +func getCloudbrainType(optionType int) int { + if optionType < 1 { + return models.TypeCloudBrainOne + } + return models.TypeCloudBrainTwo +} + +func getCodePath(jobName string, repo *models.Repository) string { + return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name } func getDescription(option api.CreateFileNotebookJobOption) string { @@ -237,8 +434,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return @@ -260,7 +457,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } } - err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return @@ -299,6 +496,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote Spec: spec, BootFile: "", AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, + BranchName: option.BranchName, } if setting.ModelartsCD.Enabled { @@ -347,17 +545,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO return true, nil } -func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { - err := downloadCode(repo, codePath, branchName) - if err != nil { - return "cloudbrain.load_code_failed" - } - - err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") - if err != nil { - return "cloudbrain.load_code_failed" - } - return "" +func getBootFile(filePath string, ownerName string, projectName string) string { + return ownerName + "/" + projectName + "/" + filePath } func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { From 3862ba6f0d0c18981f860fdf02979e535b40f3c4 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 15:10:21 +0800 Subject: [PATCH 25/85] fix issue --- options/locale/locale_en-US.ini | 4 ++-- options/locale/locale_zh-CN.ini | 4 ++-- templates/custom/task_wait_count.tmpl | 2 +- templates/repo/cloudbrain/trainjob/show.tmpl | 10 ++++++++++ templates/repo/grampus/trainjob/show.tmpl | 11 +++++++++++ templates/repo/modelarts/trainjob/show.tmpl | 11 +++++++++++ 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 702c17bce..84fd12d16 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1091,8 +1091,8 @@ image_delete_fail=Failed to delete image, please try again later. image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image? download=Download score=Score -wait_count_start = There are currently -wait_count_end = tasks queued +wait_count_start = Your current queue position is +wait_count_end = file_limit_100 = Display up to 100 files or folders in a single directory images.name = Image Tag images.name_placerholder = Please enter the image name diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index bf609ba14..dc85c5d27 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1090,8 +1090,8 @@ image_delete_fail=删除镜像失败，请稍后再试。 image_overwrite=您已经提交过相同名称的镜像，您确定要覆盖原来提交的镜像吗？ download=模型下载 score=评分 -wait_count_start = 当前有 -wait_count_end = 个任务正在排队 +wait_count_start = 您当前排队位置是第 +wait_count_end = 位 file_limit_100 = 单目录下最多显示100个文件或文件夹 images.name = 镜像Tag images.name_placerholder = 请输入镜像Tag diff --git a/templates/custom/task_wait_count.tmpl b/templates/custom/task_wait_count.tmpl index fb8ee71fb..4791327c5 100644 --- a/templates/custom/task_wait_count.tmpl +++ b/templates/custom/task_wait_count.tmpl @@ -12,7 +12,7 @@ var specsSelEl = $('select#__specs__'); var seldOption = specsSelEl.find('option:selected'); var queueCode = seldOption.attr('queueCode'); - $('span.__task_wait_count__').text(queuesDetail[queueCode] || 0); + $('span.__task_wait_count__').text(queuesDetail[queueCode]+1); }; $('body').on('change', 'select#__specs__', function(e) { changeSpecs(); diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 2a46d747e..6a83d3117 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -123,7 +123,17 @@
+ + + {{$.i18n.Tr "repo.cloudbrain_creator"}} + + +
+ {{.User.Name}} +
+ + {{$.i18n.Tr "repo.modelarts.train_job.start_time"}} diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 608bfd875..df8fbdb0e 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -123,6 +123,17 @@ + {{$.i18n.Tr "repo.cloudbrain_creator"}} + + + +
+ {{.User.Name}} +
+ + + + {{$.i18n.Tr "repo.modelarts.run_version"}} diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index a8ac1c4bf..57e97ba5c 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -159,6 +159,17 @@ + {{$.i18n.Tr "repo.cloudbrain_creator"}} + + + +
+ {{.User.Name}} +
+ + + + {{$.i18n.Tr "repo.modelarts.run_version"}} From 63f374b0199cbc5159c09565f59f7c2c116f9359 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 15:36:28 +0800 Subject: [PATCH 26/85] fix issue --- routers/repo/grampus.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 14db1a50d..8f0871581 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -1239,7 +1239,7 @@ func GrampusTrainJobShow(ctx *context.Context) { return } task.ContainerIp = "" - + task.User, _ = models.GetUserByID(task.UserID) if task.DeletedAt.IsZero() { //normal record result, err := grampus.GetJob(task.JobID) if err != nil { @@ -1308,6 +1308,7 @@ func GrampusTrainJobShow(ctx *context.Context) { taskList := make([]*models.Cloudbrain, 0) taskList = append(taskList, task) prepareSpec4Show(ctx, task) + ctx.Data["version_list_task"] = taskList ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false) ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) From a88ab8a2ac434eb89cd24c10a141f8d286a09565 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 15:39:27 +0800 Subject: [PATCH 27/85] fix issue --- models/user.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/user.go b/models/user.go index dad252d92..cb57b05e5 100755 --- a/models/user.go +++ b/models/user.go @@ -495,6 +495,9 @@ func (u *User) RealSizedAvatarLink(size int) string { // may either be a sub-URL to this site, or a full URL to an external avatar // service. func (u *User) RelAvatarLink() string { + if u.Avatar != "" { + return u.CustomAvatarPath() + } return u.SizedRelAvatarLink(base.DefaultAvatarSize) } From 5ebd80c32ef7b37b553a0e4293883c0a71ceef48 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 21 Dec 2022 15:49:43 +0800 Subject: [PATCH 28/85] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 3 +- routers/api/v1/api.go | 3 +- routers/repo/cloudbrain.go | 7 ++-- routers/repo/modelarts.go | 13 ++++--- services/cloudbrain/cloudbrainTask/notebook.go | 50 ++++++++++++++++++-------- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 099f07c3a..f94969605 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2128,7 +2128,8 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { } func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) { cloudBrain := new(Cloudbrain) - has, err := x.In("status", JobWaiting, JobRunning).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) + has, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, + ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) if has { return cloudBrain, err } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6a8090cd4..cae6c01bf 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -745,8 +745,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/file_notebook", func() { m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) - //m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) - m.Post("/status", bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) + m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) }) m.Group("/repos", func() { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 905c25a64..3b45e3074 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -911,10 +911,13 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile != "" { - ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(debugUrl, ctx.QueryTrim("file"))) } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } ctx.Redirect(debugUrl) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 76de80f71..2d555fc9a 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -439,10 +439,15 @@ func NotebookDebug2(ctx *context.Context) { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) return } - if task.BootFile != "" { - go cloudbrainTask.UploadNotebookFiles(task) + + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(result.Url, ctx.QueryTrim("file")) + "?token=" + result.Token) + } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } + ctx.Redirect(result.Url + "?token=" + result.Token) } - ctx.Redirect(result.Url + "?token=" + result.Token) } @@ -629,7 +634,7 @@ func NotebookStop(ctx *context.Context) { if task.Status != string(models.ModelArtsRunning) { log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" - errorMsg = "the job is not running" + errorMsg = ctx.Tr("cloudbrain.Already_stopped") break } diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 766966843..4a06bbc79 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -101,20 +101,25 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } if isNotebookSpecMath(option, noteBook) { - - err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) - if err != nil { - log.Error("download code failed", err) - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) - return + if !isRepoMatch(option, noteBook) { + err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } } - noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) - noteBook.BranchName += ";" + option.BranchName - err := models.UpdateJob(noteBook) - if err != nil { - log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) - return + if !isRepoFileMatch(option, noteBook) { + noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) + noteBook.BranchName += ";" + option.BranchName + noteBook.Description += ";" + getDescription(option) + + err := models.UpdateJob(noteBook) + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } } ctx.JSON(http.StatusOK, models.BaseMessageApi{ @@ -252,6 +257,21 @@ func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudb } +func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + + for _, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName { + return true + } + } + } + return false + +} + func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { bootFiles := strings.Split(book.BootFile, ";") branches := strings.Split(book.BranchName, ";") @@ -494,9 +514,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote Description: getDescription(option), ImageId: setting.FileNoteBook.ImageIdNPU, Spec: spec, - BootFile: "", + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, - BranchName: option.BranchName, + BranchName: option.BranchName, } if setting.ModelartsCD.Enabled { From 437b465f316f7872a56835350a0da047ddd44295 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 21 Dec 2022 15:50:48 +0800 Subject: [PATCH 29/85] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=E7=94=A8=E6=88=B7=E5=9B=BE=E5=83=8F?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user.go | 7 ++++++- routers/repo/grampus.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/models/user.go b/models/user.go index dad252d92..91bbe8671 100755 --- a/models/user.go +++ b/models/user.go @@ -16,6 +16,7 @@ import ( "fmt" _ "image/jpeg" // Needed for jpeg support "image/png" + "math/rand" "os" "path/filepath" "regexp" @@ -495,7 +496,11 @@ func (u *User) RealSizedAvatarLink(size int) string { // may either be a sub-URL to this site, or a full URL to an external avatar // service. func (u *User) RelAvatarLink() string { - return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append := "" + if u.Avatar != ""{ + append = "?" + fmt.Sprint(rand.Intn(100)) + } + return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append } // AvatarLink returns user avatar absolute link. diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 14db1a50d..fac4c5c36 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -1239,7 +1239,7 @@ func GrampusTrainJobShow(ctx *context.Context) { return } task.ContainerIp = "" - + task.User, _ = models.GetUserByID(task.UserID) if task.DeletedAt.IsZero() { //normal record result, err := grampus.GetJob(task.JobID) if err != nil { From d0d8bad140f6ec7762a88c786c21cf02c35ebc17 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 21 Dec 2022 15:55:31 +0800 Subject: [PATCH 30/85] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=E7=94=A8=E6=88=B7=E5=9B=BE=E5=83=8F?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user.go b/models/user.go index 91bbe8671..d34860f2d 100755 --- a/models/user.go +++ b/models/user.go @@ -497,7 +497,7 @@ func (u *User) RealSizedAvatarLink(size int) string { // service. func (u *User) RelAvatarLink() string { append := "" - if u.Avatar != ""{ + if u.UseCustomAvatar { append = "?" + fmt.Sprint(rand.Intn(100)) } return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append From 8c45b5865c50b5bc31d19bc5a2e6d021fd604d62 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 16:05:56 +0800 Subject: [PATCH 31/85] fix issue --- models/user.go | 3 --- templates/repo/cloudbrain/inference/new.tmpl | 2 +- templates/repo/cloudbrain/new.tmpl | 3 ++- templates/repo/cloudbrain/trainjob/new.tmpl | 2 +- templates/repo/grampus/notebook/gpu/new.tmpl | 2 +- templates/repo/modelsafety/new.tmpl | 2 +- web_src/js/features/cloudrbanin.js | 37 ---------------------------- 7 files changed, 6 insertions(+), 45 deletions(-) diff --git a/models/user.go b/models/user.go index cb57b05e5..dad252d92 100755 --- a/models/user.go +++ b/models/user.go @@ -495,9 +495,6 @@ func (u *User) RealSizedAvatarLink(size int) string { // may either be a sub-URL to this site, or a full URL to an external avatar // service. func (u *User) RelAvatarLink() string { - if u.Avatar != "" { - return u.CustomAvatarPath() - } return u.SizedRelAvatarLink(base.DefaultAvatarSize) } diff --git a/templates/repo/cloudbrain/inference/new.tmpl b/templates/repo/cloudbrain/inference/new.tmpl index feeac67cb..cab2035bc 100644 --- a/templates/repo/cloudbrain/inference/new.tmpl +++ b/templates/repo/cloudbrain/inference/new.tmpl @@ -40,7 +40,7 @@ {{template "base/alert" .}} {{end}} {{template "custom/alert_cb" .}} - +

{{.i18n.Tr "repo.modelarts.train_job.new_infer"}}

diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index b925f0e1b..e1ef73c85 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -23,7 +23,7 @@
{{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -38,6 +38,7 @@
{{.CsrfTokenHtml}} +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index c572f72a6..804f2bde5 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -56,7 +56,7 @@
{{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} diff --git a/templates/repo/grampus/notebook/gpu/new.tmpl b/templates/repo/grampus/notebook/gpu/new.tmpl index 4a752ec26..73878bf7b 100644 --- a/templates/repo/grampus/notebook/gpu/new.tmpl +++ b/templates/repo/grampus/notebook/gpu/new.tmpl @@ -5,7 +5,7 @@ {{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} diff --git a/templates/repo/modelsafety/new.tmpl b/templates/repo/modelsafety/new.tmpl index d261c4dde..4fee33745 100644 --- a/templates/repo/modelsafety/new.tmpl +++ b/templates/repo/modelsafety/new.tmpl @@ -49,7 +49,7 @@ {{template "repo/header" .}}
{{$Grampus := (or (eq (index (SubJumpablePath .Link) 1) "create_grampus_gpu") (eq (index (SubJumpablePath .Link) 1) "create_grampus_npu"))}} - + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} diff --git a/web_src/js/features/cloudrbanin.js b/web_src/js/features/cloudrbanin.js index 840ea4940..775f63133 100644 --- a/web_src/js/features/cloudrbanin.js +++ b/web_src/js/features/cloudrbanin.js @@ -483,43 +483,6 @@ export default async function initCloudrain() { const redirect_to = this.dataset.linkpath; debugAgain(ID, repoPath, redirect_to); }); - - function setWaitNums() { - if ($(".cloudbrain-type").length === 0 && $(".gpu-type").length === 0) { - return; - } - if ( - $(".cloudbrain-type").length !== 0 && - !$(".cloudbrain-type").data("queue") - ) { - return; - } - let waitNums = $(".cloudbrain-type").data("queue").split("map")[1]; - let test = new Map(); - let waitNumsArray = waitNums.split(" "); - waitNumsArray.forEach((element, index) => { - if (index === 0) { - test.set(element.slice(1, -2), parseInt(element.slice(-1))); - } else if (index === waitNumsArray.length - 1) { - test.set(element.slice(0, -3), parseInt(element.slice(-2, -1))); - } else { - test.set(element.slice(0, -2), parseInt(element.slice(-1))); - } - }); - $(".ui.search.dropdown.gpu-type").dropdown({ - onChange: function (value, text, $selectedItem) { - let gpuTypeNums = test.get(value); - let gpuTypeNumString = - $(".cloudbrain-type").data("queue-start") + - " " + - gpuTypeNums + - " " + - $(".cloudbrain-type").data("queue-end"); - $("#gpu-nums").text(gpuTypeNumString); - }, - }); - } - setWaitNums(); } function userSearchControll() { From 79ebde98de94b5f0e3da864b7b6a7eb6e77f25c3 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 16:27:52 +0800 Subject: [PATCH 32/85] fix issue --- templates/repo/grampus/notebook/gpu/new.tmpl | 5 ++--- templates/repo/grampus/notebook/npu/new.tmpl | 17 +++++++++++++++-- templates/repo/modelarts/notebook/new.tmpl | 19 ++++++++++++++++--- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/templates/repo/grampus/notebook/gpu/new.tmpl b/templates/repo/grampus/notebook/gpu/new.tmpl index 73878bf7b..00d4f27d8 100644 --- a/templates/repo/grampus/notebook/gpu/new.tmpl +++ b/templates/repo/grampus/notebook/gpu/new.tmpl @@ -4,8 +4,7 @@
{{template "repo/header" .}}
- - + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -20,8 +19,8 @@ {{.CsrfTokenHtml}} +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

-
diff --git a/templates/repo/grampus/notebook/npu/new.tmpl b/templates/repo/grampus/notebook/npu/new.tmpl index 0773cc346..f14f3618d 100644 --- a/templates/repo/grampus/notebook/npu/new.tmpl +++ b/templates/repo/grampus/notebook/npu/new.tmpl @@ -93,8 +93,21 @@
diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 1b3ea630a..2013be92b 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -4,7 +4,7 @@ {{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -77,8 +77,21 @@
From 31aa33f9211864248de64cba04845b8b29bea9fd Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 16:31:23 +0800 Subject: [PATCH 33/85] fix issue --- templates/repo/modelarts/notebook/new.tmpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 2013be92b..34c86610f 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -79,18 +79,18 @@ From 98e0a770d4a95e2370747d9d11aa1cab7eadcda5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 16:47:28 +0800 Subject: [PATCH 34/85] fix issue --- modules/auth/modelarts.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index 0061648ce..874bbb0a4 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -28,6 +28,7 @@ type CreateModelArtsNotebookForm struct { LabelName string `form:"label_names"` PreTrainModelUrl string `form:"pre_train_model_url"` SpecId int64 `form:"spec_id" binding:"Required"` + DatasetName string `form:"dataset_name"` } func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { From b2265d00a1e63c777f9742a1ece99f4f7a0e9831 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 16:50:44 +0800 Subject: [PATCH 35/85] fix issue --- templates/repo/grampus/notebook/npu/new.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/grampus/notebook/npu/new.tmpl b/templates/repo/grampus/notebook/npu/new.tmpl index f14f3618d..9c52e2717 100644 --- a/templates/repo/grampus/notebook/npu/new.tmpl +++ b/templates/repo/grampus/notebook/npu/new.tmpl @@ -3,7 +3,7 @@
{{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} From b0f18ff074e744fa37c8519a38299bd9ca8f2652 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 21 Dec 2022 17:35:47 +0800 Subject: [PATCH 36/85] =?UTF-8?q?=E6=8F=90=E4=BA=A4diam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 7508b70d3..6608736e8 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2148,7 +2148,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset), string(models.JobTypeModelSafety)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, From 671b55c68965c31141e5640fb2a18955d30aa842 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 17:55:02 +0800 Subject: [PATCH 37/85] fix issue --- templates/repo/modelsafety/new.tmpl | 35 +----- web_src/js/features/cloudbrainShow.js | 196 --------------------------------- web_src/js/standalone/cloudbrainNew.js | 195 ++++++++++++++++++++++++++++++++ 3 files changed, 196 insertions(+), 230 deletions(-) diff --git a/templates/repo/modelsafety/new.tmpl b/templates/repo/modelsafety/new.tmpl index 4fee33745..e259054b5 100644 --- a/templates/repo/modelsafety/new.tmpl +++ b/templates/repo/modelsafety/new.tmpl @@ -141,40 +141,7 @@ onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)">{{.description}}
- -
- -
- -
- -
- -
-
- -
- - - -
+ {{template "custom/select_model" .}}