From 32bd6e8a858ec0eee1e729017c78b28d5ab7b91f Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 24 Nov 2021 19:07:17 +0800 Subject: [PATCH 01/10] fix bug --- routers/repo/modelarts.go | 102 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 12 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index e35cf5c6b..83498b70e 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -380,6 +380,85 @@ func trainJobNewDataPrepare(ctx *context.Context) error { return nil } +func ErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { + ctx.Data["PageIsCloudBrain"] = true + + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //if !can { + // log.Error("the user can not create train-job") + // ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) + // return + //} + + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return err + } + ctx.Data["attachments"] = attachs + + var resourcePools modelarts.ResourcePool + if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engine_versions"] = versionInfos.Version + + var flavorInfos modelarts.Flavor + if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["flavor_infos"] = flavorInfos.Info + + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + ctx.Data["train_url"] = outputObsPath + + Branches, err := ctx.Repo.GitRepo.GetBranches() + if err != nil { + ctx.ServerError("GetBranches error:", err) + return err + } + ctx.Data["Branches"] = Branches + ctx.Data["BranchesCount"] = len(Branches) + + configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) + if err != nil { + ctx.ServerError("getConfigList failed:", err) + return err + } + ctx.Data["config_list"] = configList.ParaConfigs + ctx.Data["bootFile"] = form.BootFile + ctx.Data["uuid"] = form.Attachment + ctx.Data["params"] = form.Params + ctx.Data["branch_name"] = form.BranchName + + return nil +} + func TrainJobNewVersion(ctx *context.Context) { err := trainJobNewVersionDataPrepare(ctx) if err != nil { @@ -477,7 +556,7 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { return nil } -func ErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { +func VersionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") // var versionName = ctx.Params(":version-name") @@ -580,7 +659,6 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath - // codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" @@ -593,7 +671,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - trainJobNewDataPrepare(ctx) + ErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) return } @@ -792,7 +870,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } @@ -815,7 +893,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ Branch: branch_name, }); err != nil { log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobVersionNew, &form) return } @@ -823,14 +901,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ //todo: upload code (send to file_server todo this work?) if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobVersionNew, &form) return } if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobVersionNew, &form) return } @@ -839,7 +917,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ // if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobVersionNew, &form) return } @@ -859,7 +937,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ err := json.Unmarshal([]byte(params), ¶meters) if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", params, err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobVersionNew, &form) return } @@ -878,7 +956,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if isSaveParam == "on" { if form.ParameterTemplateName == "" { log.Error("ParameterTemplateName is empty") - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数时，作业参数名称不能为空", tplModelArtsTrainJobVersionNew, &form) return } @@ -902,7 +980,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if err != nil { log.Error("Failed to CreateTrainJobConfig: %v", err) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobVersionNew, &form) return } @@ -949,7 +1027,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ err = modelarts.GenerateTrainJobVersion(ctx, req, jobID) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) - ErrorDataPrepare(ctx, form) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } From 44fcae12c1f7d5ce58beca78836f956559e99ae5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 24 Nov 2021 19:20:59 +0800 Subject: [PATCH 02/10] fix issue --- templates/repo/cloudbrain/index.tmpl | 12 +++---- templates/repo/header.tmpl | 4 +-- templates/repo/modelarts/notebook/index.tmpl | 10 +++--- templates/repo/modelarts/trainjob/index.tmpl | 13 +++---- templates/repo/modelarts/trainjob/new.tmpl | 41 +++++++++++++++------- templates/repo/modelarts/trainjob/para_manage.tmpl | 2 +- templates/repo/modelarts/trainjob/show.tmpl | 15 ++++---- templates/repo/modelarts/trainjob/version_new.tmpl | 24 ++++++------- web_src/less/openi.less | 19 ++++++++++ 9 files changed, 87 insertions(+), 53 deletions(-) diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index d72ffb0c4..097758c8f 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -307,9 +307,9 @@
- - {{svg "octicon-tasklist" 16}} - {{.JobName}} + + + {{.JobName}}
@@ -380,7 +380,7 @@ {{end}} -
+
{{$.i18n.Tr "repo.download"}} @@ -399,11 +399,11 @@
{{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.delete"}} {{else}} - + {{$.i18n.Tr "repo.delete"}} {{end}} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index d5b911a79..bfa373365 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -93,7 +93,7 @@
{{if .Permission.CanRead $.UnitTypeCode}}
- + {{svg "octicon-code" 16}} {{.i18n.Tr "repo.code"}}
@@ -243,7 +243,7 @@ }) $('.question.circle.icon').hover(function(){ $(this).popup('show') - $('.ui.popup.mini.top.center').css({"border-color":'rgba(50, 145, 248, 100)',"color":"rgba(3, 102, 214, 100)","border-radius":"5px"}) + $('.ui.popup.mini.top.center').css({"border-color":'rgba(50, 145, 248, 100)',"color":"rgba(3, 102, 214, 100)","border-radius":"5px","border-shadow":"none"}) }); \ No newline at end of file diff --git a/templates/repo/modelarts/notebook/index.tmpl b/templates/repo/modelarts/notebook/index.tmpl index 021394a07..425c3b199 100755 --- a/templates/repo/modelarts/notebook/index.tmpl +++ b/templates/repo/modelarts/notebook/index.tmpl @@ -280,9 +280,9 @@
- - {{svg "octicon-tasklist" 16}} - {{.JobName}} + + + {{.JobName}}
@@ -352,11 +352,11 @@ {{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.delete"}} {{else}} - + {{$.i18n.Tr "repo.delete"}} {{end}} diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index c480e9048..58d671b9f 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -321,9 +321,9 @@
- - {{svg "octicon-tasklist" 16}} - {{.JobName}} + + + {{.JobName}}
@@ -367,7 +367,8 @@
- + + {{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} @@ -390,11 +391,11 @@ {{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.delete"}} {{else}} - + {{$.i18n.Tr "repo.delete"}} {{end}} diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index c91d67649..472abe8a2 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -159,12 +159,12 @@

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

- +
- +
@@ -173,7 +173,7 @@
- + {{range .engines}} @@ -205,7 +205,7 @@
- + {{if .bootFile}} {{else}} @@ -216,7 +216,7 @@
- + -
+
+ {{if ne 0 (len .params)}} + {{range $k ,$v := .params}} +
+
+ +
+
+ +
+ + + + +
+ {{end}} + {{end}} +
- +
diff --git a/templates/repo/modelarts/trainjob/para_manage.tmpl b/templates/repo/modelarts/trainjob/para_manage.tmpl index b8ac6c78a..e7e48f13b 100755 --- a/templates/repo/modelarts/trainjob/para_manage.tmpl +++ b/templates/repo/modelarts/trainjob/para_manage.tmpl @@ -38,7 +38,7 @@
- {{svg "octicon-tasklist" 16}} + {{.JobName}}
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index f703270ad..6b22a8412 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -226,9 +226,9 @@ td, th {
-
+
-
+
{{$.i18n.Tr "repo.modelarts.train_job.config"}} {{$.i18n.Tr "repo.modelarts.log"}} @@ -414,17 +414,14 @@ td, th { -
+ - +
-
-
+
--> +
diff --git a/templates/repo/modelarts/trainjob/version_new.tmpl b/templates/repo/modelarts/trainjob/version_new.tmpl index 05d4b92fc..51f468c4d 100644 --- a/templates/repo/modelarts/trainjob/version_new.tmpl +++ b/templates/repo/modelarts/trainjob/version_new.tmpl @@ -161,18 +161,18 @@

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

- +
- +
- +
@@ -181,7 +181,7 @@
- + {{range .engines}} @@ -224,7 +224,7 @@
- + {{if .boot_file}} {{else}} @@ -235,7 +235,7 @@
- +
@@ -275,7 +275,7 @@
- +
diff --git a/web_src/less/openi.less b/web_src/less/openi.less index e073dca0a..67707020e 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -291,9 +291,28 @@ footer .column{margin-bottom:0!important; padding-bottom:0!important;} display: inline-block; margin-top: 4px; } +.hover_active{ + +} .dropdown-menu:hover .dropdown-content { display: block; } +.dropdown-menu:hover .hover_active { + background: #fff !important; + border-color: #d4d4d5 !important; + font-weight: 700 !important; + // margin-bottom: -1px; + -webkit-box-shadow: none !important; + box-shadow: none !important; + border-radius: .28571429rem .28571429rem 0 0!important; + color: rgba(0,0,0,.95); + border-top-width: 1px; + +} + +.dropdown-menu:hover .dropdown-content { + display: block; +} .dropdown-content{ display: none; position: absolute; From c7a0e4da440f564725ae971fa015ce036c45ec66 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 24 Nov 2021 19:38:31 +0800 Subject: [PATCH 03/10] fix issue --- templates/repo/modelarts/trainjob/new.tmpl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 472abe8a2..cb303ec4f 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -175,6 +175,9 @@
- {{if ne 0 (len .params)}} +
From 45db7db86d592579d864d7fd325edb7c4219ec0f Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 24 Nov 2021 19:56:48 +0800 Subject: [PATCH 04/10] fix issue --- templates/repo/modelarts/trainjob/index.tmpl | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 58d671b9f..01c8b9d3d 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -366,7 +366,7 @@
-
+ +
+ {{$.CsrfTokenHtml}} + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.stop"}} + + {{else}} + + {{$.i18n.Tr "repo.stop"}} + + {{end}} + +
{{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} @@ -590,7 +603,17 @@ modal.style.display = "none"; } } - + function stopVersion(version_name){ + + $.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/stop_version`,{version_name:version_name},(data)=>{ + if(data.StatusOK===0){ + $('#'+version_name+'-stop').addClass('disabled') + refreshStatus(version_name) + } + }).fail(function(err) { + console.log(err); + }); + } // 显示弹窗，弹出相应的信息 function showmask() { $('#imageModal').css('display', 'none') From c24df8fc36b17ae6f1a442d06974b23dd6518316 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 24 Nov 2021 20:03:17 +0800 Subject: [PATCH 05/10] fix issue --- templates/repo/modelarts/trainjob/index.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 01c8b9d3d..e55131bca 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -391,7 +391,7 @@
{{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.stop"}} {{else}} @@ -603,9 +603,9 @@ modal.style.display = "none"; } } - function stopVersion(version_name){ + function stopVersion(version_name,jobID){ - $.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/stop_version`,{version_name:version_name},(data)=>{ + $.post(`/api/v1/repos{{$.Link}}/${.JobID}/stop_version`,{version_name:version_name},(data)=>{ if(data.StatusOK===0){ $('#'+version_name+'-stop').addClass('disabled') refreshStatus(version_name) From 7354af3bfabe9b177d07ca24d1f295be826285bb Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 24 Nov 2021 20:44:42 +0800 Subject: [PATCH 06/10] fix issue --- templates/repo/modelarts/trainjob/index.tmpl | 57 ++++++++++++---------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index e55131bca..e1e878a9a 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -391,7 +391,7 @@
{{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.stop"}} {{else}} @@ -404,11 +404,11 @@ {{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.delete"}} {{else}} - + {{$.i18n.Tr "repo.delete"}} {{end}} @@ -541,33 +541,7 @@ $('#' + jobID+ '-text').text(status) } - if(status==="RUNNING"){ - $('#model-debug-'+jobID).removeClass('disabled') - $('#model-debug-'+jobID).addClass('blue') - // $('#duration-'+jobID).text(duration) - - } - if(status!=="RUNNING"){ - $('#model-debug-'+jobID).removeClass('blue') - $('#model-debug-'+jobID).addClass('disabled') - - } - if(status!=="KILLED" || status!=="FAILED"){ - $('#stop-model-debug-'+jobID).removeClass('disabled') - $('#stop-model-debug-'+jobID).addClass('blue') - $('#model-delete-'+jobID).removeClass('red') - $('#model-delete-'+jobID).addClass('disabled') - } - if(status=="KILLED" || status=="FAILED" || status=="KILLING" || status=="COMPLETED"){ - $('#stop-model-debug-'+jobID).removeClass('blue') - $('#stop-model-debug-'+jobID).addClass('disabled') - $('#model-delete-'+jobID).removeClass('disabled') - $('#model-delete-'+jobID).addClass('red') - } - if(status=="START_FAILED"){ - $('#stop-model-debug-'+jobID).removeClass('blue') - $('#stop-model-debug-'+jobID).addClass('disabled') - } + }).fail(function(err) { console.log(err); }); @@ -605,15 +579,34 @@ } function stopVersion(version_name,jobID){ - $.post(`/api/v1/repos{{$.Link}}/${.JobID}/stop_version`,{version_name:version_name},(data)=>{ + const url = '/api/v1/repos/{{$.RepoRelPath}}/modelarts/train-job/'+jobID+'/stop_version' + $.post(url,{version_name:version_name},(data)=>{ if(data.StatusOK===0){ + $('#'+version_name+'-stop').removeClass('blue') $('#'+version_name+'-stop').addClass('disabled') - refreshStatus(version_name) + refreshStatus(version_name,jobID) } }).fail(function(err) { console.log(err); }); } + function refreshStatus(version_name,jobID){ + + const url = '/api/v1/repos/{{$.RepoRelPath}}/modelarts/train-job/'+jobID+'?version_name='+version_name + $.get(url,(data)=>{ + + + $(`#${jobID}-icon`).attr("class",data.JobStatus) + // detail status and duration + + $(`#${jobID}-text`).text(data.JobStatus) + + + }).fail(function(err) { + console.log(err); + }); + + } // 显示弹窗，弹出相应的信息 function showmask() { $('#imageModal').css('display', 'none') From 04c3ea18c28b62849748345fca4e0cc950e1e5f4 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 24 Nov 2021 20:46:37 +0800 Subject: [PATCH 07/10] fix bug --- modules/storage/obs.go | 4 ++-- routers/api/v1/repo/modelarts.go | 3 ++- routers/repo/modelarts.go | 9 ++++++++- templates/repo/modelarts/trainjob/new.tmpl | 4 ++-- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 9ca26b357..39312bfde 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -176,10 +176,10 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } -func GetObsListObject(jobName, parentDir string, versionOutputPath string) ([]FileInfo, error) { +func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, versionOutputPath, parentDir), "/") + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") strPrefix := strings.Split(input.Prefix, "/") output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 9637bae99..c53e62efa 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -303,7 +303,8 @@ func ModelList(ctx *context.APIContext) { return } VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) - models, err := storage.GetObsListObject(task.JobName, parentDir, VersionOutputPath) + parentDir = VersionOutputPath + "/" + parentDir + models, err := storage.GetObsListObject(task.JobName, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) ctx.ServerError("GetObsListObject:", err) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 83498b70e..b0ef88ebc 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -369,6 +369,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { } ctx.Data["Branches"] = Branches ctx.Data["BranchesCount"] = len(Branches) + ctx.Data["params"] = "" configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { @@ -450,16 +451,22 @@ func ErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJob ctx.ServerError("getConfigList failed:", err) return err } + var Parameters modelarts.Parameters + if err = json.Unmarshal([]byte(form.Params), &Parameters); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["params"] = Parameters.Parameter ctx.Data["config_list"] = configList.ParaConfigs ctx.Data["bootFile"] = form.BootFile ctx.Data["uuid"] = form.Attachment - ctx.Data["params"] = form.Params ctx.Data["branch_name"] = form.BranchName return nil } func TrainJobNewVersion(ctx *context.Context) { + err := trainJobNewVersionDataPrepare(ctx) if err != nil { ctx.ServerError("get new train-job info failed", err) diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index cb303ec4f..8513ee9ec 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -236,7 +236,7 @@ {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}
- + {{end}}
From 63ffb73ecad23c83104e7a5a6fc8d1eead03a31d Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 24 Nov 2021 22:28:02 +0800 Subject: [PATCH 08/10] fix bug --- modules/storage/obs.go | 47 ++++++++++++++++++++++++++-- routers/api/v1/repo/modelarts.go | 67 +++++++++++++++++++++++++++++++++++++--- routers/repo/modelarts.go | 21 ++++++++++++- 3 files changed, 128 insertions(+), 7 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 39312bfde..540463e80 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -176,10 +176,10 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } -func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { +func GetObsListObject(jobName, parentDir, versionName string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, versionName, parentDir), "/") strPrefix := strings.Split(input.Prefix, "/") output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) @@ -275,8 +275,34 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) log.Error("CreateSignedUrl failed:", err.Error()) return "", err } + log.Info("SignedUrl:%s", output.SignedUrl) + return output.SignedUrl, nil +} + +func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { + input := &obs.CreateSignedUrlInput{} + input.Bucket = bucket + input.Key = key + + input.Expires = 60 * 60 + input.Method = obs.HttpMethodGet + comma := strings.LastIndex(key, "/") + filename := key + if comma != -1 { + filename = key[comma+1:] + } + reqParams := make(map[string]string) + filename = url.QueryEscape(filename) + reqParams["response-content-disposition"] = "attachment; filename=\"" + filename + "\"" + input.QueryParams = reqParams + output, err := ObsCli.CreateSignedUrl(input) + if err != nil { + log.Error("CreateSignedUrl failed:", err.Error()) + return "", err + } return output.SignedUrl, nil + } func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { @@ -311,3 +337,20 @@ func ObsCreateObject(path string) error { return nil } + +func ObsDownloadAFile(bucket string, key string) (io.ReadCloser, error) { + input := &obs.GetObjectInput{} + input.Bucket = bucket + input.Key = key + output, err := ObsCli.GetObject(input) + if err == nil { + log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", + output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) + return output.Body, nil + } else if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + return nil, obsError + } else { + return nil, err + } +} diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index c53e62efa..568464f77 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -7,6 +7,7 @@ package repo import ( "net/http" + "path" "strconv" "strings" @@ -14,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -302,9 +304,7 @@ func ModelList(ctx *context.APIContext) { log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) return } - VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) - parentDir = VersionOutputPath + "/" + parentDir - models, err := storage.GetObsListObject(task.JobName, parentDir) + models, err := storage.GetObsListObject(task.JobName, parentDir, versionName) if err != nil { log.Info("get TrainJobListModel failed:", err) ctx.ServerError("GetObsListObject:", err) @@ -322,7 +322,7 @@ func ModelList(ctx *context.APIContext) { }) } -func ModelDownload(ctx *context.APIContext) { +func ModelDownload1(ctx *context.APIContext) { var ( err error ) @@ -346,3 +346,62 @@ func ModelDownload(ctx *context.APIContext) { } http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } + +func ModelDownload(ctx *context.Context) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + versionName := ctx.Query("version_name") + // versionName := "V0001" + parentDir := ctx.Query("parent_dir") + fileName := ctx.Query("file_name") + log.Info("DownloadSingleModelFile start.") + // id := ctx.Params(":ID") + // path := Model_prefix + models.AttachmentRelativePath(id) + "/" + parentDir + fileName + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + + path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir, fileName), "/") + log.Info("Download path is:%s", path) + if setting.PROXYURL != "" { + body, err := storage.ObsDownloadAFile(setting.Bucket, path) + if err != nil { + log.Info("download error.") + } else { + //count++ + // models.ModifyModelDownloadCount(id) + defer body.Close() + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + //fmt.Printf("%s", p[:readCount]) + } + if readErr != nil { + break + } + } + } + } else { + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + //count++ + // models.ModifyModelDownloadCount(id) + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + } +} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index b0ef88ebc..a3602e0e7 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -480,6 +480,12 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { var jobID = ctx.Params(":jobid") // var versionName = ctx.Params(":version-name") var versionName = ctx.Query("version_name") + // canNewJob, err := canUserCreateTrainJobVersion(ctx, jobID) + // if err != nil { + // ctx.ServerError("get can info failed", err) + // return err + // } + // ctx.Data["canNewJob"] = canNewJob task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) if err != nil { @@ -541,7 +547,8 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { ctx.ServerError("GetBranches error:", err) return err } - ctx.Data["branches"] = Branches + + ctx.Data["branch"] = Branches ctx.Data["branch_name"] = task.BranchName ctx.Data["description"] = task.Description ctx.Data["boot_file"] = task.BootFile @@ -1305,6 +1312,18 @@ func canUserCreateTrainJob(uid int64) (bool, error) { return org.IsOrgMember(uid) } +func canUserCreateTrainJobVersion(ctx *context.Context, jobID string) (bool, error) { + + var versionName = "V0001" + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + return false, err + } + if ctx.User.ID == task.User.ID { + return true, nil + } + return false, err +} func TrainJobGetConfigList(ctx *context.Context) { ctx.Data["PageIsTrainJob"] = true From ab42537f9a044cea52e956eb9d95f768e0e64e89 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 25 Nov 2021 09:33:41 +0800 Subject: [PATCH 09/10] fix bug --- routers/api/v1/repo/modelarts.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 568464f77..80ff57a56 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -322,31 +322,6 @@ func ModelList(ctx *context.APIContext) { }) } -func ModelDownload1(ctx *context.APIContext) { - var ( - err error - ) - - var jobID = ctx.Params(":jobid") - versionName := ctx.Query("version_name") - parentDir := ctx.Query("parent_dir") - fileName := ctx.Query("file_name") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - return - } - VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) - parentDir = VersionOutputPath + "/" + parentDir - url, err := storage.GetObsCreateSignedUrl(task.JobName, parentDir, fileName) - if err != nil { - log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrl", err) - return - } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) -} - func ModelDownload(ctx *context.Context) { var ( err error From 9afb845f1e541b0334b82ee0285557e9cb4e9808 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 25 Nov 2021 10:01:44 +0800 Subject: [PATCH 10/10] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 6b22a8412..6a36ad287 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -657,6 +657,12 @@ td, th { } } + function downloadModelFile(version_name,parentDir,filename){ + $.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/model_download?version_name=${version_name}&file_name=${filename}`, (data) => { + console.log(data) + + }) + } function renderDir(data,version_name){ let html="" html += "
" @@ -674,10 +680,10 @@ td, th { html += "" html += "" if(data.Dirs[i].IsDir){ - html += `` + html += `` html += "" + data.Dirs[i].FileName + "" }else{ - html += `` + html += `` html += "" + data.Dirs[i].FileName + "" } html += ''