From 9d57eee16b8efd5db7d032c7556eb9a8a7b48c67 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 1 Aug 2022 15:07:36 +0800 Subject: [PATCH 01/14] debug --- routers/repo/grampus.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 76f2bd98c..c48950915 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -787,8 +787,17 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo } } - commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" - command += commandCode + if processorType == grampus.ProcessorTypeNPU { + commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" + commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" + command += commandCode + } else if processorType == grampus.ProcessorTypeGPU { + commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" + command += commandCode + } + + //commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" + //command += commandCode //get exec result commandGetRes := "result=$?;" From 104fe3ff2acc1f68207be24bc0ed5754f87c067d Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 5 Aug 2022 11:57:03 +0800 Subject: [PATCH 02/14] check bootfile --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/repo/cloudbrain.go | 26 ++++++++++++++++++++++ routers/repo/grampus.go | 16 ++++++++++++++ routers/repo/modelarts.go | 48 ++++++++++++++++++++++++++++++----------- 5 files changed, 80 insertions(+), 12 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d1e61d242..c4b59f501 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1073,6 +1073,7 @@ cloudbrain_operate = Operate cloudbrain_status_createtime = Status/Createtime cloudbrain_status_runtime = Running Time cloudbrain_jobname_err=Name must start with a lowercase letter or number,can include lowercase letter,number,_ and -,can not end with _, and can be up to 36 characters long. +cloudbrain_bootfile_err=The bootfile does not exist in the repository cloudbrain_query_fail=Failed to query cloudbrain information. cloudbrain.mirror_tag = Mirror Tag cloudbrain.mirror_description = Mirror Description diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index a39118c0a..8006bf1fe 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1076,6 +1076,7 @@ cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 cloudbrain_status_runtime = 运行时长 cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。 +cloudbrain_bootfile_err=仓库中不存在启动文件 cloudbrain_query_fail=查询云脑任务失败。 cloudbrain.mirror_tag = 镜像标签 cloudbrain.mirror_description = 镜像描述 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 76bf9b076..b84797361 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -239,6 +239,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId branchName := form.BranchName + bootFile := strings.TrimSpace(form.BootFile) repo := ctx.Repo.Repository tpl := tplCloudBrainNew @@ -269,6 +270,14 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) + return + } + if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) @@ -413,6 +422,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId branchName := form.BranchName + bootFile := strings.TrimSpace(form.BootFile) labelName := form.LabelName repo := ctx.Repo.Repository @@ -450,6 +460,14 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) + return + } + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -527,6 +545,14 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra } /** + 获取启动文件的网页地址 +*/ +func getBootfileUrl(repo *models.Repository, branchName string, bootFile string) string { + url := setting.AppURL + repo.OwnerName + "/" + repo.Name + "/src/branch/" + branchName + "/" + bootFile + return url +} + +/** 检查用户传输的参数是否符合专属资源池 */ func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 060b020e2..c6fe3789a 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -215,6 +215,14 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobGPUNew, &form) + return + } + errStr := checkSpecialPool(ctx, "GPU") if errStr != "" { grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) @@ -399,6 +407,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobNPUNew, &form) + return + } + errStr := checkSpecialPool(ctx, "NPU") if errStr != "" { grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 948a0e751..632f9cff0 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1089,7 +1089,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" // dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - branch_name := form.BranchName + branchName := form.BranchName isLatestVersion := modelarts.IsLatestVersion FlavorName := form.FlavorName VersionCount := modelarts.VersionCount @@ -1117,6 +1117,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobNew, &form) + return + } + errStr := checkModelArtsSpecialPool(ctx, flavorCode, string(models.JobTypeTrain)) if errStr != "" { trainJobErrorNewDataPrepare(ctx, form) @@ -1148,9 +1156,9 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } gitRepo, _ := git.OpenRepository(repo.RepoPath()) - commitID, _ := gitRepo.GetBranchCommitID(branch_name) + commitID, _ := gitRepo.GetBranchCommitID(branchName) - if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { + if err := downloadCode(repo, codeLocalPath, branchName); err != nil { log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplModelArtsTrainJobNew, &form) @@ -1292,7 +1300,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) Parameters: param, CommitID: commitID, IsLatestVersion: isLatestVersion, - BranchName: branch_name, + BranchName: branchName, Params: form.Params, FlavorName: FlavorName, EngineName: EngineName, @@ -1394,7 +1402,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" // dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - branch_name := form.BranchName + branchName := form.BranchName PreVersionName := form.VersionName FlavorName := form.FlavorName EngineName := form.EngineName @@ -1414,6 +1422,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobVersionNew, &form) + return + } + errStr := checkModelArtsSpecialPool(ctx, flavorCode, string(models.JobTypeTrain)) if errStr != "" { versionErrorDataPrepare(ctx, form) @@ -1428,8 +1444,8 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ } gitRepo, _ := git.OpenRepository(repo.RepoPath()) - commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { + commitID, _ := gitRepo.GetBranchCommitID(branchName) + if err := downloadCode(repo, codeLocalPath, branchName); err != nil { log.Error("Failed git clone repo to local(！: %s (%v)", repo.FullName(), err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplModelArtsTrainJobVersionNew, &form) @@ -1582,7 +1598,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ Parameters: param, PreVersionId: task.VersionID, CommitID: commitID, - BranchName: branch_name, + BranchName: branchName, FlavorName: FlavorName, EngineName: EngineName, PreVersionName: PreVersionName, @@ -2025,7 +2041,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference resultObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.ResultPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - branch_name := form.BranchName + branchName := form.BranchName FlavorName := form.FlavorName EngineName := form.EngineName LabelName := form.LabelName @@ -2060,6 +2076,14 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference return } + resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) + if err != nil || resp.StatusCode != 200 { + log.Error("Get bootfile error:", err) + inferenceJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsInferenceJobNew, &form) + return + } + //Determine whether the task name of the task in the project is duplicated tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeInference), displayJobName) if err == nil { @@ -2092,9 +2116,9 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } gitRepo, _ := git.OpenRepository(repo.RepoPath()) - commitID, _ := gitRepo.GetBranchCommitID(branch_name) + commitID, _ := gitRepo.GetBranchCommitID(branchName) - if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { + if err := downloadCode(repo, codeLocalPath, branchName); err != nil { log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplModelArtsInferenceJobNew, &form) @@ -2178,7 +2202,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference Uuid: uuid, Parameters: param, //modelarts train parameters CommitID: commitID, - BranchName: branch_name, + BranchName: branchName, Params: form.Params, FlavorName: FlavorName, EngineName: EngineName, From 146f66ea3798d68344a81932bedc81811fb7a59f Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 09:49:46 +0800 Subject: [PATCH 03/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/minio_ext.go | 41 ----------------------------------- templates/repo/modelmanage/index.tmpl | 2 +- 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index 05b692335..4b738c068 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -214,47 +214,6 @@ func GetOneLevelAllObjectUnderDirMinio(bucket string, prefixRootPath string, rel ParenDir: ParenDir, } fileInfos = append(fileInfos, fileInfo) - - // log.Info("val key=" + val.Key) - // var isDir bool - // var fileName string - // if val.Key == Prefix { - // continue - // } - - // fileName = val.Key[prefixLen:] - // log.Info("fileName =" + fileName) - // files := strings.Split(fileName, "/") - // if fileMap[files[0]] { - // continue - // } else { - // fileMap[files[0]] = true - // } - // ParenDir := relativePath - // fileName = files[0] - // if len(files) > 1 { - // isDir = true - // ParenDir += fileName + "/" - // } else { - // isDir = false - // } - - // // if strings.HasSuffix(val.Key, "/") { - // // isDir = true - // // fileName = val.Key[prefixLen : len(val.Key)-1] - // // relativePath += val.Key[prefixLen:] - // // } else { - // // isDir = false - // // fileName = val.Key[prefixLen:] - // // } - // fileInfo := FileInfo{ - // ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - // FileName: fileName, - // Size: val.Size, - // IsDir: isDir, - // ParenDir: relativePath, - // } - // fileInfos = append(fileInfos, fileInfo) } return fileInfos, err } else { diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index c42f1ba7b..4d02a8e21 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -463,7 +463,7 @@ parentNodeMap = nodesMap; var fileSplits = data[i].FileName.split("/"); for(let j=0;j < fileSplits.length;j++){ - if(fileSplits[j] == ""){ + if(fileSplits[j] == "" && !(data[i].FileName[data[i].FileName.length - 1]=="/")){ break; } if(parentNodeMap[fileSplits[j]] == null){ From 6a60330adc3e863d870db167fa1e0a1d9cf6a78d Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 10:04:51 +0800 Subject: [PATCH 04/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 4d02a8e21..c42f1ba7b 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -463,7 +463,7 @@ parentNodeMap = nodesMap; var fileSplits = data[i].FileName.split("/"); for(let j=0;j < fileSplits.length;j++){ - if(fileSplits[j] == "" && !(data[i].FileName[data[i].FileName.length - 1]=="/")){ + if(fileSplits[j] == ""){ break; } if(parentNodeMap[fileSplits[j]] == null){ From aceb9ad8ec742466bf8b53b243d8fb12ad8c80b4 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 10:51:42 +0800 Subject: [PATCH 05/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index c42f1ba7b..567ff8dc7 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -325,6 +325,7 @@ $('#name').val(modelName) $('#version').val("0.0.1") } + let dirKey="isOnlyDir--:&"; function showcreate(obj) { $('.ui.modal.second') .modal({ @@ -471,6 +472,16 @@ } parentNodeMap = parentNodeMap[fileSplits[j]] } + for(let j=0;j < fileSplits.length;j++){ + if(fileSplits[j] == ""){ + if(data[i].FileName[data[i].FileName.length -1] =="/"){ + if(Object.keys(parentNodeMap[fileSplits[j]]).length ==0){ + parentNodeMap[fileSplits[j]][dirKey]="true"; + } + } + break; + } + } } convertToNode(zNodes,nodesMap); $.fn.zTree.init($("#treeDemo"), setting, zNodes); @@ -488,12 +499,18 @@ node["name"] = keyList[i]; nodeList.push(node); if(nodesMap[keyList[i]] != null && Object.keys(nodesMap[keyList[i]]).length >0){ - node["children"]=[]; - if(isFirst){ - node["open"] = true; - isFirst= false; + if(nodesMap[keyList[i]][dirKey] != null){ + node["open"] = false; + node["isParent"] = true; + }else{ + node["children"]=[]; + if(isFirst){ + node["open"] = true; + isFirst= false; + } + convertToNode(node["children"],nodesMap[keyList[i]]); } - convertToNode(node["children"],nodesMap[keyList[i]]); + } } } From 29b4628b9014f27b4c2f1d8c3ab89c659b79dd18 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 10:56:13 +0800 Subject: [PATCH 06/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 567ff8dc7..9e741cf2d 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -475,7 +475,7 @@ for(let j=0;j < fileSplits.length;j++){ if(fileSplits[j] == ""){ if(data[i].FileName[data[i].FileName.length -1] =="/"){ - if(Object.keys(parentNodeMap[fileSplits[j]]).length ==0){ + if(Object.keys(parentNodeMap[fileSplits[j-1]]).length ==0){ parentNodeMap[fileSplits[j]][dirKey]="true"; } } From 6c7bef90e983afb34e7989ea6e1e64909100c101 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 11:12:49 +0800 Subject: [PATCH 07/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 9e741cf2d..3f83ea5e1 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -470,8 +470,10 @@ if(parentNodeMap[fileSplits[j]] == null){ parentNodeMap[fileSplits[j]] = {}; } - parentNodeMap = parentNodeMap[fileSplits[j]] + parentNodeMap = parentNodeMap[fileSplits[j]]; } + + parentNodeMap = nodesMap; for(let j=0;j < fileSplits.length;j++){ if(fileSplits[j] == ""){ if(data[i].FileName[data[i].FileName.length -1] =="/"){ @@ -481,6 +483,7 @@ } break; } + parentNodeMap = parentNodeMap[fileSplits[j]]; } } convertToNode(zNodes,nodesMap); From ee51a420fb642a7312984e99d5a0f46a82af3228 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 11:20:18 +0800 Subject: [PATCH 08/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 3f83ea5e1..9fd3e1007 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -461,7 +461,7 @@ var zNodes=[]; var nodesMap={}; for (let i=0;i Date: Mon, 8 Aug 2022 11:24:25 +0800 Subject: [PATCH 09/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 9fd3e1007..e92c3bd5f 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -472,8 +472,10 @@ } parentNodeMap = parentNodeMap[fileSplits[j]]; } - - parentNodeMap = nodesMap; + } + for (let i=0;i Date: Mon, 8 Aug 2022 11:31:22 +0800 Subject: [PATCH 10/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index e92c3bd5f..52b042c46 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -479,8 +479,8 @@ for(let j=0;j < fileSplits.length;j++){ if(fileSplits[j] == ""){ if(data[i].FileName[data[i].FileName.length -1] =="/"){ - if(Object.keys(parentNodeMap[fileSplits[j-1]]).length ==0){ - parentNodeMap[fileSplits[j-1]][dirKey]="true"; + if(Object.keys(parentNodeMap).length ==0){ + parentNodeMap[dirKey]="true"; } } break; From 78cea11898fceab693c5fff11acab037a09071b3 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 8 Aug 2022 16:40:18 +0800 Subject: [PATCH 11/14] opt --- routers/repo/grampus.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index c48950915..9a1f9591c 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -787,17 +787,14 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo } } + var commandCode string if processorType == grampus.ProcessorTypeNPU { - commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" - command += commandCode } else if processorType == grampus.ProcessorTypeGPU { - commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" - command += commandCode + commandCode = "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" } - //commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" - //command += commandCode + command += commandCode //get exec result commandGetRes := "result=$?;" From a7d8c351e20f5a7a2f4ea01f6670973586c67c2b Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Aug 2022 16:44:51 +0800 Subject: [PATCH 12/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3mindspore=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/grampus/trainjob/show.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 728b876b5..25f1af82b 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -773,7 +773,7 @@ } if (srcEngine == 'mindspore') { $('#choice_Engine .default.text').text("MindSpore"); - $('#choice_Engine input[name="Engine"]').val(1) + $('#choice_Engine input[name="Engine"]').val(2) } } }else{ From ab315042b88eacfbf690296612ba8f2a1963490b Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 9 Aug 2022 14:45:00 +0800 Subject: [PATCH 13/14] checkout bootfile --- routers/repo/cloudbrain.go | 27 +++++++++------------------ routers/repo/grampus.go | 8 ++++---- routers/repo/modelarts.go | 12 ++++++------ 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b84797361..066008e49 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -270,14 +270,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { - log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) - return - } - if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) @@ -314,6 +306,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command := cloudbrain.GetCloudbrainDebugCommand() if jobType == string(models.JobTypeTrain) { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { + log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) + return + } tpl = tplCloudBrainTrainJobNew commandTrain, err := getTrainJobCommand(form) if err != nil { @@ -460,8 +459,8 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) @@ -545,14 +544,6 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra } /** - 获取启动文件的网页地址 -*/ -func getBootfileUrl(repo *models.Repository, branchName string, bootFile string) string { - url := setting.AppURL + repo.OwnerName + "/" + repo.Name + "/src/branch/" + branchName + "/" + bootFile - return url -} - -/** 检查用户传输的参数是否符合专属资源池 */ func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index c6fe3789a..c31143a33 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -215,8 +215,8 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobGPUNew, &form) @@ -407,8 +407,8 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobNPUNew, &form) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 632f9cff0..a0dfa6d05 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1117,8 +1117,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { log.Error("Get bootfile error:", err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobNew, &form) @@ -1422,8 +1422,8 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { log.Error("Get bootfile error:", err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobVersionNew, &form) @@ -2076,8 +2076,8 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference return } - resp, err := http.Get(getBootfileUrl(repo, branchName, bootFile)) - if err != nil || resp.StatusCode != 200 { + bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExict { log.Error("Get bootfile error:", err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsInferenceJobNew, &form) From e8f0f987ca5d1461ba420d2503b083e5490fcf18 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 9 Aug 2022 14:57:25 +0800 Subject: [PATCH 14/14] checkout bootfile --- routers/repo/cloudbrain.go | 8 ++++---- routers/repo/grampus.go | 8 ++++---- routers/repo/modelarts.go | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 066008e49..1f33f7023 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -306,8 +306,8 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command := cloudbrain.GetCloudbrainDebugCommand() if jobType == string(models.JobTypeTrain) { - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) @@ -459,8 +459,8 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra return } - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index c31143a33..133f270f2 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -215,8 +215,8 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain return } - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobGPUNew, &form) @@ -407,8 +407,8 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain return } - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobNPUNew, &form) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index a0dfa6d05..92236d235 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1117,8 +1117,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobNew, &form) @@ -1422,8 +1422,8 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobVersionNew, &form) @@ -2076,8 +2076,8 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference return } - bootFileExict, err := ctx.Repo.FileExists(bootFile, branchName) - if err != nil || !bootFileExict { + bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) + if err != nil || !bootFileExist { log.Error("Get bootfile error:", err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsInferenceJobNew, &form)