From da78669bc72419cd3bfe5ce7a5d4bb2453bbb219 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 9 Nov 2022 17:46:06 +0800 Subject: [PATCH 01/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 69 +++++- modules/modelarts_cd/modelarts.go | 51 +--- modules/setting/setting.go | 22 +- modules/structs/cloudbrain.go | 8 + options/locale/locale_en-US.ini | 4 + options/locale/locale_zh-CN.ini | 3 + routers/api/v1/api.go | 6 + routers/api/v1/repo/cloudbrain.go | 3 + routers/repo/aisafety.go | 9 +- routers/repo/cloudbrain.go | 26 +- routers/repo/grampus.go | 4 +- routers/repo/modelarts.go | 15 +- services/cloudbrain/cloudbrainTask/notebook.go | 330 +++++++++++++++++++++++++ services/cloudbrain/cloudbrainTask/train.go | 12 + services/cloudbrain/util.go | 39 ++- 15 files changed, 501 insertions(+), 100 deletions(-) create mode 100644 services/cloudbrain/cloudbrainTask/notebook.go diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index dd502dfd0..65984d95f 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,19 @@ package modelarts import ( + "encoding/base64" "encoding/json" "errors" "fmt" + "io/ioutil" + "net/http" + "os" "path" "strconv" "strings" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/modelarts_cd" "code.gitea.io/gitea/models" @@ -276,7 +282,7 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin return nil } -func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification) error { +func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string) (string, error) { if poolInfos == nil { json.Unmarshal([]byte(setting.PoolInfos), &poolInfos) } @@ -284,7 +290,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc imageName, err := GetNotebookImageName(imageId) if err != nil { log.Error("GetNotebookImageName failed: %v", err.Error()) - return err + return "", err } createTime := timeutil.TimeStampNow() jobResult, err := createNotebook2(models.CreateNotebook2Params{ @@ -316,10 +322,10 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc }) if errTemp != nil { log.Error("InsertCloudbrainTemp failed: %v", errTemp.Error()) - return errTemp + return "", errTemp } } - return err + return "", err } task := &models.Cloudbrain{ Status: jobResult.Status, @@ -334,6 +340,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc Uuid: uuid, ComputeResource: models.NPUResource, Image: imageName, + BootFile: bootFile, Description: description, CreatedUnix: createTime, UpdatedUnix: createTime, @@ -342,12 +349,12 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc err = models.CreateCloudbrain(task) if err != nil { - return err + return "", err } stringId := strconv.FormatInt(task.ID, 10) notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugNPUTask) - return nil + return jobResult.ID, nil } func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) { @@ -907,6 +914,11 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { if task.FlavorCode == "" { task.FlavorCode = result.Flavor } + + if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" { + uploadNoteBookFile(task, result) + + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -917,6 +929,51 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { return nil } +func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { + jupyterUrl := result.Url + "?token=" + result.Token + client := getRestyClient() + res, err := client.R().Get(jupyterUrl) + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath + if err != nil { + log.Error("browser jupyterUrl failed:%v", task.DisplayJobName, err) + + } else { + cookies := res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + } + } + + fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) + if err != nil { + log.Error("read jupyter file failed:%v", task.DisplayJobName, err) + } + + base64Content := base64.StdEncoding.EncodeToString(fileContents) + + uploadUrl := result.Url + "/api/contents/" + path.Base(task.BootFile) + res, err = client.R(). + SetCookies(cookies). + SetHeader("X-XSRFToken", xsrf). + SetBody(map[string]interface{}{ + "type": "file", + "format": "base64", + "name": path.Base(task.BootFile), + "path": path.Base(task.BootFile), + "content": base64Content}). + Put(uploadUrl) + if err != nil { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + } else if res.StatusCode() != http.StatusCreated { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + } + + } + go os.RemoveAll(codePath) +} + func SyncTempStatusJob() { jobs, err := models.GetCloudBrainTempJobs() if err != nil { diff --git a/modules/modelarts_cd/modelarts.go b/modules/modelarts_cd/modelarts.go index 330b048ca..712cdc89b 100755 --- a/modules/modelarts_cd/modelarts.go +++ b/modules/modelarts_cd/modelarts.go @@ -88,11 +88,11 @@ type Parameters struct { } `json:"parameter"` } -func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification) error { +func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string) (string, error) { imageName, err := GetNotebookImageName(imageId) if err != nil { log.Error("GetNotebookImageName failed: %v", err.Error()) - return err + return "", err } createTime := timeutil.TimeStampNow() jobResult, err := createNotebook(models.CreateNotebookWithoutPoolParams{ @@ -123,10 +123,10 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr }) if errTemp != nil { log.Error("InsertCloudbrainTemp failed: %v", errTemp.Error()) - return errTemp + return "", errTemp } } - return err + return "", err } task := &models.Cloudbrain{ Status: jobResult.Status, @@ -145,16 +145,17 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr CreatedUnix: createTime, UpdatedUnix: createTime, Spec: spec, + BootFile: bootFile, } err = models.CreateCloudbrain(task) if err != nil { - return err + return "", err } stringId := strconv.FormatInt(task.ID, 10) notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugNPUTask) - return nil + return jobResult.ID, nil } func GetNotebookImageName(imageId string) (string, error) { @@ -175,41 +176,3 @@ func GetNotebookImageName(imageId string) (string, error) { return imageName, nil } - -/* -func HandleNotebookInfo(task *models.Cloudbrain) error { - - result, err := GetNotebook(task.JobID) - if err != nil { - log.Error("GetNotebook2(%s) failed:%v", task.DisplayJobName, err) - return err - } - - if result != nil { - oldStatus := task.Status - task.Status = result.Status - if task.StartTime == 0 && result.Lease.UpdateTime > 0 { - task.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000) - } - if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { - task.EndTime = timeutil.TimeStampNow() - } - task.CorrectCreateUnix() - task.ComputeAndSetDuration() - if oldStatus != task.Status { - notification.NotifyChangeCloudbrainStatus(task, oldStatus) - } - if task.FlavorCode == "" { - task.FlavorCode = result.Flavor - } - err = models.UpdateJob(task) - if err != nil { - log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) - return err - } - } - - return nil -} - -*/ diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 88557ee60..a358a85d4 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -609,9 +609,9 @@ var ( AiCenterInfo string }{} - C2NetInfos *C2NetSqInfos - CenterInfos *AiCenterInfos - C2NetMapInfo map[string]*C2NetSequenceInfo + C2NetInfos *C2NetSqInfos + CenterInfos *AiCenterInfos + C2NetMapInfo map[string]*C2NetSequenceInfo //elk config ElkUrl string @@ -712,6 +712,16 @@ var ( TeamName string }{} + FileNoteBook = struct { + ProjectName string + ImageGPU string + SpecIdGPU int64 + SpecIdCPU int64 + ImageIdNPU string + ImageNPU string + SpecIdNPU int64 + }{} + ModelConvert = struct { GPU_PYTORCH_IMAGE string GpuQueue string @@ -1655,9 +1665,9 @@ func getGrampusConfig() { if err := json.Unmarshal([]byte(Grampus.C2NetSequence), &C2NetInfos); err != nil { log.Error("Unmarshal(C2NetSequence) failed:%v", err) } - C2NetMapInfo=make(map[string]*C2NetSequenceInfo) - for _,value :=range C2NetInfos.C2NetSqInfo{ - C2NetMapInfo[value.Name]=value + C2NetMapInfo = make(map[string]*C2NetSequenceInfo) + for _, value := range C2NetInfos.C2NetSqInfo { + C2NetMapInfo[value.Name] = value } } Grampus.SyncScriptProject = sec.Key("SYNC_SCRIPT_PROJECT").MustString("script_for_grampus") diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 866c85dad..9ea5601c9 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -41,6 +41,14 @@ type CreateTrainJobOption struct { SpecId int64 `json:"spec_id" binding:"Required"` } +type CreateFileNotebookJobOption struct { + Type int `json:"type"` //0 CPU 1 GPU 2 NPU + File string `json:"file" binding:"Required"` + BranchName string `json:"branch_name" binding:"Required"` + OwnerName string `json:"owner_name" binding:"Required"` + ProjectName string `json:"project_name" binding:"Required"` +} + type Cloudbrain struct { ID int64 `json:"id"` JobID string `json:"job_id"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e8e8722e2..3f0c0a010 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1007,6 +1007,7 @@ readme = README readme_helper = Select a README file template. auto_init = Initialize Repository (Adds .gitignore, License and README) create_repo = Create Repository +failed_to_create_repo=Failed to create repository, please try again later. create_course = Publish Course failed_to_create_course=Failed to publish course, please try again later. default_branch = Default Branch @@ -1041,6 +1042,9 @@ model_experience = Model Experience model_noright=You have no right to do the operation. model_rename=Duplicate model name, please modify model name. +notebook_file_not_exist=Notebook file does not exist. +notebook_select_wrong=Please select a Notebook(.ipynb) file first. + date=Date repo_add=Project Increment repo_total=Project Total diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index d13c99443..8b635112b 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1013,6 +1013,7 @@ readme=自述 readme_helper=选择自述文件模板。 auto_init=初始化存储库 (添加. gitignore、许可证和自述文件) create_repo=创建项目 +failed_to_create_repo=创建项目失败，请稍后再试。 create_course=发布课程 failed_to_create_course=发布课程失败，请稍后再试。 default_branch=默认分支 @@ -1041,6 +1042,8 @@ model_experience = 模型体验 model_noright=您没有操作权限。 model_rename=模型名称重复，请修改模型名称 +notebook_file_not_exist=Notebook文件不存在。 +notebook_select_wrong=请先选择Notebook（.ipynb）文件。 date=日期 repo_add=新增项目 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 813b77c8c..b7e3c6415 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -736,6 +736,12 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/my_favorite", repo.MyFavoriteDatasetMultiple) }, reqToken(), repoAssignment()) + m.Group("/file_notebook", func() { + + m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) + + }) + m.Group("/repos", func() { m.Get("/search", repo.Search) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 2e25fdefe..5ea797e97 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -79,6 +79,9 @@ func CloudBrainShow(ctx *context.APIContext) { ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)}) } +func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { + cloudbrainTask.FileNotebookCreate(ctx.Context, option) +} func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == cloudbrainTask.TaskTypeCloudbrainOne { diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index b638a486b..6176fcda5 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -11,7 +11,8 @@ import ( "os" "strconv" "strings" - "time" + + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/aisafety" @@ -483,7 +484,6 @@ func isTaskNotFinished(status string) bool { } func AiSafetyCreateForGetGPU(ctx *context.Context) { - t := time.Now() ctx.Data["PageIsCloudBrain"] = true ctx.Data["IsCreate"] = true ctx.Data["type"] = models.TypeCloudBrainOne @@ -497,7 +497,7 @@ func AiSafetyCreateForGetGPU(ctx *context.Context) { log.Info("GPUBaseDataSetUUID=" + setting.ModelSafetyTest.GPUBaseDataSetUUID) log.Info("GPUCombatDataSetName=" + setting.ModelSafetyTest.GPUCombatDataSetName) log.Info("GPUCombatDataSetUUID=" + setting.ModelSafetyTest.GPUCombatDataSetUUID) - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName prepareCloudbrainOneSpecs(ctx) queuesDetail, _ := cloudbrain.GetQueuesDetail() @@ -514,12 +514,11 @@ func AiSafetyCreateForGetGPU(ctx *context.Context) { } func AiSafetyCreateForGetNPU(ctx *context.Context) { - t := time.Now() ctx.Data["PageIsCloudBrain"] = true ctx.Data["IsCreate"] = true ctx.Data["type"] = models.TypeCloudBrainTwo ctx.Data["compute_resource"] = models.NPUResource - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName ctx.Data["datasetType"] = models.TypeCloudBrainTwo ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.NPUBaseDataSetName diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 61c2925b8..607d05d3c 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,7 +2,6 @@ package repo import ( "bufio" - "code.gitea.io/gitea/modules/urfs_client/urchin" "encoding/json" "errors" "fmt" @@ -16,6 +15,10 @@ import ( "time" "unicode/utf8" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + + "code.gitea.io/gitea/modules/urfs_client/urchin" + "code.gitea.io/gitea/modules/dataset" "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" @@ -91,28 +94,9 @@ func MustEnableCloudbrain(ctx *context.Context) { } } -func cutString(str string, lens int) string { - if len(str) < lens { - return str - } - return str[:lens] -} - -func jobNamePrefixValid(s string) string { - lowStr := strings.ToLower(s) - re := regexp.MustCompile(`[^a-z0-9_\\-]+`) - - removeSpecial := re.ReplaceAllString(lowStr, "") - - re = regexp.MustCompile(`^[_\\-]+`) - return re.ReplaceAllString(removeSpecial, "") - -} - func cloudBrainNewDataPrepare(ctx *context.Context, jobType string) error { ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand() diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 0620350f6..c94bc2a5b 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -11,7 +11,6 @@ import ( "path" "strconv" "strings" - "time" "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" @@ -75,8 +74,7 @@ func GrampusTrainJobNPUNew(ctx *context.Context) { func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName //get valid images diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index fabf7e555..214d4b0a1 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -15,6 +15,8 @@ import ( "time" "unicode/utf8" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" "code.gitea.io/gitea/modules/dataset" @@ -128,8 +130,7 @@ func NotebookNew(ctx *context.Context) { func notebookNewDataPrepare(ctx *context.Context) error { ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) @@ -239,9 +240,9 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm } if setting.ModelartsCD.Enabled { - err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, uuid, description, imageId, spec) + _, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, uuid, description, imageId, spec, "") } else { - err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, uuid, description, imageId, spec) + _, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, uuid, description, imageId, spec, "") } if err != nil { @@ -714,8 +715,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { // return //} - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) @@ -2351,8 +2351,7 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { ctx.Data["PageIsCloudBrain"] = true ctx.Data["newInference"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go new file mode 100644 index 000000000..ea1f8b081 --- /dev/null +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -0,0 +1,330 @@ +package cloudbrainTask + +import ( + "fmt" + "net/http" + "os" + + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/modelarts_cd" + + "code.gitea.io/gitea/modules/git" + + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward/point/account" + + "code.gitea.io/gitea/modules/setting" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + repo_service "code.gitea.io/gitea/services/repository" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { + + if ctx.Written() { + return + } + //create repo if not exist + repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) + if repo == nil { + repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ + Name: setting.FileNoteBook.ProjectName, + Alias: "", + Description: "", + IssueLabels: "", + Gitignores: "", + License: "", + Readme: "Default", + IsPrivate: false, + AutoInit: true, + DefaultBranch: "master", + }) + } + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("repo.failed_to_create_repo")) + } + +} + +func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { + + displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + jobType := string(models.JobTypeDebug) + + codePath := getCodePath(jobName) + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + defer lock.UnLock() + isOk, err := lock.Lock(models.CloudbrainKeyDuration) + if !isOk { + log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } + } + + count, err := GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain.morethanonejob"))) + return + } + } + repoPath := models.RepoPath(repo.OwnerName, repo.Name) + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) + return + } + // We opened it, we should close it + defer func() { + // If it's been set to nil then assume someone else has closed it. + if ctx.Repo.GitRepo != nil { + ctx.Repo.GitRepo.Close() + } + }() + + ctx.Repo = &context.Repository{ + Repository: repo, + GitRepo: gitRepo, + } + + fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) + if err != nil || !fileExist { + log.Error("Get file error:", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + command := cloudbrain.GetCloudbrainDebugCommand() + + errStr := uploadCodeFile(repo, codePath, option.BranchName, option.File, jobName) + if errStr != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(option.BranchName) + specId := setting.FileNoteBook.SpecIdGPU + if option.Type == 0 { + specId = setting.FileNoteBook.SpecIdCPU + } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobType(jobType), + ComputeResource: models.GPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainOne}) + if err != nil || spec == nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.wrong_specification"))) + return + } + + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) + return + } + + req := cloudbrain.GenerateCloudBrainTaskReq{ + Ctx: ctx, + DisplayJobName: displayJobName, + JobName: jobName, + Image: setting.FileNoteBook.ImageGPU, + Command: command, + Uuids: "", + DatasetNames: "", + DatasetInfos: nil, + CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), + Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + JobType: jobType, + Description: getDescription(option), + BranchName: option.BranchName, + BootFile: option.File, + Params: "{\"parameter\":[]}", + CommitID: commitID, + BenchmarkTypeID: 0, + BenchmarkChildTypeID: 0, + ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), + Spec: spec, + } + + jobId, err := cloudbrain.GenerateTask(req) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) + +} + +func getCodePath(jobName string) string { + return setting.JobPath + jobName + cloudbrain.CodeMountPath +} + +func getDescription(option api.CreateFileNotebookJobOption) string { + return option.OwnerName + "/" + option.ProjectName + "/" + option.File +} + +func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { + displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName)) + isOk, err := lock.Lock(models.CloudbrainKeyDuration) + if !isOk { + log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + defer lock.UnLock() + + count, err := GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug)) + + if err != nil { + log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) + + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain.morethanonejob"))) + return + } + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeDebug), displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } + } + repoPath := models.RepoPath(repo.OwnerName, repo.Name) + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) + return + } + // We opened it, we should close it + defer func() { + // If it's been set to nil then assume someone else has closed it. + if ctx.Repo.GitRepo != nil { + ctx.Repo.GitRepo.Close() + } + }() + + ctx.Repo = &context.Repository{ + Repository: repo, + GitRepo: gitRepo, + } + + fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) + if err != nil || !fileExist { + log.Error("Get file error:", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + err = downloadCode(repo, getCodePath(jobName), option.BranchName) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + + var aiCenterCode = models.AICenterOfCloudBrainTwo + if setting.ModelartsCD.Enabled { + aiCenterCode = models.AICenterOfChengdu + } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, setting.FileNoteBook.SpecIdNPU, models.FindSpecsOptions{ + JobType: models.JobTypeDebug, + ComputeResource: models.NPU, + Cluster: models.OpenICluster, + AiCenterCode: aiCenterCode}) + if err != nil || spec == nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.wrong_specification"))) + return + } + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) + return + } + + var jobId string + if setting.ModelartsCD.Enabled { + jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) + } else { + jobId, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) + } + + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + + return + } + + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) + +} + +func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { + err := downloadCode(repo, codePath, branchName) + if err != nil { + return "cloudbrain.load_code_failed" + } + + err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") + if err != nil { + return "cloudbrain.load_code_failed" + } + go os.RemoveAll(codePath) + return "" +} diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 8e4673d66..5de69335a 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -810,6 +810,18 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { return nil } +func uploadOneFileToMinio(codePath, filePath, jobName, parentDir string) error { + destObject := setting.CBCodePathPrefix + jobName + parentDir + path.Base(filePath) + sourceFile := codePath + filePath + err := storage.Attachments.UploadObject(destObject, sourceFile) + if err != nil { + log.Error("UploadObject(%s) failed: %s", filePath, err.Error()) + return err + } + return nil + +} + func readDir(dirname string) ([]os.FileInfo, error) { f, err := os.Open(dirname) if err != nil { diff --git a/services/cloudbrain/util.go b/services/cloudbrain/util.go index ab738927e..d9f0510be 100644 --- a/services/cloudbrain/util.go +++ b/services/cloudbrain/util.go @@ -1,27 +1,31 @@ package cloudbrain import ( + "regexp" + "strconv" + "strings" + "time" + "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" - "strings" ) -func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ +func GetAiCenterShow(aiCenter string, ctx *context.Context) string { aiCenterInfo := strings.Split(aiCenter, "+") - if len(aiCenterInfo) == 2{ - if setting.C2NetMapInfo!=nil { - if info,ok:=setting.C2NetMapInfo[aiCenterInfo[0]];ok { + if len(aiCenterInfo) == 2 { + if setting.C2NetMapInfo != nil { + if info, ok := setting.C2NetMapInfo[aiCenterInfo[0]]; ok { if ctx.Language() == "zh-CN" { return info.Content } else { return info.ContentEN } - }else{ + } else { return aiCenterInfo[1] } - }else{ + } else { return aiCenterInfo[1] } @@ -29,5 +33,26 @@ func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ return "" +} +func GetDisplayJobName(username string) string { + t := time.Now() + return jobNamePrefixValid(cutString(username, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] +} + +func cutString(str string, lens int) string { + if len(str) < lens { + return str + } + return str[:lens] +} + +func jobNamePrefixValid(s string) string { + lowStr := strings.ToLower(s) + re := regexp.MustCompile(`[^a-z0-9_\\-]+`) + + removeSpecial := re.ReplaceAllString(lowStr, "") + + re = regexp.MustCompile(`^[_\\-]+`) + return re.ReplaceAllString(removeSpecial, "") } From 5e4217242af8ea8cf9d72156196f79436246d4eb Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 10 Nov 2022 16:29:15 +0800 Subject: [PATCH 02/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 58 +++++++++---- modules/setting/setting.go | 9 ++ routers/api/v1/repo/modelarts.go | 7 +- services/cloudbrain/cloudbrainTask/notebook.go | 116 ++++++++++++++----------- 4 files changed, 123 insertions(+), 67 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 65984d95f..1b4d093a1 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -7,11 +7,12 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "path" "strconv" "strings" + "github.com/go-resty/resty/v2" + "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/modelarts_cd" @@ -932,20 +933,12 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { jupyterUrl := result.Url + "?token=" + result.Token client := getRestyClient() - res, err := client.R().Get(jupyterUrl) - codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath - if err != nil { - log.Error("browser jupyterUrl failed:%v", task.DisplayJobName, err) - + cookies, xsrf := getCookiesAndCsrf(client, jupyterUrl) + if xsrf == "" { + log.Error("browser jupyterUrl failed:%v", task.DisplayJobName) } else { - cookies := res.Cookies() - xsrf := "" - for _, cookie := range cookies { - if cookie.Name == "_xsrf" { - xsrf = cookie.Value - } - } + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) if err != nil { log.Error("read jupyter file failed:%v", task.DisplayJobName, err) @@ -953,8 +946,8 @@ func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Resu base64Content := base64.StdEncoding.EncodeToString(fileContents) - uploadUrl := result.Url + "/api/contents/" + path.Base(task.BootFile) - res, err = client.R(). + uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile) + res, err := client.R(). SetCookies(cookies). SetHeader("X-XSRFToken", xsrf). SetBody(map[string]interface{}{ @@ -971,7 +964,40 @@ func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Resu } } - go os.RemoveAll(codePath) + +} + +func getJupyterBaseUrl(url string) string { + jupyterUrlLength := len(url) + baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))] + return baseUrl +} + +func getCookiesAndCsrf(client *resty.Client, jupyterUrl string) ([]*http.Cookie, string) { + + var cookies []*http.Cookie + for i := 0; i < 4; i++ { + res, err := client.R().Get(jupyterUrl) + if err != nil { + log.Error("browser jupyterUrl failed.") + return cookies, "" + } else { + cookies = res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + } + + } + if xsrf != "" { + return cookies, xsrf + } + + } + } + return cookies, "" + } func SyncTempStatusJob() { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a358a85d4..c9dc02fb6 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1587,6 +1587,15 @@ func NewContext() { Course.OrgName = sec.Key("org_name").MustString("") Course.TeamName = sec.Key("team_name").MustString("") + sec = Cfg.Section("file_notebook") + FileNoteBook.ProjectName = sec.Key("project_name").MustString("openi-notebook") + FileNoteBook.ImageIdNPU = sec.Key("imageid_npu").MustString("") + FileNoteBook.ImageNPU = sec.Key("image_npu").MustString("") + FileNoteBook.ImageGPU = sec.Key("image_gpu").MustString("") + FileNoteBook.SpecIdCPU = sec.Key("specid_cpu").MustInt64(-1) + FileNoteBook.SpecIdGPU = sec.Key("specid_gpu").MustInt64(-1) + FileNoteBook.SpecIdNPU = sec.Key("specid_npu").MustInt64(-1) + getGrampusConfig() getModelartsCDConfig() getModelConvertConfig() diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index cd129dd5f..d9426749f 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -39,8 +39,11 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) if err != nil { - ctx.NotFound(err) - return + job, err = models.GetCloudbrainByJobID(ID) + if err != nil { + ctx.NotFound(err) + return + } } err = modelarts.HandleNotebookInfo(job) if err != nil { diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index ea1f8b081..8180a2b0f 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -3,7 +3,7 @@ package cloudbrainTask import ( "fmt" "net/http" - "os" + "path" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts_cd" @@ -28,11 +28,25 @@ import ( "code.gitea.io/gitea/modules/util" ) +const NoteBookExtension = ".ipynb" + func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { if ctx.Written() { return } + + if path.Ext(option.File) != NoteBookExtension { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) + return + } + + isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) + if !isNotebookFileExist { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + //create repo if not exist repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { @@ -51,6 +65,12 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("repo.failed_to_create_repo")) + return + } + if option.Type <= 1 { + cloudBrainFileNoteBookCreate(ctx, option, repo) + } else { + modelartsFileNoteBookCreate(ctx, option, repo) } } @@ -99,40 +119,19 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot return } } - repoPath := models.RepoPath(repo.OwnerName, repo.Name) - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) - return - } - // We opened it, we should close it - defer func() { - // If it's been set to nil then assume someone else has closed it. - if ctx.Repo.GitRepo != nil { - ctx.Repo.GitRepo.Close() - } - }() - ctx.Repo = &context.Repository{ - Repository: repo, - GitRepo: gitRepo, - } - - fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) - if err != nil || !fileExist { - log.Error("Get file error:", err, ctx.Data["MsgID"]) + sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) + if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return } - command := cloudbrain.GetCloudbrainDebugCommand() - - errStr := uploadCodeFile(repo, codePath, option.BranchName, option.File, jobName) + errStr := uploadCodeFile(sourceRepo, codePath, option.BranchName, option.File, jobName) if errStr != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return } - + command := cloudbrain.GetCloudbrainDebugCommand() commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(option.BranchName) specId := setting.FileNoteBook.SpecIdGPU if option.Type == 0 { @@ -242,33 +241,14 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote return } } - repoPath := models.RepoPath(repo.OwnerName, repo.Name) - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) - return - } - // We opened it, we should close it - defer func() { - // If it's been set to nil then assume someone else has closed it. - if ctx.Repo.GitRepo != nil { - ctx.Repo.GitRepo.Close() - } - }() - ctx.Repo = &context.Repository{ - Repository: repo, - GitRepo: gitRepo, - } - - fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) - if err != nil || !fileExist { - log.Error("Get file error:", err, ctx.Data["MsgID"]) + sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) + if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return } - err = downloadCode(repo, getCodePath(jobName), option.BranchName) + err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return @@ -292,6 +272,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) return } + ctx.Repo = &context.Repository{ + Repository: repo, + } var jobId string if setting.ModelartsCD.Enabled { @@ -315,6 +298,30 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } +func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobOption) (bool, error) { + repoPathOfNoteBook := models.RepoPath(option.OwnerName, option.ProjectName) + + gitRepoOfNoteBook, err := git.OpenRepository(repoPathOfNoteBook) + if err != nil { + log.Error("RepoRef Invalid repo "+repoPathOfNoteBook, err.Error()) + return false, err + } + // We opened it, we should close it + defer func() { + // If it's been set to nil then assume someone else has closed it. + if gitRepoOfNoteBook != nil { + gitRepoOfNoteBook.Close() + } + }() + fileExist, err := fileExists(gitRepoOfNoteBook, option.File, option.BranchName) + if err != nil || !fileExist { + log.Error("Get file error:", err, ctx.Data["MsgID"]) + + return false, err + } + return true, nil +} + func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { err := downloadCode(repo, codePath, branchName) if err != nil { @@ -325,6 +332,17 @@ func uploadCodeFile(repo *models.Repository, codePath string, branchName string, if err != nil { return "cloudbrain.load_code_failed" } - go os.RemoveAll(codePath) return "" } + +func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { + + commit, err := gitRepo.GetBranchCommit(branch) + if err != nil { + return false, err + } + if _, err := commit.GetTreeEntryByPath(path); err != nil { + return false, err + } + return true, nil +} From 3129a31307a4337b068eaf17fbba229cde83cd7c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 10 Nov 2022 17:11:26 +0800 Subject: [PATCH 03/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 18 ++++++++++-------- services/cloudbrain/cloudbrainTask/notebook.go | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c9dc02fb6..0131f9062 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -713,13 +713,14 @@ var ( }{} FileNoteBook = struct { - ProjectName string - ImageGPU string - SpecIdGPU int64 - SpecIdCPU int64 - ImageIdNPU string - ImageNPU string - SpecIdNPU int64 + ProjectName string + ImageGPU string + SpecIdGPU int64 + SpecIdCPU int64 + ImageIdNPU string + SpecIdNPU int64 + ImageIdNPUCD string + SpecIdNPUCD int64 }{} ModelConvert = struct { @@ -1590,11 +1591,12 @@ func NewContext() { sec = Cfg.Section("file_notebook") FileNoteBook.ProjectName = sec.Key("project_name").MustString("openi-notebook") FileNoteBook.ImageIdNPU = sec.Key("imageid_npu").MustString("") - FileNoteBook.ImageNPU = sec.Key("image_npu").MustString("") FileNoteBook.ImageGPU = sec.Key("image_gpu").MustString("") FileNoteBook.SpecIdCPU = sec.Key("specid_cpu").MustInt64(-1) FileNoteBook.SpecIdGPU = sec.Key("specid_gpu").MustInt64(-1) FileNoteBook.SpecIdNPU = sec.Key("specid_npu").MustInt64(-1) + FileNoteBook.ImageIdNPUCD = sec.Key("imageid_npu_cd").MustString("") + FileNoteBook.SpecIdNPUCD = sec.Key("specid_npu_cd").MustInt64(-1) getGrampusConfig() getModelartsCDConfig() diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 8180a2b0f..85e582b56 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -255,10 +255,12 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } var aiCenterCode = models.AICenterOfCloudBrainTwo + var specId = setting.FileNoteBook.SpecIdNPU if setting.ModelartsCD.Enabled { aiCenterCode = models.AICenterOfChengdu + specId = setting.FileNoteBook.SpecIdNPUCD } - spec, err := resource.GetAndCheckSpec(ctx.User.ID, setting.FileNoteBook.SpecIdNPU, models.FindSpecsOptions{ + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ JobType: models.JobTypeDebug, ComputeResource: models.NPU, Cluster: models.OpenICluster, @@ -278,7 +280,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote var jobId string if setting.ModelartsCD.Enabled { - jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) + jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPUCD, spec, option.File) } else { jobId, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) } From 5088e8da18dbe9f082405d0db37337a75390180d Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 11 Nov 2022 08:51:20 +0800 Subject: [PATCH 04/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/notebook.go | 4 +--- services/cloudbrain/cloudbrainTask/train.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 85e582b56..e5e5f3596 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -81,8 +81,6 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot jobName := util.ConvertDisplayJobNameToJobName(displayJobName) jobType := string(models.JobTypeDebug) - codePath := getCodePath(jobName) - lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) defer lock.UnLock() isOk, err := lock.Lock(models.CloudbrainKeyDuration) @@ -126,7 +124,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot return } - errStr := uploadCodeFile(sourceRepo, codePath, option.BranchName, option.File, jobName) + errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) if errStr != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 5de69335a..00d01a7ce 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -812,7 +812,7 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { func uploadOneFileToMinio(codePath, filePath, jobName, parentDir string) error { destObject := setting.CBCodePathPrefix + jobName + parentDir + path.Base(filePath) - sourceFile := codePath + filePath + sourceFile := codePath + "/" + filePath err := storage.Attachments.UploadObject(destObject, sourceFile) if err != nil { log.Error("UploadObject(%s) failed: %s", filePath, err.Error()) From 84c4666a31fa6d0c3cb08ac6d792c4eb09f2237c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 11 Nov 2022 09:07:25 +0800 Subject: [PATCH 05/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/notebook.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index e5e5f3596..71aa71770 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -130,7 +130,6 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot return } command := cloudbrain.GetCloudbrainDebugCommand() - commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(option.BranchName) specId := setting.FileNoteBook.SpecIdGPU if option.Type == 0 { specId = setting.FileNoteBook.SpecIdCPU @@ -150,6 +149,9 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) return } + ctx.Repo = &context.Repository{ + Repository: repo, + } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, @@ -170,7 +172,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot BranchName: option.BranchName, BootFile: option.File, Params: "{\"parameter\":[]}", - CommitID: commitID, + CommitID: "", BenchmarkTypeID: 0, BenchmarkChildTypeID: 0, ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), From 9aefd14b9a78abf7a632cd38240bf430a9c06379 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 11 Nov 2022 09:59:12 +0800 Subject: [PATCH 06/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + services/cloudbrain/cloudbrainTask/notebook.go | 38 +++++++++++++++----------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3f0c0a010..c14955dff 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1044,6 +1044,7 @@ model_rename=Duplicate model name, please modify model name. notebook_file_not_exist=Notebook file does not exist. notebook_select_wrong=Please select a Notebook(.ipynb) file first. +notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. date=Date repo_add=Project Increment diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8b635112b..fc0743c59 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1044,6 +1044,7 @@ model_rename=模型名称重复，请修改模型名称 notebook_file_not_exist=Notebook文件不存在。 notebook_select_wrong=请先选择Notebook（.ipynb）文件。 +notebook_file_no_right=您没有这个Notebook文件的读权限。 date=日期 repo_add=新增项目 diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 71aa71770..a598c47df 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -47,6 +47,24 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp return } + sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + permission, err := models.GetUserRepoPermission(sourceRepo, ctx.User) + if err != nil { + log.Error("Get permission failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_no_right"))) + return + } + + if !permission.CanRead(models.UnitTypeCode) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_no_right"))) + return + } + //create repo if not exist repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { @@ -68,14 +86,14 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp return } if option.Type <= 1 { - cloudBrainFileNoteBookCreate(ctx, option, repo) + cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) } else { - modelartsFileNoteBookCreate(ctx, option, repo) + modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) } } -func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { +func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) jobName := util.ConvertDisplayJobNameToJobName(displayJobName) @@ -118,12 +136,6 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } } - sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) - return - } - errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) if errStr != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) @@ -199,7 +211,7 @@ func getDescription(option api.CreateFileNotebookJobOption) string { return option.OwnerName + "/" + option.ProjectName + "/" + option.File } -func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { +func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) jobName := util.ConvertDisplayJobNameToJobName(displayJobName) @@ -242,12 +254,6 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } } - sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) - return - } - err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) From 9d23ea7a4b97303c2fd3fb0409f10cd7e05b698b Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 14 Nov 2022 15:28:59 +0800 Subject: [PATCH 07/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/api.go | 2 +- routers/api/v1/repo/cloudbrain.go | 59 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b7e3c6415..96de00c55 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -737,7 +737,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken(), repoAssignment()) m.Group("/file_notebook", func() { - + m.Get("", reqToken(), repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) }) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 5ea797e97..22b533baa 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -83,6 +83,56 @@ func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJo cloudbrainTask.FileNotebookCreate(ctx.Context, option) } +func GetFileNoteBookInfo(ctx *context.APIContext) { + //image description spec description waiting count + + specs, err := models.GetResourceSpecificationByIds([]int64{setting.FileNoteBook.SpecIdCPU, setting.FileNoteBook.SpecIdGPU, setting.FileNoteBook.SpecIdNPU, setting.FileNoteBook.SpecIdNPUCD}) + if err != nil { + log.Error("Fail to query specifications", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_query_fail"))) + return + } + + var specCPU, specGpu, specNPU, specNPUCD *api.SpecificationShow + var specGpuQueueCode string + for _, spec := range specs { + if spec.ID == setting.FileNoteBook.SpecIdCPU { + specCPU = convert.ToSpecification(spec) + } else if spec.ID == setting.FileNoteBook.SpecIdGPU { + specGpu = convert.ToSpecification(spec) + specGpuQueueCode = spec.QueueCode + } else if spec.ID == setting.FileNoteBook.SpecIdNPU { + specNPU = convert.ToSpecification(spec) + } else if spec.ID == setting.FileNoteBook.SpecIdNPUCD { + specNPUCD = convert.ToSpecification(spec) + } + } + + waitCountNpu := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") + + queuesMap, err := cloudbrain.GetQueuesDetail() + if err != nil { + log.Error("Fail to query gpu queues waiting count", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_query_fail"))) + return + } + waitCountGPU := (*queuesMap)[specGpuQueueCode] + ctx.JSON(http.StatusOK, map[string]interface{}{ + "code": 0, + "specCpu": specCPU, + "specGpu": specGpu, + "specNpu": specNPU, + "specNpuCd": specNPUCD, + "waitCountGpu": waitCountGPU, + "waitCountNpu": waitCountNpu, + "imageCpuDescription": setting.FileNoteBook.ImageCPUDescription, + "imageGpuDescription": setting.FileNoteBook.ImageGPUDescription, + "imageNpuDescription": setting.FileNoteBook.ImageNPUDescription, + "imageNpuCDDescription": setting.FileNoteBook.ImageNPUCDDescription, + }) + +} + func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == cloudbrainTask.TaskTypeCloudbrainOne { cloudbrainTask.CloudbrainOneTrainJobCreate(ctx.Context, option) @@ -146,9 +196,12 @@ func GetCloudbrainTask(ctx *context.APIContext) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) if err != nil { - ctx.NotFound(err) - log.Error("GetCloudbrainByID failed:", err) - return + job, err = models.GetCloudbrainByJobID(ID) + if err != nil { + ctx.NotFound(err) + log.Error("GetCloudbrainByID failed:", err) + return + } } if job.JobType == string(models.JobTypeModelSafety) { routerRepo.GetAiSafetyTaskByJob(job) From 4f2cbc6baf7460bb525b2147bbe8d24a44ccfbf6 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 14 Nov 2022 15:42:58 +0800 Subject: [PATCH 08/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/resource_specification.go | 9 +++++++++ modules/setting/setting.go | 24 ++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/models/resource_specification.go b/models/resource_specification.go index 2da8d015d..3f88fe29f 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -285,6 +285,15 @@ func ResourceSpecOffShelf(id int64) (int64, error) { return n, err } +func GetResourceSpecificationByIds(ids []int64) ([]*Specification, error) { + r := make([]*Specification, 0) + err := x.In("id", ids). + Join("INNER", "resource_queue", "resource_queue.id = resource_specification.queue_id"). + Find(&r) + return r, err + +} + func GetResourceSpecification(r *ResourceSpecification) (*ResourceSpecification, error) { has, err := x.Get(r) if err != nil { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0131f9062..90b86eb4a 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -713,14 +713,18 @@ var ( }{} FileNoteBook = struct { - ProjectName string - ImageGPU string - SpecIdGPU int64 - SpecIdCPU int64 - ImageIdNPU string - SpecIdNPU int64 - ImageIdNPUCD string - SpecIdNPUCD int64 + ProjectName string + ImageGPU string + SpecIdGPU int64 + SpecIdCPU int64 + ImageIdNPU string + SpecIdNPU int64 + ImageIdNPUCD string + SpecIdNPUCD int64 + ImageCPUDescription string + ImageGPUDescription string + ImageNPUDescription string + ImageNPUCDDescription string }{} ModelConvert = struct { @@ -1597,6 +1601,10 @@ func NewContext() { FileNoteBook.SpecIdNPU = sec.Key("specid_npu").MustInt64(-1) FileNoteBook.ImageIdNPUCD = sec.Key("imageid_npu_cd").MustString("") FileNoteBook.SpecIdNPUCD = sec.Key("specid_npu_cd").MustInt64(-1) + FileNoteBook.ImageCPUDescription = sec.Key("image_cpu_desc").MustString("") + FileNoteBook.ImageGPUDescription = sec.Key("image_gpu_desc").MustString("") + FileNoteBook.ImageNPUDescription = sec.Key("image_npu_desc").MustString("") + FileNoteBook.ImageNPUCDDescription = sec.Key("image_npu_cd_desc").MustString("") getGrampusConfig() getModelartsCDConfig() From ab34d3f14446b8a3c8452f333ed7b64685703e49 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Mon, 14 Nov 2022 16:40:12 +0800 Subject: [PATCH 09/72] =?UTF-8?q?#3180=20=E8=AE=AD=E7=BB=83=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E8=AF=A6=E6=83=85=E9=A1=B5=E6=B2=A1=E6=9C=89=E5=B1=95?= =?UTF-8?q?=E7=A4=BA=E4=BB=A3=E7=A0=81=E4=BB=93=E5=AF=B9=E5=BA=94=E7=9A=84?= =?UTF-8?q?=E7=89=88=E6=9C=AC=EF=BC=8C=E5=8F=AA=E6=9C=89=E5=88=86=E6=94=AF?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/repo/cloudbrain/inference/show.tmpl | 1 + templates/repo/cloudbrain/show.tmpl | 1 + templates/repo/cloudbrain/trainjob/show.tmpl | 1 + templates/repo/grampus/trainjob/show.tmpl | 1 + templates/repo/modelarts/inferencejob/show.tmpl | 1 + templates/repo/modelarts/trainjob/show.tmpl | 1 + templates/repo/modelsafety/show.tmpl | 1 + 7 files changed, 7 insertions(+) diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl index 3154b8ac6..11ed2f0c5 100644 --- a/templates/repo/cloudbrain/inference/show.tmpl +++ b/templates/repo/cloudbrain/inference/show.tmpl @@ -446,6 +446,7 @@
{{.BranchName}} + {{SubStr .CommitID 0 10}}
diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index cca85fb32..27c3cc556 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -337,6 +337,7 @@
{{.BranchName}} + {{SubStr .CommitID 0 10}}
diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index ab6c40ead..c06df57c2 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -412,6 +412,7 @@
{{.BranchName}} + {{SubStr .CommitID 0 10}}
diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index e8d4082f4..9d46a5256 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -422,6 +422,7 @@
{{.BranchName}} + {{SubStr .CommitID 0 10}}
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl index 29bd1aaf3..5fa07f067 100644 --- a/templates/repo/modelarts/inferencejob/show.tmpl +++ b/templates/repo/modelarts/inferencejob/show.tmpl @@ -388,6 +388,7 @@ td, th {
{{.BranchName}} + {{SubStr .CommitID 0 10}}
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index d615f90a8..bbe2d52e7 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -442,6 +442,7 @@
{{.BranchName}} + {{SubStr .CommitID 0 10}}
diff --git a/templates/repo/modelsafety/show.tmpl b/templates/repo/modelsafety/show.tmpl index a46077443..2090441c9 100644 --- a/templates/repo/modelsafety/show.tmpl +++ b/templates/repo/modelsafety/show.tmpl @@ -833,6 +833,7 @@ $(`[vfield="Description"]`).text(res['Description'] || '--'); $(`[vfield="Parameters"]`).text(res['Parameters'] || '--'); + $(`[vfield="BranchName"]`).html(res['BranchName'] + '' + res['CommitID'].slice(0, 10) + ''); var imageName = res['Image'] || res['EngineName']; $(`[vimagetitle="Image"] span`).hide(); From 0c78e3bc787a941ae208273f67eecafb53d8af9c Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 15 Nov 2022 10:37:50 +0800 Subject: [PATCH 10/72] fix issue --- templates/repo/home.tmpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index 7c44aa147..c938f0200 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -200,6 +200,12 @@ class="section">{{EllipsisString $v 30}}{{end}}{{end}} {{end}} +
+ +
{{if .Repository.CanEnableEditor}} @@ -223,6 +229,7 @@ {{end}}
+
From 464de2a6afd966dcf8005ce2c977a03281fd4bd7 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 15 Nov 2022 17:44:25 +0800 Subject: [PATCH 11/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/repo/cloudbrain.go | 42 ++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 22b533baa..53eaf28c6 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -117,19 +117,33 @@ func GetFileNoteBookInfo(ctx *context.APIContext) { return } waitCountGPU := (*queuesMap)[specGpuQueueCode] - ctx.JSON(http.StatusOK, map[string]interface{}{ - "code": 0, - "specCpu": specCPU, - "specGpu": specGpu, - "specNpu": specNPU, - "specNpuCd": specNPUCD, - "waitCountGpu": waitCountGPU, - "waitCountNpu": waitCountNpu, - "imageCpuDescription": setting.FileNoteBook.ImageCPUDescription, - "imageGpuDescription": setting.FileNoteBook.ImageGPUDescription, - "imageNpuDescription": setting.FileNoteBook.ImageNPUDescription, - "imageNpuCDDescription": setting.FileNoteBook.ImageNPUCDDescription, - }) + if !setting.ModelartsCD.Enabled{ + ctx.JSON(http.StatusOK, map[string]interface{}{ + "code": 0, + "specCpu": specCPU, + "specGpu": specGpu, + "specNpu": specNPU, + "waitCountGpu": waitCountGPU, + "waitCountNpu": waitCountNpu, + "imageCpuDescription": setting.FileNoteBook.ImageCPUDescription, + "imageGpuDescription": setting.FileNoteBook.ImageGPUDescription, + "imageNpuDescription": setting.FileNoteBook.ImageNPUDescription, + + }) + } else{ + ctx.JSON(http.StatusOK, map[string]interface{}{ + "code": 0, + "specCpu": specCPU, + "specGpu": specGpu, + "specNpu": specNPUCD, + "waitCountGpu": waitCountGPU, + "waitCountNpu": waitCountNpu, + "imageCpuDescription": setting.FileNoteBook.ImageCPUDescription, + "imageGpuDescription": setting.FileNoteBook.ImageGPUDescription, + "imageNpuDescription": setting.FileNoteBook.ImageNPUCDDescription, + }) + + } } @@ -690,7 +704,7 @@ func CloudbrainGetLog(ctx *context.APIContext) { endLine += 1 } } - + result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath) if result == nil { log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"]) From 7452b22d5af9876041f816aff7fd2dc95e35571e Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 15 Nov 2022 18:22:05 +0800 Subject: [PATCH 12/72] fix issue --- templates/repo/home.tmpl | 5 +- web_src/vuepages/apis/modules/notobook.js | 41 +++ web_src/vuepages/pages/notebook/debug/index.vue | 368 +++++++++++++++++++++ .../pages/notebook/debug/vp-notebook-debug.js | 11 + 4 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 web_src/vuepages/apis/modules/notobook.js create mode 100644 web_src/vuepages/pages/notebook/debug/index.vue create mode 100644 web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index c938f0200..23e380e37 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -99,6 +99,7 @@ animation-fill-mode: both; } +
{{template "repo/header" .}}
@@ -201,7 +202,7 @@ title="{{$v}}">{{EllipsisString $v 30}}{{end}}{{end}}
{{end}}
- @@ -400,6 +401,7 @@
+
+ {{template "base/footer" .}} \ No newline at end of file diff --git a/web_src/vuepages/apis/modules/notobook.js b/web_src/vuepages/apis/modules/notobook.js new file mode 100644 index 000000000..a13b46558 --- /dev/null +++ b/web_src/vuepages/apis/modules/notobook.js @@ -0,0 +1,41 @@ +import service from '../service'; + +// Notebook新建页面需要的信息 +export const getFileNotebook = () => { + return service({ + url: '/api/v1/file_notebook', + method: 'get', + params: {}, + }); +} + +// 算力积分获取、消耗明细 +// operate-INCREASE 表示获取明细 DECREASE表示消耗明细, page-当前页, pageSize-每页条数 +export const getPointList = (params) => { + return service({ + url: '/reward/point/record/list', + method: 'get', + params, + }); +} + +// 管理员充值、扣减用户积分 +// TargetUserId, OperateType-INCREASE,DECREASE, Amount, Remark, RewardType-POINT +export const setPointOperate = (data) => { + return service({ + url: '/operation/reward/point/account/operate', + method: 'post', + data, + params: {} + }); +} + +// 算力积分页面 +export const getPoint = () => { + return service({ + url: '/reward/point', + method: 'get', + params: {}, + data: {}, + }); +} diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue new file mode 100644 index 000000000..a4b95f18b --- /dev/null +++ b/web_src/vuepages/pages/notebook/debug/index.vue @@ -0,0 +1,368 @@ + + + diff --git a/web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js b/web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js new file mode 100644 index 000000000..b92aef211 --- /dev/null +++ b/web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js @@ -0,0 +1,11 @@ +import Vue from 'vue'; +import {Dialog} from 'element-ui'; +import 'element-ui/lib/theme-chalk/index.css'; +import { i18n, lang } from '~/langs'; +import App from './index.vue'; + +Vue.use(Dialog) +new Vue({ + i18n, + render: (h) => h(App), +}).$mount('#__vue-root'); From 675261f2d452a01dae3bcb9a6fcb86c64723848f Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 16 Nov 2022 10:23:40 +0800 Subject: [PATCH 13/72] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/resource_specification.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/resource_specification.go b/models/resource_specification.go index 3f88fe29f..e04cc6ed9 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -287,7 +287,7 @@ func ResourceSpecOffShelf(id int64) (int64, error) { func GetResourceSpecificationByIds(ids []int64) ([]*Specification, error) { r := make([]*Specification, 0) - err := x.In("id", ids). + err := x.In("resource_specification.id", ids). Join("INNER", "resource_queue", "resource_queue.id = resource_specification.queue_id"). Find(&r) return r, err From 4be102021bfd556d1ce0218ad7e9d3babce3ef6b Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 16 Nov 2022 16:03:32 +0800 Subject: [PATCH 14/72] #3027 add repo watch notify type --- models/repo.go | 2 +- models/repo_watch.go | 33 ++++++++++++++++++++++++++++++--- modules/context/repo.go | 1 + routers/repo/repo.go | 4 +++- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/models/repo.go b/models/repo.go index f0760108b..832e3fc37 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1279,7 +1279,7 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository, opts ...Cr } if setting.Service.AutoWatchNewRepos { - if err = watchRepo(ctx.e, doer.ID, repo.ID, true); err != nil { + if err = watchRepo(ctx.e, doer.ID, repo.ID, true, ReceiveAllNotification); err != nil { return fmt.Errorf("watchRepo: %v", err) } } diff --git a/models/repo_watch.go b/models/repo_watch.go index 573a2d78a..7c43ee352 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -24,6 +24,14 @@ const ( RepoWatchModeAuto // 3 ) +// NotifyType specifies what kind of watch the user has on a repository +type NotifyType int8 + +const ( + RejectAllNotification NotifyType = 0 + ReceiveAllNotification NotifyType = 9 +) + var ActionChan = make(chan *Action, 200) var ActionChan4Task = make(chan Action, 200) @@ -34,6 +42,7 @@ type Watch struct { RepoID int64 `xorm:"UNIQUE(watch)"` Mode RepoWatchMode `xorm:"SMALLINT NOT NULL DEFAULT 1"` CreatedUnix int64 `xorm:"created"` + NotifyType NotifyType `xorm:"SMALLINT NOT NULL DEFAULT 0"` } // getWatch gets what kind of subscription a user has on a given repository; returns dummy record if none found @@ -60,8 +69,20 @@ func IsWatching(userID, repoID int64) bool { return err == nil && isWatchMode(watch.Mode) } +// GetWatchNotifyType +func GetWatchNotifyType(userID, repoID int64) NotifyType { + watch, err := getWatch(x, userID, repoID) + if err != nil { + return RejectAllNotification + } + return watch.NotifyType +} + func watchRepoMode(e Engine, watch Watch, mode RepoWatchMode) (err error) { if watch.Mode == mode { + if _, err := e.ID(watch.ID).Cols("notify_type").Update(watch); err != nil { + return err + } return nil } if mode == RepoWatchModeAuto && (watch.Mode == RepoWatchModeDont || isWatchMode(watch.Mode)) { @@ -109,7 +130,7 @@ func WatchRepoMode(userID, repoID int64, mode RepoWatchMode) (err error) { return watchRepoMode(x, watch, mode) } -func watchRepo(e Engine, userID, repoID int64, doWatch bool) (err error) { +func watchRepo(e Engine, userID, repoID int64, doWatch bool, notifyTypes ...NotifyType) (err error) { var watch Watch if watch, err = getWatch(e, userID, repoID); err != nil { return err @@ -119,14 +140,19 @@ func watchRepo(e Engine, userID, repoID int64, doWatch bool) (err error) { } else if !doWatch { err = watchRepoMode(e, watch, RepoWatchModeNone) } else { + notifyType := RejectAllNotification + if len(notifyTypes) > 0 { + notifyType = notifyTypes[0] + } + watch.NotifyType = notifyType err = watchRepoMode(e, watch, RepoWatchModeNormal) } return err } // WatchRepo watch or unwatch repository. -func WatchRepo(userID, repoID int64, watch bool) (err error) { - return watchRepo(x, userID, repoID, watch) +func WatchRepo(userID, repoID int64, watch bool, notifyType ...NotifyType) (err error) { + return watchRepo(x, userID, repoID, watch, notifyType...) } func getWatchers(e Engine, repoID int64) ([]*Watch, error) { @@ -156,6 +182,7 @@ func getRepoWatchersIDs(e Engine, repoID int64) ([]int64, error) { return ids, e.Table("watch"). Where("watch.repo_id=?", repoID). And("watch.mode<>?", RepoWatchModeDont). + And("watch.notify_type > ?", RejectAllNotification). Select("user_id"). Find(&ids) } diff --git a/modules/context/repo.go b/modules/context/repo.go index 7c425c8c0..3bdc34f0d 100755 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -474,6 +474,7 @@ func RepoAssignment() macaron.Handler { if ctx.IsSigned { ctx.Data["IsWatchingRepo"] = models.IsWatching(ctx.User.ID, repo.ID) + ctx.Data["WatchNotifyType"] = models.GetWatchNotifyType(ctx.User.ID, repo.ID) ctx.Data["IsStaringRepo"] = models.IsStaring(ctx.User.ID, repo.ID) ctx.Data["IsStaringDataset"] = models.IsDatasetStaringByRepoId(ctx.User.ID, repo.ID) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 2c8c2f45b..4919b2487 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -414,7 +414,9 @@ func Action(ctx *context.Context) { var err error switch ctx.Params(":action") { case "watch": - err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, true) + err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, true, models.ReceiveAllNotification) + case "watch_but_reject": + err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, true, models.RejectAllNotification) case "unwatch": err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, false) case "star": From 9c6a1b6e2e639ac58aea5afd49eb093f1f11aec9 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 17 Nov 2022 09:54:56 +0800 Subject: [PATCH 15/72] fix issue --- templates/repo/home.tmpl | 5 + web_src/vuepages/apis/modules/notobook.js | 48 ++++--- web_src/vuepages/apis/service.js | 1 - web_src/vuepages/pages/notebook/debug/index.vue | 163 ++++++++++++++++++++---- 4 files changed, 163 insertions(+), 54 deletions(-) diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index 23e380e37..e88f6f52d 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -402,6 +402,7 @@
+
{{template "base/footer" .}} \ No newline at end of file diff --git a/web_src/vuepages/apis/modules/notobook.js b/web_src/vuepages/apis/modules/notobook.js index a13b46558..31e2209c7 100644 --- a/web_src/vuepages/apis/modules/notobook.js +++ b/web_src/vuepages/apis/modules/notobook.js @@ -1,41 +1,39 @@ -import service from '../service'; +import service from "../service"; // Notebook新建页面需要的信息 export const getFileNotebook = () => { return service({ - url: '/api/v1/file_notebook', - method: 'get', + url: "/api/v1/file_notebook", + method: "get", params: {}, }); -} +}; -// 算力积分获取、消耗明细 -// operate-INCREASE 表示获取明细 DECREASE表示消耗明细, page-当前页, pageSize-每页条数 -export const getPointList = (params) => { +// Notebook新建调试任务 +// type, file, branch_name, owner_name, project_name +export const createNotebook = (data) => { return service({ - url: '/reward/point/record/list', - method: 'get', - params, + url: "/api/v1/file_notebook/create", + method: "post", + data, + params: {}, }); -} +}; -// 管理员充值、扣减用户积分 -// TargetUserId, OperateType-INCREASE,DECREASE, Amount, Remark, RewardType-POINT -export const setPointOperate = (data) => { +// Notebook新建页面需要的信息 +export const getCb1Notebook = (jobid) => { return service({ - url: '/operation/reward/point/account/operate', - method: 'post', - data, - params: {} + url: `/api/v1/cloudbrain/${jobid}`, + method: "get", + params: {}, }); -} +}; -// 算力积分页面 -export const getPoint = () => { +// Notebook新建页面需要的信息 +export const getCb2Notebook = (jobid) => { return service({ - url: '/reward/point', - method: 'get', + url: `/api/v1/modelarts/notebook/${jobid}`, + method: "get", params: {}, - data: {}, }); -} +}; diff --git a/web_src/vuepages/apis/service.js b/web_src/vuepages/apis/service.js index 292b9ef78..c676a7d18 100644 --- a/web_src/vuepages/apis/service.js +++ b/web_src/vuepages/apis/service.js @@ -2,7 +2,6 @@ import axios from 'axios'; const service = axios.create({ baseURL: '/', - timeout: 20000, }); service.interceptors.request.use((config) => { diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue index a4b95f18b..9b375b5b8 100644 --- a/web_src/vuepages/pages/notebook/debug/index.vue +++ b/web_src/vuepages/pages/notebook/debug/index.vue @@ -5,12 +5,13 @@ :visible.sync="dialogVisible" width="50%" :close-on-click-modal="false" + @closed="handleClose" >
@@ -19,14 +20,14 @@
CPU 环境
- 规格：CPU: 4，内存: 32GB，共享内存: 16GB + {{cpuSpec}}
- 镜像： ubuntuxx. xx-py37-torch1.9.1 + 镜像：{{notebookInfo.imageCpuDescription}}
- +
-
- N +
+ G
-
CPU 环境
+
GPU 环境
- 规格：CPU: 4，内存: 32GB，共享内存: 16GB + {{gpuSpec}}
- 镜像： ubuntuxx. xx-py37-torch1.9.1 + 镜像：{{notebookInfo.imageGpuDescription}}
- + +
-
- G +
+ N
-
CPU 环境
+
NPU 环境
- 规格：CPU: 4，内存: 32GB，共享内存: 16GB + {{npuSpec}}
- 镜像： ubuntuxx. xx-py37-torch1.9.1 + 镜像：{{notebookInfo.imageCpuDescription}}
- +
-
- +
+ + + + 当前有个{{waitCount}}任务在排队 +
+
+ + + + 当前有个{{waitCount}}任务在排队 +
+
+ + +
+ + +
+ 当前有个{{waitCount}}任务在排队
@@ -98,24 +118,35 @@
+{{if and (.IsViewFile) (IpynbBool .TreePath)}} +{{end}} {{template "base/footer" .}} \ No newline at end of file diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js index bcd1cfe43..3ad30b5e8 100644 --- a/web_src/vuepages/langs/config/en-US.js +++ b/web_src/vuepages/langs/config/en-US.js @@ -192,7 +192,34 @@ const en = { dataDesensitizationModelExperience:'Data desensitization model experience', dataDesensitizationModelDesc:'Use AI technology to desensitize the face and license plate number in the picture. For more information about this model, please visit the project', limitFilesUpload:'Only jpg/jpeg/png files can be uploaded', - limitSizeUpload:'The size of the uploaded file cannot exceed 20M!', + limitSizeUpload: 'The size of the uploaded file cannot exceed 20M!', + notebook: { + createNewNotebook: "Create new notebook debug task", + sameTaskTips1: "You have created an", + sameTaskTips2: "equivalent task", + sameTaskTips3: "that is waiting or running, please wait for the task to finish before creating it.", + sameTaskTips4: "You can view all your Cloudbrain tasks in", + sameTaskTips5: "Home", + sameTaskTips6: "Cloudbrain Task", + sameTaskTips7: "", + cpuEnv: "CPU Environment", + gpuEnv: "GPU Environment", + npuEnv: "NPU environment", + newTask: "Create new task", + noQuene: "The current resources are sufficient and no tasks are queued", + queneTips1: "There are currently", + queneTips2: "tasks queued", + watiResource: "Waiting for resources to be allocated, please be patient", + debug: "Debug", + stop: "Stop", + stopping: "Stopping", + notebookRunning: "The debugging environment has been started, and it will automatically close after a single connection for 4 hours or no operation for half an hour", + stopSuccess: "Stop task succeeded", + specification: "Specification", + graphicMemory: "Graphic Memory", + memory: "Memory", + sharedMemory: "Shared Memory", + }, } export default en; diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js index 5b8e800d8..835beb3e0 100644 --- a/web_src/vuepages/langs/config/zh-CN.js +++ b/web_src/vuepages/langs/config/zh-CN.js @@ -1,198 +1,226 @@ const zh = { - loading: '加载中...', - noData: '暂无数据', - date: '日期', + loading: "加载中...", + noData: "暂无数据", + date: "日期", - confirm: '确定', - cancel: '取消', - confirm1: '确认', - pleaseCompleteTheInformationFirst: '请先完善信息！', - submittedSuccessfully: '提交成功！', - submittedFailed: '提交失败！', - operation: '操作', - edit: '修改', - delete: '删除', - tips: '提示', + confirm: "确定", + cancel: "取消", + confirm1: "确认", + pleaseCompleteTheInformationFirst: "请先完善信息！", + submittedSuccessfully: "提交成功！", + submittedFailed: "提交失败！", + operation: "操作", + edit: "修改", + delete: "删除", + tips: "提示", - accomplishTask: '积分任务', - adminOperate: '管理员操作', - runCloudBrainTask: '运行云脑任务', - operating: '消耗中', - succeeded: '已完成', - debugTask: '调试任务', - trainTask: '训练任务', - inferenceTask: '推理任务', - benchmarkTask: '评测任务', - createPublicProject: '创建公开项目', - dailyPutforwardTasks: '每日提出任务', - dailyPR: '每日提出PR', - comment: '发表评论', - uploadDatasetFile: '上传数据集文件', - importNewModel: '导入新模型', - completeWechatCodeScanningVerification: '完成微信扫码验证', - dailyRunCloudbrainTasks: '每日运行云脑任务', - datasetRecommendedByThePlatform: '数据集被平台推荐', - submitNewPublicImage: '提交新公开镜像', - imageRecommendedByThePlatform: '镜像被平台推荐', - firstChangeofAvatar: '首次更换头像', - dailyCommit: '每日commit', - calcPointDetails: '算力积分明细', - calcPointAcquisitionInstructions: '积分获取说明', - CurrAvailableCalcPoints: '当前可用算力积分(分)', - totalGainCalcPoints: '总获取算力积分(分)', - totalConsumeCalcPoints: '总消耗算力积分(分)', - gainDetail: '获取明细', - consumeDetail: '消耗明细', - serialNumber: '流水号', - time: '时间', - scene: '场景', - behaviorOfPoint: '积分行为', - explanation: '说明', - points: '积分', - status: '状态', - runTime: '运行时长', - taskName: '任务名称', + accomplishTask: "积分任务", + adminOperate: "管理员操作", + runCloudBrainTask: "运行云脑任务", + operating: "消耗中", + succeeded: "已完成", + debugTask: "调试任务", + trainTask: "训练任务", + inferenceTask: "推理任务", + benchmarkTask: "评测任务", + createPublicProject: "创建公开项目", + dailyPutforwardTasks: "每日提出任务", + dailyPR: "每日提出PR", + comment: "发表评论", + uploadDatasetFile: "上传数据集文件", + importNewModel: "导入新模型", + completeWechatCodeScanningVerification: "完成微信扫码验证", + dailyRunCloudbrainTasks: "每日运行云脑任务", + datasetRecommendedByThePlatform: "数据集被平台推荐", + submitNewPublicImage: "提交新公开镜像", + imageRecommendedByThePlatform: "镜像被平台推荐", + firstChangeofAvatar: "首次更换头像", + dailyCommit: "每日commit", + calcPointDetails: "算力积分明细", + calcPointAcquisitionInstructions: "积分获取说明", + CurrAvailableCalcPoints: "当前可用算力积分(分)", + totalGainCalcPoints: "总获取算力积分(分)", + totalConsumeCalcPoints: "总消耗算力积分(分)", + gainDetail: "获取明细", + consumeDetail: "消耗明细", + serialNumber: "流水号", + time: "时间", + scene: "场景", + behaviorOfPoint: "积分行为", + explanation: "说明", + points: "积分", + status: "状态", + runTime: "运行时长", + taskName: "任务名称", - createdRepository: '创建了项目', - repositoryWasDel: '项目已删除', - openedIssue: '创建了任务', - createdPullRequest: '创建了合并请求', - commentedOnIssue: '评论了任务', - uploadDataset: '上传了数据集文件', - createdNewModel: '导入了新模型', - firstBindingWechatRewards: '首次绑定微信奖励', - created: '创建了', - type: '类型', - dataset: '数据集', - setAsRecommendedDataset: '被设置为推荐数据集', - committedImage: '提交了镜像', - image: '镜像', - setAsRecommendedImage: '被设置为推荐镜像', - updatedAvatar: '更新了头像', - pushedBranch: '推送了{branch}分支代码到', - deleteBranch: '从{repo}删除分支{branch}', - pushedTag: '推送了标签{tag}到', - deleteTag: '从{repo}删除了标签{tag}', - dailyMaxTips: '达到每日上限积分，不能拿满分', - memory: '内存', - sharedMemory: '共享内存', - ';': '；', + createdRepository: "创建了项目", + repositoryWasDel: "项目已删除", + openedIssue: "创建了任务", + createdPullRequest: "创建了合并请求", + commentedOnIssue: "评论了任务", + uploadDataset: "上传了数据集文件", + createdNewModel: "导入了新模型", + firstBindingWechatRewards: "首次绑定微信奖励", + created: "创建了", + type: "类型", + dataset: "数据集", + setAsRecommendedDataset: "被设置为推荐数据集", + committedImage: "提交了镜像", + image: "镜像", + setAsRecommendedImage: "被设置为推荐镜像", + updatedAvatar: "更新了头像", + pushedBranch: "推送了{branch}分支代码到", + deleteBranch: "从{repo}删除分支{branch}", + pushedTag: "推送了标签{tag}到", + deleteTag: "从{repo}删除了标签{tag}", + dailyMaxTips: "达到每日上限积分，不能拿满分", + memory: "内存", + sharedMemory: "共享内存", + ";": "；", - noPointGainRecord: '还没有积分获取记录', - noPointConsumeRecord: '还没有积分消耗记录', + noPointGainRecord: "还没有积分获取记录", + noPointConsumeRecord: "还没有积分消耗记录", resourcesManagement: { - OpenI: '启智集群', - C2Net: '智算集群', - OpenIOne: '云脑一', - OpenITwo: '云脑二', - OpenIChengdu: '启智成都智算', - chengduCenter: '成都智算', - pclcci: '鹏城云计算所', - hefeiCenter: '合肥类脑类脑智能开放平台', - xuchangCenter: '中原人工智能计算中心', - willOnShelf: '待上架', - onShelf: '已上架', - offShelf: '已下架', - toOnShelf: '上架', - toOffShelf: '下架', - toSetPriceAndOnShelf: '定价上架', - status: '状态', - allStatus: '全部状态', - syncAiNetwork: '同步智算网络', - resQueue: '资源池（队列）', - allResQueue: '全部资源池（队列）', - addResQueue: '新建资源池（队列）', - addResQueueBtn: '新增资源池', - editResQueue: '修改资源池（队列）', - resQueueName: '资源池（队列）名称', - whichCluster: '所属集群', - allCluster: '全部集群', - aiCenter: '智算中心', - aiCenterID: '智算中心ID', - allAiCenter: '全部智算中心', - computeResource: '计算资源', - allComputeResource: '全部计算资源', - accCardType: '卡类型', - allAccCardType: '全部卡类型', - cardsTotalNum: '卡数', - accCardsNum: '卡数', - remark: '备注', - pleaseEnterRemark: '请输入备注(最大长度不超过255)', - pleaseEnterPositiveIntegerCardsTotalNum: '请输入正整数的卡数！', - addResSpecificationAndPriceInfo: '新增资源规格和单价信息', - addResSpecificationBtn: '新增资源规格', - editResSpecificationAndPriceInfo: '修改资源规格和单价信息', - resSpecificationAndPriceManagement: '资源规格单价管理', - sourceSpecCode: '对应资源编码', - sourceSpecCodeTips: '云脑II需要填写对应的资源编码', - sourceSpecId: '智算网络资源规格ID', - cpuNum: 'CPU数', - gpuMem: '显存', - mem: '内存', - shareMem: '共享内存', - unitPrice: '单价', - point_hr: '积分/时', - node: '节点', - free: '免费', - onShelfConfirm: '请确认上架该规格？', - offShelfConfirm: '请确认下架该规格？', - onShelfCode1001: '上架失败，资源池（队列）不可用。', - onShelfCode1003: '上架失败，资源规格不可用。', - offShelfDlgTip1: '当前资源规格已在以下场景中使用：', - offShelfDlgTip2: '请确认进行下架操作？', - resSceneManagement: '算力资源应用场景管理', - addResScene: '新建算力资源应用场景', - addResSceneBtn: '新增应用场景', - editResScene: '修改算力资源应用场景', - resSceneName: '应用场景名称', - jobType: '任务类型', - allJobType: '全部任务类型', - isExclusive: '是否专属', - allExclusiveAndCommonUse: '全部专属和通用', - exclusive: '专属', - commonUse: '通用', - exclusiveOrg: '专属组织', - exclusiveOrgTips: '多个组织名之间用英文分号隔开', - computeCluster: '算力集群', - resourceSpecification: '资源规格', - lastUpdateTime: '最后更新时间', - resSceneDeleteConfirm: '是否确认删除当前应用场景？', - resourceSpecificationIsAvailable: '资源规格是否可用', - resourceSpecificationIsAvailableAll: '资源规格是否可用（全部）', - available: '可用', - notAvailable: '不可用', + OpenI: "启智集群", + C2Net: "智算集群", + OpenIOne: "云脑一", + OpenITwo: "云脑二", + OpenIChengdu: "启智成都智算", + chengduCenter: "成都智算", + pclcci: "鹏城云计算所", + hefeiCenter: "合肥类脑类脑智能开放平台", + xuchangCenter: "中原人工智能计算中心", + willOnShelf: "待上架", + onShelf: "已上架", + offShelf: "已下架", + toOnShelf: "上架", + toOffShelf: "下架", + toSetPriceAndOnShelf: "定价上架", + status: "状态", + allStatus: "全部状态", + syncAiNetwork: "同步智算网络", + resQueue: "资源池（队列）", + allResQueue: "全部资源池（队列）", + addResQueue: "新建资源池（队列）", + addResQueueBtn: "新增资源池", + editResQueue: "修改资源池（队列）", + resQueueName: "资源池（队列）名称", + whichCluster: "所属集群", + allCluster: "全部集群", + aiCenter: "智算中心", + aiCenterID: "智算中心ID", + allAiCenter: "全部智算中心", + computeResource: "计算资源", + allComputeResource: "全部计算资源", + accCardType: "卡类型", + allAccCardType: "全部卡类型", + cardsTotalNum: "卡数", + accCardsNum: "卡数", + remark: "备注", + pleaseEnterRemark: "请输入备注(最大长度不超过255)", + pleaseEnterPositiveIntegerCardsTotalNum: "请输入正整数的卡数！", + addResSpecificationAndPriceInfo: "新增资源规格和单价信息", + addResSpecificationBtn: "新增资源规格", + editResSpecificationAndPriceInfo: "修改资源规格和单价信息", + resSpecificationAndPriceManagement: "资源规格单价管理", + sourceSpecCode: "对应资源编码", + sourceSpecCodeTips: "云脑II需要填写对应的资源编码", + sourceSpecId: "智算网络资源规格ID", + cpuNum: "CPU数", + gpuMem: "显存", + mem: "内存", + shareMem: "共享内存", + unitPrice: "单价", + point_hr: "积分/时", + node: "节点", + free: "免费", + onShelfConfirm: "请确认上架该规格？", + offShelfConfirm: "请确认下架该规格？", + onShelfCode1001: "上架失败，资源池（队列）不可用。", + onShelfCode1003: "上架失败，资源规格不可用。", + offShelfDlgTip1: "当前资源规格已在以下场景中使用：", + offShelfDlgTip2: "请确认进行下架操作？", + resSceneManagement: "算力资源应用场景管理", + addResScene: "新建算力资源应用场景", + addResSceneBtn: "新增应用场景", + editResScene: "修改算力资源应用场景", + resSceneName: "应用场景名称", + jobType: "任务类型", + allJobType: "全部任务类型", + isExclusive: "是否专属", + allExclusiveAndCommonUse: "全部专属和通用", + exclusive: "专属", + commonUse: "通用", + exclusiveOrg: "专属组织", + exclusiveOrgTips: "多个组织名之间用英文分号隔开", + computeCluster: "算力集群", + resourceSpecification: "资源规格", + lastUpdateTime: "最后更新时间", + resSceneDeleteConfirm: "是否确认删除当前应用场景？", + resourceSpecificationIsAvailable: "资源规格是否可用", + resourceSpecificationIsAvailableAll: "资源规格是否可用（全部）", + available: "可用", + notAvailable: "不可用", }, user: { - inviteFriends: '邀请好友', - inviteFriendsTips: '复制二维码或者注册邀请链接分享给好友', - clickToViewTheEventDetails: '点击查看活动详情', - copyRegistrationInvitationLink: '复制注册邀请链接', - registrationAdress: '注册地址：', - recommender: '推荐人：', - invitedFriends: '已邀请好友', - registrationTime: '注册时间', - theSharedContentHasBeenCopiedToTheClipboard: '分享内容已复制到剪切板', - copyError: '复制错误', - Activated: '已激活', - notActive: '未激活', + inviteFriends: "邀请好友", + inviteFriendsTips: "复制二维码或者注册邀请链接分享给好友", + clickToViewTheEventDetails: "点击查看活动详情", + copyRegistrationInvitationLink: "复制注册邀请链接", + registrationAdress: "注册地址：", + recommender: "推荐人：", + invitedFriends: "已邀请好友", + registrationTime: "注册时间", + theSharedContentHasBeenCopiedToTheClipboard: "分享内容已复制到剪切板", + copyError: "复制错误", + Activated: "已激活", + notActive: "未激活", }, - tranformImageFailed:'图片脱敏失败', - originPicture:'原始图片', - desensitizationPicture:'脱敏图片', - desensitizationObject:'脱敏对象', - example:'示例', - startDesensitization:'开始处理', - all:'全部', - onlyFace:'仅人脸', - onlyLicensePlate:'仅车牌', - dragThePictureHere:'拖动图片到这里', - or:'或', - clickUpload:'点击上传', - dataDesensitizationModelExperience:'数据脱敏模型体验', - dataDesensitizationModelDesc:'利用人工智能AI技术，把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目', - limitFilesUpload:'只能上传 jpg/jpeg/png 格式的文件', - limitSizeUpload:'上传文件大小不能超过 20M !', -} + tranformImageFailed: "图片脱敏失败", + originPicture: "原始图片", + desensitizationPicture: "脱敏图片", + desensitizationObject: "脱敏对象", + example: "示例", + startDesensitization: "开始处理", + all: "全部", + onlyFace: "仅人脸", + onlyLicensePlate: "仅车牌", + dragThePictureHere: "拖动图片到这里", + or: "或", + clickUpload: "点击上传", + dataDesensitizationModelExperience: "数据脱敏模型体验", + dataDesensitizationModelDesc: + "利用人工智能AI技术，把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目", + limitFilesUpload: "只能上传 jpg/jpeg/png 格式的文件", + limitSizeUpload: "上传文件大小不能超过 20M !", + notebook: { + createNewNotebook: "新建Notebook调试任务", + sameTaskTips1: "您已经有", + sameTaskTips2: "同类任务", + sameTaskTips3: "正在等待或运行中，请等待任务结束再创建", + sameTaskTips4: "可以在", + sameTaskTips5: "个人中心", + sameTaskTips6: "云脑任务", + sameTaskTips7: "查看您所有的云脑任务。", + cpuEnv: "CPU 环境", + gpuEnv: "GPU 环境", + npuEnv: "NPU 环境", + newTask: "新建任务", + noQuene: "当前资源充足，没有任务排队", + queneTips1: "当前有", + queneTips2: "个任务正在排队", + watiResource: "正在等待分配资源，请耐心等待", + debug: "调试", + stop: "停止", + stopping: "停止中", + notebookRunning: "调试环境已启动，单次连接 4 小时或无操作半小时后自动关闭", + stopSuccess: "停止任务成功", + specification: "规格", + graphicMemory: "显存", + memory: "内存", + sharedMemory: "共享内存", + }, +}; export default zh; diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue index 5faee62a4..b305816cb 100644 --- a/web_src/vuepages/pages/notebook/debug/index.vue +++ b/web_src/vuepages/pages/notebook/debug/index.vue @@ -1,7 +1,7 @@