From b69e3d81f09217f65eb3699db3f6099f665f4969 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 29 Nov 2021 10:53:53 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=AD=98=E5=82=A8=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E5=A2=9E=E5=8A=A0=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/storage.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/storage/storage.go b/modules/storage/storage.go index e0e875650..191871d93 100755 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -51,6 +51,7 @@ func Init() error { switch setting.Attachment.StoreType { case LocalStorageType: Attachments, err = NewLocalStorage(setting.Attachment.Path) + log.Info("local storage inited.") case MinioStorageType: minio := setting.Attachment.Minio Attachments, err = NewMinioStorage( @@ -62,6 +63,7 @@ func Init() error { minio.BasePath, minio.UseSSL, ) + log.Info("minio storage inited.") default: return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType) } @@ -71,6 +73,7 @@ func Init() error { log.Error("obs.New failed:", err) return err } + log.Info("obs cli inited.") if err != nil { return err From 2bbd2e38061e6da5e710343be3ff68dbd12ecf4b Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 29 Nov 2021 11:49:58 +0800 Subject: [PATCH 2/5] fix 992 --- routers/repo/modelarts.go | 8 ++++++-- templates/repo/modelarts/trainjob/new.tmpl | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 069a1a0b5..ec65baaad 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -364,13 +364,17 @@ func trainJobNewDataPrepare(ctx *context.Context) error { Branches, err := ctx.Repo.GitRepo.GetBranches() if err != nil { + log.Error("GetBranches failed:%v", err) ctx.ServerError("GetBranches error:", err) return err } - ctx.Data["Branches"] = Branches + if Branches != nil { + ctx.Data["Branches"] = Branches + } + ctx.Data["BranchesCount"] = len(Branches) ctx.Data["params"] = "" - ctx.Data["BranchName"] = ctx.Repo.BranchName + ctx.Data["branchName"] = ctx.Repo.BranchName configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 1a2a3c9ce..537424859 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -183,9 +183,9 @@ {{end}} {{end}} {{else}} - + {{range $k, $v :=.Branches}} - {{ if ne $v $.BranchName }} + {{ if ne $v $.branchName }} {{end}} {{end}} From ee95b280707779c4c800750d16b039536a5338a2 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 30 Nov 2021 10:35:52 +0800 Subject: [PATCH 3/5] fix-982 --- modules/auth/auth.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/modules/auth/auth.go b/modules/auth/auth.go index 16ea9f15e..352e50ca0 100644 --- a/modules/auth/auth.go +++ b/modules/auth/auth.go @@ -9,6 +9,9 @@ import ( "reflect" "strings" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth/sso" "code.gitea.io/gitea/modules/validation" @@ -31,6 +34,8 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool) return nil, false } + checkAutoLogin(ctx, sess) + // Try to sign in with each of the enabled plugins for _, ssoMethod := range sso.Methods() { if !ssoMethod.IsEnabled() { @@ -46,6 +51,23 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool) return nil, false } +func checkAutoLogin(ctx *macaron.Context, sess session.Store) { + uid := sess.Get("uid") + if uid == nil { + uname := ctx.GetCookie(setting.CookieUserName) + + u, err := models.GetUserByName(uname) + if err == nil { + + if val, ok := ctx.GetSuperSecureCookie( + base.EncodeMD5(u.Rands+u.Passwd), setting.CookieRememberName); ok && val == u.Name { + sess.Set("uid", u.ID) + } + } + } + +} + // Form form binding interface type Form interface { binding.Validator From 42753155060d3062f0aa6e9c61ecebe07852eff8 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 30 Nov 2021 16:34:42 +0800 Subject: [PATCH 4/5] sync_cb_status --- models/cloudbrain.go | 36 ++++++++++++++++++++++ modules/cron/tasks_basic.go | 13 ++++++++ routers/repo/cloudbrain.go | 75 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index bb1241247..dd3d3531c 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -31,6 +31,7 @@ const ( JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" + //notebook ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 ModelArtsCreateFailed ModelArtsJobStatus = "CREATE_FAILED" //创建失败 @@ -46,6 +47,30 @@ const ( ModelArtsDeleted ModelArtsJobStatus = "DELETED" //已删除 ModelArtsResizing ModelArtsJobStatus = "RESIZING" //规格变更中 ModelArtsResizFailed ModelArtsJobStatus = "RESIZE_FAILED" //规格变更失败 + + //trainjob + ModelArtsTrainJobUnknown ModelArtsJobStatus = "UNKNOWN" //作业状态未知 + ModelArtsTrainJobInit ModelArtsJobStatus = "INIT" //作业初始化状态 + ModelArtsTrainJobImageCreating ModelArtsJobStatus = "IMAGE_CREATING" //作业镜像正在创建 + ModelArtsTrainJobImageFailed ModelArtsJobStatus = "IMAGE_FAILED" //作业镜像创建失败 + ModelArtsTrainJobSubmitTrying ModelArtsJobStatus = "SUBMIT_TRYING" //作业正在提交 + ModelArtsTrainJobSubmitFailed ModelArtsJobStatus = "SUBMIT_FAILED" //作业提交失败 + ModelArtsTrainJobDeleteFailed ModelArtsJobStatus = "DELETE_FAILED" //作业删除失败 + ModelArtsTrainJobWaiting ModelArtsJobStatus = "WAITING" //作业正在排队中 + ModelArtsTrainJobRunning ModelArtsJobStatus = "RUNNING" //作业正在运行中 + ModelArtsTrainJobKilling ModelArtsJobStatus = "KILLING" //作业正在取消 + ModelArtsTrainJobCompleted ModelArtsJobStatus = "COMPLETED" //作业已经完成 + ModelArtsTrainJobFailed ModelArtsJobStatus = "FAILED" //作业运行失败 + ModelArtsTrainJobKilled ModelArtsJobStatus = "KILLED" //作业取消成功 + ModelArtsTrainJobCanceled ModelArtsJobStatus = "CANCELED" //作业取消 + ModelArtsTrainJobLost ModelArtsJobStatus = "LOST" //作业丢失 + ModelArtsTrainJobScaling ModelArtsJobStatus = "SCALING" //作业正在扩容 + ModelArtsTrainJobSubmitModelFailed ModelArtsJobStatus = "SUBMIT_MODEL_FAILED" //提交模型失败 + ModelArtsTrainJobDeployServiceFailed ModelArtsJobStatus = "DEPLOY_SERVICE_FAILED" //部署服务失败 + ModelArtsTrainJobCheckInit ModelArtsJobStatus = "CHECK_INIT" //审核作业初始化 + ModelArtsTrainJobCheckRunning ModelArtsJobStatus = "CHECK_RUNNING" //审核作业正在运行中 + ModelArtsTrainJobCheckRunningCompleted ModelArtsJobStatus = "CHECK_RUNNING_COMPLETED" //审核作业已经完成 + ModelArtsTrainJobCheckFailed ModelArtsJobStatus = "CHECK_FAILED" //审核作业失败 ) type Cloudbrain struct { @@ -1091,3 +1116,14 @@ func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { } return false } + +func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0, 10) + return cloudbrains, x. + NotIn("status", + JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, + ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, + ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). + Limit(100). + Find(&cloudbrains) +} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 207018c20..294690d45 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -185,6 +185,17 @@ func registerHandleSummaryStatistic() { }) } +func registerSyncCloudbrainStatus() { + RegisterTaskFatal("sync_cloudbrain_status", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 10m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SyncCloudbrainStatus() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -202,4 +213,6 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + + registerSyncCloudbrainStatus() } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 28f3a0184..bf0fffc18 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -715,3 +715,78 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc return nil } + +func SyncCloudbrainStatus() { + cloudBrains, err := models.GetCloudBrainUnStoppedJob() + if err != nil { + log.Error("GetCloudBrainUnStoppedJob failed:", err.Error()) + return + } + + for _, task := range cloudBrains { + if task.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + task.Status = taskRes.TaskStatuses[0].State + if task.Status != string(models.JobWaiting) { + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } + } else if task.Type == models.TypeCloudBrainTwo { + if task.JobType == string(models.JobTypeDebug) { + result, err := modelarts.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + task.Status = result.Status + + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else if task.JobType == string(models.JobTypeTrain) { + result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + task.Status = modelarts.TransTrainJobStatus(result.IntStatus) + task.Duration = result.Duration + task.TrainJobDuration = result.TrainJobDuration + + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else { + log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) + } + + } else { + log.Error("task.Type(%s) is error:%d", task.JobName, task.Type) + } + } + + return +} From 5cf137c1d78101514f449284e1d757aa912f47a3 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 1 Dec 2021 08:49:28 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 1 + routers/repo/ai_model_manage.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 711ffd53d..00f844881 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -30,6 +30,7 @@ type AiModelManage struct { CodeBranch string `xorm:"varchar(400) NULL"` CodeCommitID string `xorm:"NULL"` UserId int64 `xorm:"NOT NULL"` + UserName string `xorm:"NULL"` TrainTaskInfo string `xorm:"text NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index a5a1a7d9c..a02561e02 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -25,7 +25,7 @@ const ( MODEL_NOT_LATEST = 0 ) -func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64, userName string) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { @@ -82,6 +82,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio AttachmentId: aiTask.Uuid, RepoId: aiTask.RepoID, UserId: userId, + UserName: userName, CodeBranch: aiTask.BranchName, CodeCommitID: aiTask.CommitID, Engine: aiTask.EngineID, @@ -122,7 +123,7 @@ func SaveModel(ctx *context.Context) { return } - err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID) + err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID, ctx.User.Name) if err != nil { log.Info("save model error." + err.Error())