From a30ce23c4efaef1e85eec94985fb3dfd415bbe5a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 18 Aug 2021 12:34:35 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AF=E4=BB=A3=E7=A0=81=E5=90=88=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 5 ++-- modules/cloudbrain/resty.go | 73 ++++++++++++++++++++++----------------------- routers/home.go | 7 ++++- routers/repo/cloudbrain.go | 47 ++++++++++++++++++++++++----- routers/routes/routes.go | 6 +++- 5 files changed, 89 insertions(+), 49 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7a2966f74..cd9a508b5 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -123,8 +123,9 @@ type GetImagesResult struct { } type GetImagesPayload struct { - Count int `json:"count"` - ImageInfo []*ImageInfo `json:"rows"` + Count int `json:"count"` + TotalPages int `json:"totalPages,omitempty"` + ImageInfo []*ImageInfo `json:"rows"` } type CloudbrainsOptions struct { diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index fa9b25011..ef7d70ef3 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -1,9 +1,11 @@ package cloudbrain import ( - "code.gitea.io/gitea/modules/log" "encoding/json" "fmt" + "strings" + + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/setting" @@ -11,13 +13,16 @@ import ( ) var ( - restyClient *resty.Client - HOST string - TOKEN string + restyClient *resty.Client + HOST string + TOKEN string + ImagesUrlMap = map[string]string{Public: "/rest-server/api/v1/image/public/list/", Custom: "/rest-server/api/v1/image/list/"} ) const ( JobHasBeenStopped = "S410" + Public = "public" + Custom = "custom" ) func getRestyClient() *resty.Client { @@ -126,6 +131,16 @@ sendjob: } func GetImages() (*models.GetImagesResult, error) { + + return GetImagesPageable(1, 100, Custom, "") + +} + +func GetPublicImages() (*models.GetImagesResult, error) { + return GetImagesPageable(1, 100, Public, "") +} + +func GetImagesPageable(page int, size int, imageType string, name string) (*models.GetImagesResult, error) { checkSetting() client := getRestyClient() var getImagesResult models.GetImagesResult @@ -136,9 +151,9 @@ sendjob: res, err := client.R(). SetHeader("Content-Type", "application/json"). SetAuthToken(TOKEN). - SetQueryString("pageIndex=1&pageSize=100"). + SetQueryString(getQueryString(page, size, name)). SetResult(&getImagesResult). - Get(HOST + "/rest-server/api/v1/image/list/") + Get(HOST + ImagesUrlMap[imageType]) if err != nil { return nil, fmt.Errorf("resty GetImages: %v", err) @@ -157,48 +172,30 @@ sendjob: goto sendjob } - if len(response.Code) != 0 { - log.Error("getImagesResult failed(%s): %s", response.Code, response.Msg) - return &getImagesResult, fmt.Errorf("getImagesResult failed(%s): %s", response.Code, response.Msg) - } - if getImagesResult.Code != Success { return &getImagesResult, fmt.Errorf("getImagesResult err: %s", res.String()) } + getImagesResult.Payload.TotalPages = getTotalPages(getImagesResult, size) return &getImagesResult, nil } -func GetPublicImages() (*models.GetImagesResult, error) { - checkSetting() - client := getRestyClient() - var getImagesResult models.GetImagesResult - - retry := 0 - -sendjob: - res, err := client.R(). - SetHeader("Content-Type", "application/json"). - SetAuthToken(TOKEN). - SetQueryString("pageIndex=1&pageSize=100"). - SetResult(&getImagesResult). - Get(HOST + "/rest-server/api/v1/image/public/list/") - - if err != nil { - return nil, fmt.Errorf("resty GetPublicImages: %v", err) - } - - if getImagesResult.Code == "S401" && retry < 1 { - retry++ - _ = loginCloudbrain() - goto sendjob +func getTotalPages(getImagesResult models.GetImagesResult, size int) int { + totalCount := getImagesResult.Payload.Count + var totalPages int + if totalCount%size != 0 { + totalPages = totalCount/size + 1 + } else { + totalPages = totalCount / size } + return totalPages +} - if getImagesResult.Code != Success { - return &getImagesResult, fmt.Errorf("getImgesResult err: %s", res.String()) +func getQueryString(page int, size int, name string) string { + if strings.TrimSpace(name) == "" { + return fmt.Sprintf("pageIndex=%d&pageSize=%d", page, size) } - - return &getImagesResult, nil + return fmt.Sprintf("pageIndex=%d&pageSize=%d&name=%s", page, size, name) } func CommitImage(jobID string, params models.CommitImageParams) error { diff --git a/routers/home.go b/routers/home.go index 9ff860c09..2fc9db023 100755 --- a/routers/home.go +++ b/routers/home.go @@ -32,7 +32,8 @@ const ( // tplExploreOrganizations explore organizations page template tplExploreOrganizations base.TplName = "explore/organizations" // tplExploreCode explore code page template - tplExploreCode base.TplName = "explore/code" + tplExploreCode base.TplName = "explore/code" + tplExploreImages base.TplName = "explore/images" ) // Home render home page @@ -475,6 +476,10 @@ func ExploreCode(ctx *context.Context) { ctx.HTML(200, tplExploreCode) } +func ExploreImages(ctx *context.Context) { + ctx.HTML(200, tplExploreImages) +} + // NotFound render 404 page func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index c4b11d020..50396aedb 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1,8 +1,6 @@ package repo import ( - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/storage" "encoding/json" "errors" "net/http" @@ -13,6 +11,9 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -23,10 +24,10 @@ import ( ) const ( - tplCloudBrainIndex base.TplName = "repo/cloudbrain/index" - tplCloudBrainNew base.TplName = "repo/cloudbrain/new" - tplCloudBrainShow base.TplName = "repo/cloudbrain/show" - tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index" + tplCloudBrainIndex base.TplName = "repo/cloudbrain/index" + tplCloudBrainNew base.TplName = "repo/cloudbrain/new" + tplCloudBrainShow base.TplName = "repo/cloudbrain/show" + tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index" ) var ( @@ -397,6 +398,38 @@ func CloudBrainShowModels(ctx *context.Context) { ctx.HTML(200, tplCloudBrainShowModels) } +func GetPublicImages(ctx *context.Context) { + + getImages(ctx, cloudbrain.Public) + +} + +func GetCustomImages(ctx *context.Context) { + + getImages(ctx, cloudbrain.Custom) + +} + +func getImages(ctx *context.Context, imageType string) { + log.Info("Get images begin") + + page := ctx.QueryInt("page") + size := ctx.QueryInt("size") + name := ctx.Query("name") + getImagesResult, err := cloudbrain.GetImagesPageable(page, size, imageType, name) + if err != nil { + log.Error("Can not get images:%v", err) + ctx.JSON(http.StatusOK, models.GetImagesPayload{ + Count: 0, + TotalPages: 0, + ImageInfo: []*models.ImageInfo{}, + }) + } else { + ctx.JSON(http.StatusOK, getImagesResult.Payload) + } + log.Info("Get images end") +} + func getModelDirs(jobName string, parentDir string) (string, error) { var req string modelActualPath := setting.JobPath + jobName + "/model/" @@ -413,7 +446,7 @@ func CloudBrainDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") jobName := ctx.Query("jobName") - filePath := "jobs/" +jobName + "/model/" + parentDir + filePath := "jobs/" + jobName + "/model/" + parentDir url, err := storage.Attachments.PresignedGetURL(filePath, fileName) if err != nil { log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"]) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index b3565878e..84e2c7487 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,13 +6,14 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/secure" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/secure" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/context" @@ -313,11 +314,14 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") }) + m.Get("/images/public", repo.GetPublicImages) + m.Get("/images/custom", repo.GetCustomImages) m.Get("/repos", routers.ExploreRepos) m.Get("/datasets", routers.ExploreDatasets) m.Get("/users", routers.ExploreUsers) m.Get("/organizations", routers.ExploreOrganizations) m.Get("/code", routers.ExploreCode) + m.Get("/images", routers.ExploreImages) }, ignSignIn) m.Combo("/install", routers.InstallInit).Get(routers.Install). Post(bindIgnErr(auth.InstallForm{}), routers.InstallPost) From 4a876c63e6634b535feb55b606d977a18b051745 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 18 Aug 2021 17:47:42 +0800 Subject: [PATCH 2/4] get real error_msg --- modules/cloudbrain/resty.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index ef7d70ef3..4e30ea0e4 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -82,6 +82,12 @@ sendjob: Post(HOST + "/rest-server/api/v1/jobs/") if err != nil { + if res != nil { + var response models.CloudBrainResult + json.Unmarshal(res.Body(), &response) + log.Error("code(%s), msg(%s)", response.Code, response.Msg) + return nil, fmt.Errorf(response.Msg) + } return nil, fmt.Errorf("resty create job: %s", err) } From cd0a64a6ae235b2d78ecaac61e4205cebce6242b Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 19 Aug 2021 11:01:31 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=88=9B=E5=BB=BA=E4=BA=91=E8=84=91?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=8A=A5=E9=94=99=E6=97=B6=EF=BC=8C=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E5=8F=AF=E7=BB=A7=E7=BB=AD=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 50396aedb..dc1ead481 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -94,9 +94,8 @@ func jobNamePrefixValid(s string) string { return re.ReplaceAllString(lowStr, "") } -func CloudBrainNew(ctx *context.Context) { +func cloudBrainNewDataPrepare(ctx *context.Context) error{ ctx.Data["PageIsCloudBrain"] = true - t := time.Now() var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["job_name"] = jobName @@ -104,7 +103,7 @@ func CloudBrainNew(ctx *context.Context) { result, err := cloudbrain.GetImages() if err != nil { ctx.Data["error"] = err.Error() - log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["msgID"]) + log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"]) } for i, payload := range result.Payload.ImageInfo { @@ -120,7 +119,7 @@ func CloudBrainNew(ctx *context.Context) { resultPublic, err := cloudbrain.GetPublicImages() if err != nil { ctx.Data["error"] = err.Error() - log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["msgID"]) + log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"]) } for i, payload := range resultPublic.Payload.ImageInfo { @@ -135,8 +134,8 @@ func CloudBrainNew(ctx *context.Context) { attachs, err := models.GetAllUserAttachments(ctx.User.ID) if err != nil { - ctx.ServerError("GetAllUserAttachments failed:", err) - return + log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"]) + return err } ctx.Data["attachments"] = attachs @@ -163,6 +162,16 @@ func CloudBrainNew(ctx *context.Context) { ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled + + return nil +} + +func CloudBrainNew(ctx *context.Context) { + err := cloudBrainNewDataPrepare(ctx) + if err != nil { + ctx.ServerError("get new cloudbrain info failed", err) + return + } ctx.HTML(200, tplCloudBrainNew) } @@ -178,7 +187,8 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { resourceSpecId := form.ResourceSpecId if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) { - log.Error("jobtype error:", jobType, ctx.Data["msgID"]) + log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form) return } @@ -186,11 +196,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { _, err := models.GetCloudbrainByName(jobName) if err == nil { log.Error("the job name did already exist", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("the job name did already exist", tplCloudBrainNew, &form) return } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainNew, &form) return } @@ -201,6 +213,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath err = os.MkdirAll(modelPath, os.ModePerm) if err != nil { + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return } @@ -224,6 +237,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue, resourceSpecId) if err != nil { + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return } From 5a8998c931208787a31bad8315dcf7ec7bfc57d5 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 19 Aug 2021 11:38:05 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix-92=E5=90=8E=E7=AB=AF=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=90=8D=E7=A7=B0=E4=B8=8D=E5=90=AB.,?= =?UTF-8?q?=E5=8F=AF=E4=BB=A5=E5=90=AB=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index dc1ead481..7a78f914a 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -90,7 +90,7 @@ func cutString(str string, lens int) string { func jobNamePrefixValid(s string) string { lowStr := strings.ToLower(s) - re := regexp.MustCompile(`[^a-z0-9\\.\\-]+`) + re := regexp.MustCompile(`[^a-z0-9_\\-]+`) return re.ReplaceAllString(lowStr, "") }