From 30d16031764ea5bfce1a469893c444f937cafb16 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 26 Sep 2022 14:09:37 +0800 Subject: [PATCH 01/94] #2943 add specification api --- models/resource_specification.go | 26 ++++++++++++++++ routers/admin/resources.go | 2 +- routers/api/v1/api.go | 4 +++ routers/api/v1/repo/spec.go | 36 ++++++++++++++++++++++ routers/response/response.go | 6 +++- routers/response/response_list.go | 2 ++ .../cloudbrain/resource/resource_specification.go | 12 ++++++++ 7 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 routers/api/v1/repo/spec.go diff --git a/models/resource_specification.go b/models/resource_specification.go index 2da8d015d..f7a7dccbd 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -194,6 +194,32 @@ func (Specification) TableName() string { return "resource_specification" } +func (s *Specification) ToShow() *SpecificationShow { + return &SpecificationShow{ + ID: s.ID, + AccCardsNum: s.AccCardsNum, + AccCardType: s.AccCardType, + CpuCores: s.CpuCores, + MemGiB: s.MemGiB, + GPUMemGiB: s.GPUMemGiB, + ShareMemGiB: s.ShareMemGiB, + ComputeResource: s.ComputeResource, + UnitPrice: s.UnitPrice, + } +} + +type SpecificationShow struct { + ID int64 + AccCardsNum int + AccCardType string + CpuCores int + MemGiB float32 + GPUMemGiB float32 + ShareMemGiB float32 + ComputeResource string + UnitPrice int +} + func InsertResourceSpecification(r ResourceSpecification) (int64, error) { return x.Insert(&r) } diff --git a/routers/admin/resources.go b/routers/admin/resources.go index 8a8c55f86..bbdae2609 100644 --- a/routers/admin/resources.go +++ b/routers/admin/resources.go @@ -182,7 +182,7 @@ func UpdateResourceSpecification(ctx *context.Context, req models.ResourceSpecif if err != nil { log.Error("UpdateResourceSpecification error. %v", err) - ctx.JSON(http.StatusOK, response.ResponseError(err)) + ctx.JSON(http.StatusOK, response.ResponseBizError(err)) return } ctx.JSON(http.StatusOK, response.Success()) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 0b941b400..30ca7751b 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -689,6 +689,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/migrate", reqToken(), bind(auth.MigrateRepoForm{}), repo.Migrate) + m.Group("/specification", func() { + m.Get("", repo.GetResourceSpec) + }, reqToken()) + m.Group("/:username/:reponame", func() { m.Combo("").Get(reqAnyRepoReader(), repo.Get). Delete(reqToken(), reqOwner(), repo.Delete). diff --git a/routers/api/v1/repo/spec.go b/routers/api/v1/repo/spec.go new file mode 100644 index 000000000..1fac207c9 --- /dev/null +++ b/routers/api/v1/repo/spec.go @@ -0,0 +1,36 @@ +package repo + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/cloudbrain/resource" +) + +func GetResourceSpec(ctx *context.APIContext) { + jobType := ctx.Query("jobType") + computeResource := ctx.Query("compute") + cluster := ctx.Query("cluster") + aiCenterCode := ctx.Query("center") + if jobType == "" || computeResource == "" || cluster == "" { + log.Info("GetResourceSpec api.param error") + ctx.JSON(200, response.ResponseBizError(response.PARAM_ERROR)) + return + } + specs, err := resource.FindAvailableSpecs4Show(ctx.User.ID, models.FindSpecsOptions{ + JobType: models.JobType(jobType), + ComputeResource: computeResource, + Cluster: cluster, + AiCenterCode: aiCenterCode, + }) + if err != nil { + log.Error("GetResourceSpec api error. %v", err) + ctx.JSON(200, response.ResponseError(err)) + return + } + + specMap := make(map[string]interface{}, 0) + specMap["Specs"] = specs + ctx.JSON(200, response.SuccessWithData(specMap)) +} diff --git a/routers/response/response.go b/routers/response/response.go index ccd6be445..ff654e5dc 100644 --- a/routers/response/response.go +++ b/routers/response/response.go @@ -24,10 +24,14 @@ func ServerError(msg string) *AiforgeResponse { return &AiforgeResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: msg} } -func ResponseError(err *BizError) *AiforgeResponse { +func ResponseBizError(err *BizError) *AiforgeResponse { return &AiforgeResponse{Code: err.Code, Msg: err.Err} } +func ResponseError(err error) *AiforgeResponse { + return &AiforgeResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: err.Error()} +} + func SuccessWithData(data interface{}) *AiforgeResponse { return &AiforgeResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS, Data: data} } diff --git a/routers/response/response_list.go b/routers/response/response_list.go index 6514f3edd..7807813c6 100644 --- a/routers/response/response_list.go +++ b/routers/response/response_list.go @@ -1,5 +1,7 @@ package response +var PARAM_ERROR = &BizError{Code: 9001, Err: "param error"} + var RESOURCE_QUEUE_NOT_AVAILABLE = &BizError{Code: 1001, Err: "resource queue not available"} var SPECIFICATION_NOT_EXIST = &BizError{Code: 1002, Err: "specification not exist"} var SPECIFICATION_NOT_AVAILABLE = &BizError{Code: 1003, Err: "specification not available"} diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index b68abbb88..c0b20b21a 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -210,6 +210,18 @@ func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.S return specs, err } +func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*models.SpecificationShow, error) { + specs, err := FindAvailableSpecs(userId, opts) + if err != nil { + return nil, err + } + result := make([]*models.SpecificationShow, len(specs)) + for i, v := range specs { + result[i] = v.ToShow() + } + return result, nil +} + func filterExclusiveSpecs(r []*models.Specification, userId int64) []*models.Specification { specs := make([]*models.Specification, 0, len(r)) specMap := make(map[int64]string, 0) From 5455a2b277119d25c487d49cf825a7c84d860189 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 24 Oct 2022 17:00:14 +0800 Subject: [PATCH 02/94] =?UTF-8?q?=E9=80=82=E9=85=8Drestful=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E5=B0=86=E4=BA=A4=E4=BA=92=E7=9A=84=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E5=85=A8=E9=83=A8=E6=9B=B4=E6=94=B9=E4=B8=BA=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E5=BC=80=E5=A4=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 116 ++++++++++---------- routers/api/v1/api.go | 3 + routers/api/v1/repo/modelmanage.go | 30 ++++++ routers/repo/ai_model_convert.go | 32 +++--- routers/repo/ai_model_manage.go | 54 ++++++---- templates/repo/modelmanage/convertIndex.tmpl | 22 ++-- templates/repo/modelmanage/index.tmpl | 54 +++++----- templates/repo/modelmanage/showinfo.tmpl | 106 +++++++++--------- web_src/js/components/Model.vue | 156 +++++++++++++-------------- 9 files changed, 311 insertions(+), 262 deletions(-) create mode 100644 routers/api/v1/repo/modelmanage.go diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a88da8fe5..7005b7923 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -12,67 +12,67 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"INDEX NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - StatusDesc string `xorm:"varchar(500)"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"INDEX NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string - UserRelAvatarLink string - TrainTaskInfo string `xorm:"text NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` - IsCanOper bool - IsCanDelete bool + ID string `xorm:"pk" json:"id"` + Name string `xorm:"INDEX NOT NULL" json:"name"` + Version string `xorm:"NOT NULL" json:"version"` + VersionCount int `xorm:"NOT NULL DEFAULT 0" json:"versionCount"` + New int `xorm:"NOT NULL" json:"new"` + Type int `xorm:"NOT NULL" json:"type"` + Size int64 `xorm:"NOT NULL" json:"size"` + Description string `xorm:"varchar(2000)" json:"description"` + Label string `xorm:"varchar(1000)" json:"label"` + Path string `xorm:"varchar(400) NOT NULL" json:"path"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0" json:"downloadCount"` + Engine int64 `xorm:"NOT NULL DEFAULT 0" json:"engine"` + Status int `xorm:"NOT NULL DEFAULT 0" json:"status"` + StatusDesc string `xorm:"varchar(500)" json:"statusDesc"` + Accuracy string `xorm:"varchar(1000)" json:"accuracy"` + AttachmentId string `xorm:"NULL" json:"attachmentId"` + RepoId int64 `xorm:"INDEX NULL" json:"repoId"` + CodeBranch string `xorm:"varchar(400) NULL" json:"codeBranch"` + CodeCommitID string `xorm:"NULL" json:"codeCommitID"` + UserId int64 `xorm:"NOT NULL" json:"userId"` + UserName string `json:"userName"` + UserRelAvatarLink string `json:"userRelAvatarLink"` + TrainTaskInfo string `xorm:"text NULL" json:"trainTaskInfo"` + CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"` + IsCanOper bool `json:"isCanOper"` + IsCanDelete bool `json:"isCanDelete"` } type AiModelConvert struct { - ID string `xorm:"pk"` - Name string `xorm:"INDEX NOT NULL"` - Status string `xorm:"NULL"` - StatusResult string `xorm:"NULL"` - SrcEngine int `xorm:"NOT NULL DEFAULT 0"` - RepoId int64 `xorm:"INDEX NULL"` - ModelId string `xorm:"NOT NULL"` - ModelName string `xorm:"NULL"` - ModelVersion string `xorm:"NOT NULL"` - ModelPath string `xorm:"NULL"` - DestFormat int `xorm:"NOT NULL DEFAULT 0"` - NetOutputFormat int `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - CloudBrainTaskId string `xorm:"NULL"` - ModelArtsVersionId string `xorm:"NULL"` - ContainerID string - ContainerIp string - RunTime int64 `xorm:"NULL"` - TrainJobDuration string - InputShape string `xorm:"varchar(2000)"` - InputDataFormat string `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Path string `xorm:"varchar(400) NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - UserName string - UserRelAvatarLink string - IsCanOper bool - IsCanDelete bool + ID string `xorm:"pk" json:"id"` + Name string `xorm:"INDEX NOT NULL" json:"name"` + Status string `xorm:"NULL" json:"status"` + StatusResult string `xorm:"NULL" json:"statusResult"` + SrcEngine int `xorm:"NOT NULL DEFAULT 0" json:"srcEngine"` + RepoId int64 `xorm:"INDEX NULL" json:"repoId"` + ModelId string `xorm:"NOT NULL" json:"modelId"` + ModelName string `xorm:"NULL" json:"modelName"` + ModelVersion string `xorm:"NOT NULL" json:"modelVersion"` + ModelPath string `xorm:"NULL" json:"modelPath"` + DestFormat int `xorm:"NOT NULL DEFAULT 0" json:"destFormat"` + NetOutputFormat int `xorm:"NULL" json:"netOutputFormat"` + UserId int64 `xorm:"NOT NULL" json:"userId"` + CloudBrainTaskId string `xorm:"NULL" json:"cloudBrainTaskId"` + ModelArtsVersionId string `xorm:"NULL" json:"modelArtsVersionId"` + ContainerID string `json:"containerID"` + ContainerIp string `json:"containerIp"` + RunTime int64 `xorm:"NULL" json:"runTime"` + TrainJobDuration string `json:"trainJobDuration"` + InputShape string `xorm:"varchar(2000)" json:"inputShape"` + InputDataFormat string `xorm:"NOT NULL" json:"inputDataFormat"` + Description string `xorm:"varchar(2000)" json:"description"` + Path string `xorm:"varchar(400) NOT NULL" json:"path"` + CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"` + StartTime timeutil.TimeStamp `json:"startTime"` + EndTime timeutil.TimeStamp `json:"endTime"` + UserName string `json:"userName"` + UserRelAvatarLink string `json:"userRelAvatarLink"` + IsCanOper bool `json:"isCanOper"` + IsCanDelete bool `json:"isCanDelete"` } type AiModelQueryOptions struct { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e1d725ed..7667cfcfa 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -947,6 +947,9 @@ func RegisterRoutes(m *macaron.Macaron) { }) }, reqRepoReader(models.UnitTypeCloudBrain)) m.Group("/modelmanage", func() { + m.Post("/create_new_model", repo.CreateNewModel) + m.Get("/show_model_api", repo.ShowModelManageApi) + m.Delete("/delete_model", repo.DeleteModel) m.Get("/:id", repo.GetCloudbrainModelConvertTask) m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog) m.Get("/:id/modelartlog", repo.TrainJobForModelConvertGetLog) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go new file mode 100644 index 000000000..fce7cd886 --- /dev/null +++ b/routers/api/v1/repo/modelmanage.go @@ -0,0 +1,30 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + routerRepo "code.gitea.io/gitea/routers/repo" +) + +func CreateNewModel(ctx *context.APIContext) { + log.Info("create new model by api.") + ctx.Context.SetParams("JobId", ctx.Query("jobId")) + ctx.Context.SetParams("VersionName", ctx.Query("versionName")) + ctx.Context.SetParams("Name", ctx.Query("name")) + ctx.Context.SetParams("Version", ctx.Query("version")) + ctx.Context.SetParams("Engine", ctx.Query("engine")) + ctx.Context.SetParams("modelSelectedFile", ctx.Query("modelSelectedFile")) + ctx.Context.SetParams("Label", ctx.Query("label")) + ctx.Context.SetParams("Description", ctx.Query("description")) + + routerRepo.SaveModel(ctx.Context) + +} + +func ShowModelManageApi(ctx *context.APIContext) { + routerRepo.ShowModelPageInfo(ctx.Context) +} + +func DeleteModel(ctx *context.APIContext) { + routerRepo.DeleteModel(ctx.Context) +} diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 9a5874956..27ff0091f 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -72,27 +72,27 @@ func SaveModelConvert(ctx *context.Context) { log.Info("save model convert start.") if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.no_operate_right"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.no_operate_right"), }) return } name := ctx.Query("name") desc := ctx.Query("desc") modelId := ctx.Query("modelId") - modelPath := ctx.Query("ModelFile") - SrcEngine := ctx.QueryInt("SrcEngine") + modelPath := ctx.Query("modelFile") + SrcEngine := ctx.QueryInt("srcEngine") InputShape := ctx.Query("inputshape") InputDataFormat := ctx.Query("inputdataformat") - DestFormat := ctx.QueryInt("DestFormat") - NetOutputFormat := ctx.QueryInt("NetOutputFormat") + DestFormat := ctx.QueryInt("destFormat") + NetOutputFormat := ctx.QueryInt("netOutputFormat") task, err := models.QueryModelById(modelId) if err != nil { log.Error("no such model!", err.Error()) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.model_not_exist"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.model_not_exist"), }) return } @@ -103,8 +103,8 @@ func SaveModelConvert(ctx *context.Context) { if convert.Name == name { log.Info("convert.Name=" + name + " convert.id=" + convert.ID) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.create_error1"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.create_error1"), }) return } @@ -117,8 +117,8 @@ func SaveModelConvert(ctx *context.Context) { if isRunningTask(convert.Status) { log.Info("convert.Status=" + convert.Status + " convert.id=" + convert.ID) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.create_error2"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.create_error2"), }) return } @@ -148,7 +148,7 @@ func SaveModelConvert(ctx *context.Context) { go goCreateTask(modelConvert, ctx, task) ctx.JSON(200, map[string]string{ - "result_code": "0", + "code": "0", }) } @@ -588,11 +588,11 @@ func StopModelConvert(ctx *context.Context) { } func ShowModelConvertInfo(ctx *context.Context) { - ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["ID"] = ctx.Query("id") ctx.Data["isModelManage"] = true ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) - job, err := models.QueryModelConvertById(ctx.Query("ID")) + job, err := models.QueryModelConvertById(ctx.Query("id")) if err == nil { if job.TrainJobDuration == "" { job.TrainJobDuration = "00:00:00" @@ -741,7 +741,7 @@ func ModelConvertDownloadModel(ctx *context.Context) { ctx.ServerError("Not found task.", err) return } - AllDownload := ctx.QueryBool("AllDownload") + AllDownload := ctx.QueryBool("allDownload") if AllDownload { if job.IsGpuTrainTask() { path := setting.CBCodePathPrefix + job.ID + "/model/" diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 5b358b83b..6385cc674 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -175,7 +175,7 @@ func SaveNewNameModel(ctx *context.Context) { ctx.Error(403, ctx.Tr("repo.model_noright")) return } - name := ctx.Query("Name") + name := ctx.Query("name") if name == "" { ctx.Error(500, fmt.Sprintf("name or version is null.")) return @@ -197,13 +197,13 @@ func SaveModel(ctx *context.Context) { return } log.Info("save model start.") - JobId := ctx.Query("JobId") - VersionName := ctx.Query("VersionName") - name := ctx.Query("Name") - version := ctx.Query("Version") - label := ctx.Query("Label") - description := ctx.Query("Description") - engine := ctx.QueryInt("Engine") + JobId := ctx.Query("jobId") + VersionName := ctx.Query("versionName") + name := ctx.Query("name") + version := ctx.Query("version") + label := ctx.Query("label") + description := ctx.Query("description") + engine := ctx.QueryInt("engine") modelSelectedFile := ctx.Query("modelSelectedFile") log.Info("engine=" + fmt.Sprint(engine) + " modelSelectedFile=" + modelSelectedFile) @@ -293,13 +293,17 @@ func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir func DeleteModel(ctx *context.Context) { log.Info("delete model start.") - id := ctx.Query("ID") + id := ctx.Query("id") err := deleteModelByID(ctx, id) if err != nil { - ctx.JSON(500, err.Error()) + re := map[string]string{ + "code": "-1", + } + re["msg"] = err.Error() + ctx.JSON(200, re) } else { ctx.JSON(200, map[string]string{ - "result_code": "0", + "code": "0", }) } } @@ -356,7 +360,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in func DownloadMultiModelFile(ctx *context.Context) { log.Info("DownloadMultiModelFile start.") - id := ctx.Query("ID") + id := ctx.Query("id") log.Info("id=" + id) task, err := models.QueryModelById(id) if err != nil { @@ -489,7 +493,10 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con func QueryTrainJobVersionList(ctx *context.Context) { log.Info("query train job version list. start.") - JobID := ctx.Query("JobID") + JobID := ctx.Query("jobId") + if JobID == "" { + JobID = ctx.Query("JobId") + } VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) @@ -521,7 +528,10 @@ func QueryTrainModelList(ctx *context.Context) { log.Info("query train job list. start.") jobName := ctx.Query("jobName") taskType := ctx.QueryInt("type") - VersionName := ctx.Query("VersionName") + VersionName := ctx.Query("versionName") + if VersionName == "" { + VersionName = ctx.Query("VersionName") + } if taskType == models.TypeCloudBrainTwo { objectkey := path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionName) + "/" modelDbResult, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, objectkey) @@ -614,7 +624,7 @@ func DownloadSingleModelFile(ctx *context.Context) { } func ShowModelInfo(ctx *context.Context) { - ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["ID"] = ctx.Query("id") ctx.Data["name"] = ctx.Query("name") ctx.Data["isModelManage"] = true ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) @@ -830,8 +840,8 @@ func ModifyModel(id string, description string) error { func ModifyModelInfo(ctx *context.Context) { log.Info("modify model start.") - id := ctx.Query("ID") - description := ctx.Query("Description") + id := ctx.Query("id") + description := ctx.Query("description") task, err := models.QueryModelById(id) if err != nil { @@ -896,7 +906,10 @@ func QueryModelListForPredict(ctx *context.Context) { } func QueryModelFileForPredict(ctx *context.Context) { - id := ctx.Query("ID") + id := ctx.Query("id") + if id == "" { + id = ctx.Query("ID") + } model, err := models.QueryModelById(id) if err == nil { if model.Type == models.TypeCloudBrainTwo { @@ -917,7 +930,10 @@ func QueryModelFileForPredict(ctx *context.Context) { } func QueryOneLevelModelFile(ctx *context.Context) { - id := ctx.Query("ID") + id := ctx.Query("id") + if id == "" { + id = ctx.Query("ID") + } parentDir := ctx.Query("parentDir") model, err := models.QueryModelById(id) if err != nil { diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 92eefca2e..36b763174 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -366,7 +366,7 @@ data['desc']= $('#Description').val() data['modelId'] = $('#ModelVersion').val() - data['SrcEngine'] = $('#SrcEngine').val(); + data['srcEngine'] = $('#SrcEngine').val(); data['inputshape']= $('#inputshape').val(); if(inputshapeNotValid(data['inputshape'])){ @@ -379,10 +379,10 @@ } data['inputdataformat']= $('#inputdataformat').val(); - data['DestFormat'] = $('#DestFormat').val(); - data['NetOutputFormat']= $('#NetOutputFormat').val(); - data['ModelFile'] = $('#ModelFile').val(); - if(data['ModelFile']==""){ + data['destFormat'] = $('#DestFormat').val(); + data['netOutputFormat']= $('#NetOutputFormat').val(); + data['modelFile'] = $('#ModelFile').val(); + if(data['modelFile']==""){ $('.ui.error.message').text("{{.i18n.Tr "repo.modelconvert.modelfileempty"}}") $('.ui.error.message').css('display','block') $("#ModelFile_Div").addClass("error") @@ -392,11 +392,11 @@ } $.post(`${repolink}/modelmanage/create_model_convert`,data,(result) => { console.log("result=" + result); - if(result.result_code ==0){ + if(result.code ==0){ $('.ui.modal.second').modal('hide'); window.location.reload(); }else{ - $('.ui.error.message').text(result.message) + $('.ui.error.message').text(result.msg) $('.ui.error.message').css('display','block') } }) @@ -509,7 +509,7 @@ if(modelId ==null || modelId ==""){ console.log("modelId is null"); }else{ - $.get(`${repolink}/modelmanage/query_modelfile_for_predict?ID=${modelId}`, (data) => { + $.get(`${repolink}/modelmanage/query_modelfile_for_predict?id=${modelId}`, (data) => { const n_length = data.length let file_html='' let firstFileName ='' @@ -578,13 +578,13 @@ let nameMap = modelData.nameMap let versionList = nameMap[modelName] if(versionList != null && versionList.length >0){ - if(versionList[0].Engine == engineOption){ + if(versionList[0].engine == engineOption){ return "selected=\"selected\""; }else{ - if((versionList[0].Engine==122 || versionList[0].Engine==37) && engineOption==2){ + if((versionList[0].engine==122 || versionList[0].engine==37) && engineOption==2){ return "selected=\"selected\""; } - if((versionList[0].Engine==121 || versionList[0].Engine==38) && engineOption==1){ + if((versionList[0].engine==121 || versionList[0].engine==38) && engineOption==1){ return "selected=\"selected\""; } } diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index b2994f0c2..9716637ed 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -138,20 +138,20 @@
- +
- +  
@@ -364,9 +364,9 @@ $("#task_name").removeClass("error") } - data['desc']= $('#Description').val() - data['modelId'] = $('#ModelVersion').val() - data['srcEngine'] = $('#SrcEngine').val(); + data['desc']= $('#description').val() + data['modelId'] = $('#modelVersion').val() + data['srcEngine'] = $('#srcEngine').val(); data['inputshape']= $('#inputshape').val(); if(inputshapeNotValid(data['inputshape'])){ @@ -379,9 +379,9 @@ } data['inputdataformat']= $('#inputdataformat').val(); - data['destFormat'] = $('#DestFormat').val(); - data['netOutputFormat']= $('#NetOutputFormat').val(); - data['modelFile'] = $('#ModelFile').val(); + data['destFormat'] = $('#destFormat').val(); + data['netOutputFormat']= $('#netOutputFormat').val(); + data['modelFile'] = $('#modelFile').val(); if(data['modelFile']==""){ $('.ui.error.message').text("{{.i18n.Tr "repo.modelconvert.modelfileempty"}}") $('.ui.error.message').css('display','block') @@ -456,7 +456,7 @@ $('#choice_version').dropdown({ onChange:function(value){ console.log("model version:" + value); - $('#choice_version input[name="ModelVersion"]').val(value) + $('#choice_version input[name="modelVersion"]').val(value) loadModelFile(value); } }) @@ -464,26 +464,26 @@ $('#choice_file').dropdown({ onChange:function(value){ console.log("model file:" + value); - $('#choice_file input[name="ModelFile"]').val(value) + $('#choice_file input[name="modelFile"]').val(value) } }) }) function srcEngineChanged(){ - var ele = window.document.getElementById("SrcEngine"); + var ele = window.document.getElementById("srcEngine"); var index=ele.selectedIndex; var options=ele.options; var option = options[index]; - console.log("SrcEngine value=" + option); + console.log("srcEngine value=" + option); let destFormatHtml = ""; let netOutputFormatHtml = ""; if(option==null || option =="undefined" || option.value == 0){ destFormatHtml += "" netOutputFormatHtml += ""; } - $('#DestFormat').html(destFormatHtml); - $('#NetOutputFormat').html(netOutputFormatHtml); + $('#destFormat').html(destFormatHtml); + $('#netOutputFormat').html(netOutputFormatHtml); } function loadModelList(){ @@ -550,25 +550,25 @@ n_length = versionList.length let train_html='' for (let i=0;i${versionList[i].Version}
` + train_html += `
${versionList[i].version}
` train_html += '
' } $("#model-version").append(train_html) - $('#choice_version .default.text').text(versionList[0].Version) - $('#choice_version input[name="ModelVersion"]').val(versionList[0].ID) - loadModelFile(versionList[0].ID); + $('#choice_version .default.text').text(versionList[0].version) + $('#choice_version input[name="modelVersion"]').val(versionList[0].id) + loadModelFile(versionList[0].id); } setEngineValue(value); } function setEngineValue(value){ - $('#SrcEngine').dropdown('clear'); + $('#srcEngine').dropdown('clear'); console.log("setEngineValue value=" + value); let html = "" html +=""; html +=""; html +=""; - $('#SrcEngine').html(html); + $('#srcEngine').html(html); srcEngineChanged(); } function getSelected(engineOption, modelName){ From ae1cf98f7713aa8f33a2a2271994ef0f5c3b87d5 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 25 Oct 2022 15:08:28 +0800 Subject: [PATCH 28/94] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/convertIndex.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index cc334707e..25891dbda 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -93,7 +93,7 @@
- + {{.Name}}
@@ -141,7 +141,7 @@ {{if .IsCanOper}} - + {{$.i18n.Tr "repo.modelconvert.download"}} {{else}} From 26d8cae2622e041242f58efc3c1519b08446a546 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 25 Oct 2022 17:17:38 +0800 Subject: [PATCH 29/94] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 25891dbda..d21cd0abf 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -526,7 +526,7 @@ } $("#model-file").append(file_html) $('#choice_file .default.text').text(firstFileName) - $('#choice_file input[name="ModelFile"]').val(firstFileName) + $('#choice_file input[name="modelFile"]').val(firstFileName) }) } From f453610611c17722b13d731105d055d9134cc9a4 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 25 Oct 2022 17:53:51 +0800 Subject: [PATCH 30/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/base_message.go | 6 ++ options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/api/v1/repo/cloudbrain.go | 46 +++++++++++- services/cloudbrain/cloudbrainTask/count.go | 22 +++--- services/cloudbrain/cloudbrainTask/train.go | 112 ++++++++++++++++++++++++++++ 6 files changed, 175 insertions(+), 13 deletions(-) diff --git a/models/base_message.go b/models/base_message.go index f0fa19bf2..f76e18175 100644 --- a/models/base_message.go +++ b/models/base_message.go @@ -29,3 +29,9 @@ func BaseErrorMessageApi(message string) BaseMessageApi { 1, message, } } + +type BaseMessageWithDataApi struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data"` +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 324930544..ce80a31fe 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -616,6 +616,7 @@ organization = Organizations uid = Uid u2f = Security Keys bind_wechat = Bind WeChat +no_wechat_bind = Can not do the operation, please bind WeChat first. wechat_bind = WeChat Binding bind_account_information = Bind account information bind_time = Bind Time diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index fd5596a2e..fdbafd814 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -621,6 +621,7 @@ organization=组织 uid=用户 ID u2f=安全密钥 wechat_bind = 微信绑定 +no_wechat_bind = 不能创建任务，请先绑定微信。 bind_wechat = 绑定微信 bind_account_information = 绑定账号信息 bind_time = 绑定时间 diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 58dd5b801..380de0ddb 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -33,6 +33,48 @@ import ( routerRepo "code.gitea.io/gitea/routers/repo" ) +func CloudBrainShow(ctx *context.APIContext) { + + task, err := models.GetCloudbrainByJobID(ctx.Params(":jobid")) + + if err != nil { + log.Info("error:" + err.Error()) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("repo.cloudbrain_query_fail")) + return + } + cloudbrainTask.PrepareSpec4Show(task) + task.ContainerIp = "" + if cloudbrainTask.IsTaskNotStop(task) { + cloudbrainTask.SyncTaskStatus(task) + } + + if task.TrainJobDuration == "" { + if task.Duration == 0 { + var duration int64 + if task.Status == string(models.JobWaiting) { + duration = 0 + } else if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.CreatedUnix) + } else { + duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) + } + task.Duration = duration + } + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) + } + //to unify image output + if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter { + task.ImageID = strconv.FormatInt(task.EngineID, 10) + task.Image = task.EngineName + + } else if task.Type == models.TypeC2Net { + task.Image = task.EngineName + } + + ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: task}) + +} + func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == 2 { @@ -47,10 +89,10 @@ func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) func CreateCloudBrainInferenceTask(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == 0 { - cloudbrainTask.GrampusTrainJobGpuCreate(ctx.Context, option) + cloudbrainTask.CloudBrainInferenceJobCreate(ctx.Context, option) } if option.Type == 1 { - cloudbrainTask.GrampusTrainJobNpuCreate(ctx.Context, option) + cloudbrainTask.ModelArtsInferenceJobCreate(ctx.Context, option) } } diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index a9b254618..34b334445 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -14,28 +14,28 @@ type StatusInfo struct { ComputeResource string } -var cloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)} -var cloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)} -var grampusTwoNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning} +var CloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)} +var CloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)} +var GrampusNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning} var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeDebug}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeInference}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo, models.TypeCDCenter}, @@ -45,22 +45,22 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: cloudbrainTwoNotFinalStatuses, + NotFinalStatuses: CloudbrainTwoNotFinalStatuses, ComputeResource: models.NPUResource, }, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo}, JobType: []models.JobType{models.JobTypeInference}, - NotFinalStatuses: cloudbrainTwoNotFinalStatuses, + NotFinalStatuses: CloudbrainTwoNotFinalStatuses, ComputeResource: models.NPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.GPUResource: { CloudBrainTypes: []int{models.TypeC2Net}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: grampusTwoNotFinalStatuses, + NotFinalStatuses: GrampusNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.NPUResource: { CloudBrainTypes: []int{models.TypeC2Net}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: grampusTwoNotFinalStatuses, + NotFinalStatuses: GrampusNotFinalStatuses, ComputeResource: models.NPUResource, }} diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index d31038536..6f836557c 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -12,6 +12,10 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/modules/timeutil" + + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/git" @@ -680,3 +684,111 @@ func getPoolId() string { return resourcePools.Info[0].ID } + +func PrepareSpec4Show(task *models.Cloudbrain) { + s, err := resource.GetCloudbrainSpec(task.ID) + if err != nil { + log.Info("error:" + err.Error()) + return + } + task.Spec = s +} + +func IsTaskNotStop(task *models.Cloudbrain) bool { + statuses := CloudbrainOneNotFinalStatuses + if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter { + statuses = CloudbrainTwoNotFinalStatuses + } else { + statuses = GrampusNotFinalStatuses + } + + for _, status := range statuses { + if task.Status == status { + return true + } + } + return false + +} + +func SyncTaskStatus(task *models.Cloudbrain) error { + if task.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(task.JobID) + if err != nil { + log.Info("error:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + + if result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + + oldStatus := task.Status + task.Status = taskRes.TaskStatuses[0].State + + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) + + if task.DeletedAt.IsZero() { //normal record + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } + err = models.UpdateJob(task) + if err != nil { + return fmt.Errorf("repo.cloudbrain_query_fail") + + } + } + + } else { + log.Info("error:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + } else if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter { + err := modelarts.HandleTrainJobInfo(task) + if err != nil { + return fmt.Errorf("repo.cloudbrain_query_fail") + } + + } else if task.Type == models.TypeC2Net { + result, err := grampus.GetJob(task.JobID) + if err != nil { + log.Error("GetJob failed:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + + if result != nil { + if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { + task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] + } + oldStatus := task.Status + task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) + if task.Status != result.JobInfo.Status || result.JobInfo.Status == models.GrampusStatusRunning { + task.Duration = result.JobInfo.RunSec + if task.Duration < 0 { + task.Duration = 0 + } + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) + + if task.StartTime == 0 && result.JobInfo.StartedAt > 0 { + task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt) + } + if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { + task.EndTime = task.StartTime.Add(task.Duration) + } + task.CorrectCreateUnix() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob failed:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + } + } + } + return nil + +} From 9e513d2ad79dd33ddb030ee5d44e9242df976453 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 26 Oct 2022 10:59:25 +0800 Subject: [PATCH 31/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4API=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 5 ++++ routers/api/v1/repo/modelmanage.go | 53 ++++++++++++++++++++++++++++++-------- routers/repo/ai_model_convert.go | 29 ++++++++++++++++----- 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b3694ced9..b03db6327 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -966,6 +966,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/create_new_model", repo.CreateNewModel) m.Get("/show_model_api", repo.ShowModelManageApi) m.Delete("/delete_model", repo.DeleteModel) + m.Get("/downloadall", repo.DownloadModel) + m.Get("/query_model_for_predict", repo.QueryModelListForPredict) + m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) + m.Post("/create_model_convert", repo.CreateModelConvert) + m.Get("/show_model_convert_page") m.Get("/:id", repo.GetCloudbrainModelConvertTask) m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog) m.Get("/:id/modelartlog", repo.TrainJobForModelConvertGetLog) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index fce7cd886..b32726997 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -1,30 +1,61 @@ package repo import ( + "net/http" + "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" routerRepo "code.gitea.io/gitea/routers/repo" ) func CreateNewModel(ctx *context.APIContext) { - log.Info("create new model by api.") - ctx.Context.SetParams("JobId", ctx.Query("jobId")) - ctx.Context.SetParams("VersionName", ctx.Query("versionName")) - ctx.Context.SetParams("Name", ctx.Query("name")) - ctx.Context.SetParams("Version", ctx.Query("version")) - ctx.Context.SetParams("Engine", ctx.Query("engine")) - ctx.Context.SetParams("modelSelectedFile", ctx.Query("modelSelectedFile")) - ctx.Context.SetParams("Label", ctx.Query("label")) - ctx.Context.SetParams("Description", ctx.Query("description")) - + log.Info("CreateNewModel by api.") routerRepo.SaveModel(ctx.Context) - } func ShowModelManageApi(ctx *context.APIContext) { + log.Info("ShowModelManageApi by api.") routerRepo.ShowModelPageInfo(ctx.Context) } func DeleteModel(ctx *context.APIContext) { + log.Info("DeleteModel by api.") routerRepo.DeleteModel(ctx.Context) } + +func DownloadModel(ctx *context.APIContext) { + log.Info("DownloadModel by api.") + routerRepo.DownloadMultiModelFile(ctx.Context) +} + +func QueryModelListForPredict(ctx *context.APIContext) { + log.Info("QueryModelListForPredict by api.") + routerRepo.QueryModelListForPredict(ctx.Context) +} + +func QueryModelFileForPredict(ctx *context.APIContext) { + log.Info("QueryModelFileForPredict by api.") + routerRepo.QueryModelFileForPredict(ctx.Context) +} + +func CreateModelConvert(ctx *context.APIContext) { + log.Info("CreateModelConvert by api.") + routerRepo.SaveModelConvert(ctx.Context) +} + +func ShowModelConvertPage(ctx *context.APIContext) { + log.Info("ShowModelConvertPage by api.") + modelResult, count, err := routerRepo.GetModelConvertPageData(ctx.Context) + if err == nil { + mapInterface := make(map[string]interface{}) + mapInterface["data"] = modelResult + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) + } else { + mapInterface := make(map[string]interface{}) + mapInterface["data"] = nil + mapInterface["count"] = 0 + ctx.JSON(http.StatusOK, mapInterface) + } + +} diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 27ff0091f..959f4bc50 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -699,6 +699,26 @@ func ShowModelConvertPageInfo(ctx *context.Context) { if pageSize <= 0 { pageSize = setting.UI.IssuePagingNum } + modelResult, count, err := GetModelConvertPageData(ctx) + if err == nil { + pager := context.NewPagination(int(count), page, pageSize, 5) + ctx.Data["Page"] = pager + ctx.Data["Tasks"] = modelResult + ctx.Data["MODEL_CONVERT_COUNT"] = count + } else { + ctx.ServerError("Query data error.", err) + } +} + +func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) { + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } repoId := ctx.Repo.Repository.ID modelResult, count, err := models.QueryModelConvert(&models.AiModelQueryOptions{ ListOptions: models.ListOptions{ @@ -709,10 +729,8 @@ func ShowModelConvertPageInfo(ctx *context.Context) { }) if err != nil { log.Info("query db error." + err.Error()) - ctx.ServerError("Cloudbrain", err) - return + return nil, 0, err } - ctx.Data["MODEL_CONVERT_COUNT"] = count userIds := make([]int64, len(modelResult)) for i, model := range modelResult { model.IsCanOper = isOper(ctx, model.UserId) @@ -727,10 +745,7 @@ func ShowModelConvertPageInfo(ctx *context.Context) { model.UserRelAvatarLink = value.RelAvatarLink() } } - pager := context.NewPagination(int(count), page, pageSize, 5) - ctx.Data["Page"] = pager - ctx.Data["Tasks"] = modelResult - + return modelResult, count, nil } func ModelConvertDownloadModel(ctx *context.Context) { From ea48719bd528c978f14a5f984789f7974fa4f115 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 26 Oct 2022 15:22:34 +0800 Subject: [PATCH 32/94] #3032 update specification query api --- models/resource_specification.go | 18 +++++++++--------- routers/api/v1/repo/spec.go | 8 ++++---- routers/response/api_response.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 routers/response/api_response.go diff --git a/models/resource_specification.go b/models/resource_specification.go index f7a7dccbd..d69cd7208 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -209,15 +209,15 @@ func (s *Specification) ToShow() *SpecificationShow { } type SpecificationShow struct { - ID int64 - AccCardsNum int - AccCardType string - CpuCores int - MemGiB float32 - GPUMemGiB float32 - ShareMemGiB float32 - ComputeResource string - UnitPrice int + ID int64 `json:"id"` + AccCardsNum int `json:"acc_cards_num"` + AccCardType string `json:"acc_card_type"` + CpuCores int `json:"cpu_cores"` + MemGiB float32 `json:"mem_gi_b"` + GPUMemGiB float32 `json:"gpu_mem_gi_b"` + ShareMemGiB float32 `json:"share_mem_gi_b"` + ComputeResource string `json:"compute_resource"` + UnitPrice int `json:"unit_price"` } func InsertResourceSpecification(r ResourceSpecification) (int64, error) { diff --git a/routers/api/v1/repo/spec.go b/routers/api/v1/repo/spec.go index 1fac207c9..a8aa28d98 100644 --- a/routers/api/v1/repo/spec.go +++ b/routers/api/v1/repo/spec.go @@ -15,7 +15,7 @@ func GetResourceSpec(ctx *context.APIContext) { aiCenterCode := ctx.Query("center") if jobType == "" || computeResource == "" || cluster == "" { log.Info("GetResourceSpec api.param error") - ctx.JSON(200, response.ResponseBizError(response.PARAM_ERROR)) + ctx.JSON(200, response.OuterBizError(response.PARAM_ERROR)) return } specs, err := resource.FindAvailableSpecs4Show(ctx.User.ID, models.FindSpecsOptions{ @@ -26,11 +26,11 @@ func GetResourceSpec(ctx *context.APIContext) { }) if err != nil { log.Error("GetResourceSpec api error. %v", err) - ctx.JSON(200, response.ResponseError(err)) + ctx.JSON(200, response.OuterServerError(err.Error())) return } specMap := make(map[string]interface{}, 0) - specMap["Specs"] = specs - ctx.JSON(200, response.SuccessWithData(specMap)) + specMap["specs"] = specs + ctx.JSON(200, response.OuterSuccessWithData(specMap)) } diff --git a/routers/response/api_response.go b/routers/response/api_response.go new file mode 100644 index 000000000..5cc6ff78e --- /dev/null +++ b/routers/response/api_response.go @@ -0,0 +1,30 @@ +package response + +type AiforgeOuterResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data interface{} `json:"data"` +} + +func OuterSuccess() *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS} +} + +func OuterError(code int, msg string) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: code, Msg: msg} +} + +func OuterServerError(msg string) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: msg} +} + +func OuterBizError(err *BizError) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: err.Code, Msg: err.Err} +} + +func OuterSuccessWithData(data interface{}) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS, Data: data} +} +func OuterErrorWithData(code int, msg string, data interface{}) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: code, Msg: msg, Data: data} +} From 331d615412e4a2c63b00eea8054b5a7897eb3981 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 26 Oct 2022 16:19:41 +0800 Subject: [PATCH 33/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 42 +++++++-------- models/dataset.go | 40 +++++++------- models/repo.go | 120 +++++++++++++++++++++--------------------- modules/structs/attachment.go | 47 +++++++++++++++++ modules/structs/cloudbrain.go | 40 ++++++++++++++ 5 files changed, 188 insertions(+), 101 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 5ebb6f0f3..2b5fa8efc 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -33,27 +33,27 @@ const ( // Attachment represent a attachment of issue/comment/release. type Attachment struct { - ID int64 `xorm:"pk autoincr" json:"id"` - UUID string `xorm:"uuid UNIQUE" json:"uuid"` - IssueID int64 `xorm:"INDEX" json:"issueId"` - DatasetID int64 `xorm:"INDEX DEFAULT 0" json:"datasetId"` - ReleaseID int64 `xorm:"INDEX" json:"releaseId"` - UploaderID int64 `xorm:"INDEX DEFAULT 0" json:"uploaderId"` // Notice: will be zero before this column added - CommentID int64 `json:"commentId"` - Name string `json:"name"` - Description string `xorm:"TEXT" json:"description"` - DownloadCount int64 `xorm:"DEFAULT 0" json:"downloadCount"` - UseNumber int64 `xorm:"DEFAULT 0" json:"useNumber"` - Size int64 `xorm:"DEFAULT 0" json:"size"` - IsPrivate bool `xorm:"DEFAULT false" json:"isPrivate"` - DecompressState int32 `xorm:"DEFAULT 0" json:"decompressState"` - Type int `xorm:"DEFAULT 0" json:"type"` - CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` - - FileChunk *FileChunk `xorm:"-" json:"fileChunk"` - CanDel bool `xorm:"-" json:"canDel"` - Uploader *User `xorm:"-" json:"uploader"` - Md5 string `xorm:"-" json:"md5"` + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + IssueID int64 `xorm:"INDEX"` + DatasetID int64 `xorm:"INDEX DEFAULT 0"` + ReleaseID int64 `xorm:"INDEX"` + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 + Name string + Description string `xorm:"TEXT"` + DownloadCount int64 `xorm:"DEFAULT 0"` + UseNumber int64 `xorm:"DEFAULT 0"` + Size int64 `xorm:"DEFAULT 0"` + IsPrivate bool `xorm:"DEFAULT false"` + DecompressState int32 `xorm:"DEFAULT 0"` + Type int `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + + FileChunk *FileChunk `xorm:"-"` + CanDel bool `xorm:"-"` + Uploader *User `xorm:"-"` + Md5 string `xorm:"-"` } type AttachmentUsername struct { diff --git a/models/dataset.go b/models/dataset.go index f3c2adbd5..972503641 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -21,26 +21,26 @@ const ( ) type Dataset struct { - ID int64 `xorm:"pk autoincr" json:"id"` - Title string `xorm:"INDEX NOT NULL" json:"title"` - Status int32 `xorm:"INDEX" json:"status"` // normal_private: 0, pulbic: 1, is_delete: 2 - Category string `json:"category"` - Description string `xorm:"TEXT" json:"description"` - DownloadTimes int64 `json:"downloadTimes"` - UseCount int64 `xorm:"DEFAULT 0" json:"useCount"` - NumStars int `xorm:"INDEX NOT NULL DEFAULT 0" json:"numStars"` - Recommend bool `xorm:"INDEX NOT NULL DEFAULT false" json:"recommend"` - License string `json:"license"` - Task string `json:"task"` - ReleaseID int64 `xorm:"INDEX" json:"releaseId"` - UserID int64 `xorm:"INDEX" json:"userId"` - RepoID int64 `xorm:"INDEX" json:"repoId"` - Repo *Repository `xorm:"-" json:"repo"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"createdUnix"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"updatedUnix"` - - User *User `xorm:"-" json:"user"` - Attachments []*Attachment `xorm:"-" json:"attachments"` + ID int64 `xorm:"pk autoincr"` + Title string `xorm:"INDEX NOT NULL""` + Status int32 `xorm:"INDEX""` // normal_private: 0, pulbic: 1, is_delete: 2 + Category string + Description string `xorm:"TEXT"` + DownloadTimes int64 + UseCount int64 `xorm:"DEFAULT 0"` + NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` + Recommend bool `xorm:"INDEX NOT NULL DEFAULT false"` + License string + Task string + ReleaseID int64 `xorm:"INDEX"` + UserID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX"` + Repo *Repository `xorm:"-"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + + User *User `xorm:"-"` + Attachments []*Attachment `xorm:"-"` } type DatasetWithStar struct { diff --git a/models/repo.go b/models/repo.go index ccb0d680c..333492cee 100755 --- a/models/repo.go +++ b/models/repo.go @@ -162,79 +162,79 @@ const ( // Repository represents a git repository. type Repository struct { - ID int64 `xorm:"pk autoincr" json:"id"` - OwnerID int64 `xorm:"UNIQUE(s) index" json:"ownerId"` - OwnerName string `json:"ownerName"` - Owner *User `xorm:"-" json:"owner"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL" json:"lowerName"` - Name string `xorm:"INDEX NOT NULL" json:"name"` - Description string `xorm:"TEXT" json:"description"` - Website string `xorm:"VARCHAR(2048)" json:"website"` - OriginalServiceType api.GitServiceType `xorm:"index" json:"originalServiceType"` - OriginalURL string `xorm:"VARCHAR(2048)" json:"originalUrl"` - DefaultBranch string `json:"defaultBranch"` - CreatorID int64 `xorm:"INDEX NOT NULL DEFAULT 0" json:"creatorId"` - Creator *User `xorm:"-" json:"creator"` - NumWatches int `json:"numWatches"` - NumStars int `json:"numStars"` - NumForks int `json:"numForks"` - NumIssues int `json:"numIssues"` - NumClosedIssues int `json:"numClosedIssues"` - NumOpenIssues int `xorm:"-" json:"numOpenIssues"` - NumPulls int `json:"numPulls"` - NumClosedPulls int `json:"numClosedPulls"` - NumOpenPulls int `xorm:"-" json:"numOpenPulls"` - NumMilestones int `xorm:"NOT NULL DEFAULT 0" json:"numMilestones"` - NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0" json:"numClosedMilestones"` - NumOpenMilestones int `xorm:"-" json:"numOpenMilestones"` - NumCommit int64 `xorm:"NOT NULL DEFAULT 0" json:"numCommit"` - RepoType RepoType `xorm:"NOT NULL DEFAULT 0" json:"repoType"` - - IsPrivate bool `xorm:"INDEX" json:"isPrivate"` - IsEmpty bool `xorm:"INDEX" json:"isEmpty"` - IsArchived bool `xorm:"INDEX" json:"isArchived"` - IsMirror bool `xorm:"INDEX" json:"isMirror"` + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) index"` + OwnerName string + Owner *User `xorm:"-"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Description string `xorm:"TEXT"` + Website string `xorm:"VARCHAR(2048)"` + OriginalServiceType api.GitServiceType `xorm:"index"` + OriginalURL string `xorm:"VARCHAR(2048)"` + DefaultBranch string + CreatorID int64 `xorm:"INDEX NOT NULL DEFAULT 0"` + Creator *User `xorm:"-"` + NumWatches int + NumStars int + NumForks int + NumIssues int + NumClosedIssues int + NumOpenIssues int `xorm:"-"` + NumPulls int + NumClosedPulls int + NumOpenPulls int `xorm:"-"` + NumMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumOpenMilestones int `xorm:"-"` + NumCommit int64 `xorm:"NOT NULL DEFAULT 0"` + RepoType RepoType `xorm:"NOT NULL DEFAULT 0"` + + IsPrivate bool `xorm:"INDEX"` + IsEmpty bool `xorm:"INDEX"` + IsArchived bool `xorm:"INDEX"` + IsMirror bool `xorm:"INDEX"` *Mirror `xorm:"-"` - Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0" json:"status"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` RenderingMetas map[string]string `xorm:"-"` - Units []*RepoUnit `xorm:"-" json:"units"` - PrimaryLanguage *LanguageStat `xorm:"-" json:"primaryLanguage"` - - IsFork bool `xorm:"INDEX NOT NULL DEFAULT false" json:"isFork"` - ForkID int64 `xorm:"INDEX" json:"forkId"` - BaseRepo *Repository `xorm:"-" json:"baseRepo"` - IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false" json:"isTemplate"` - TemplateID int64 `xorm:"INDEX" json:"templateId"` - TemplateRepo *Repository `xorm:"-" json:"templateRepo"` - Size int64 `xorm:"NOT NULL DEFAULT 0" json:"size"` - CodeIndexerStatus *RepoIndexerStatus `xorm:"-" json:"codeIndexerStatus"` - StatsIndexerStatus *RepoIndexerStatus `xorm:"-" json:"statsIndexerStatus"` - IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true" json:"isFsckEnabled"` - CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false" json:"closeIssuesViaCommitInAnyBranch"` - Topics []string `xorm:"TEXT JSON" json:"topics"` + Units []*RepoUnit `xorm:"-"` + PrimaryLanguage *LanguageStat `xorm:"-"` + + IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"` + ForkID int64 `xorm:"INDEX"` + BaseRepo *Repository `xorm:"-"` + IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"` + TemplateID int64 `xorm:"INDEX"` + TemplateRepo *Repository `xorm:"-"` + Size int64 `xorm:"NOT NULL DEFAULT 0"` + CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` + StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` + IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` + CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"` + Topics []string `xorm:"TEXT JSON"` // Avatar: ID(10-20)-md5(32) - must fit into 64 symbols - Avatar string `xorm:"VARCHAR(64)" json:"avatar"` + Avatar string `xorm:"VARCHAR(64)"` //blockchain - ContractAddress string `xorm:"INDEX" json:"contractAddress"` - Balance string `xorm:"NOT NULL DEFAULT '0'" json:"balance"` - BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0" json:"blockChainStatus"` + ContractAddress string `xorm:"INDEX"` + Balance string `xorm:"NOT NULL DEFAULT '0'"` + BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0"` // git clone and git pull total count - CloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"clone_cnt" json:"cloneCnt"` + CloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"clone_cnt"` // only git clone total count - GitCloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"git_clone_cnt" json:"gitCloneCnt"` + GitCloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"git_clone_cnt"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"createdUnix"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"updatedUnix"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Hot int64 `xorm:"-" json:"hot"` - Active int64 `xorm:"-" json:"active"` - Alias string `xorm:"INDEX" json:"alias"` - LowerAlias string `xorm:"INDEX" json:"lowerAlias"` + Hot int64 `xorm:"-"` + Active int64 `xorm:"-"` + Alias string `xorm:"INDEX"` + LowerAlias string `xorm:"INDEX"` } type RepositoryShow struct { diff --git a/modules/structs/attachment.go b/modules/structs/attachment.go index 0f3c2ed6a..3ccc2742a 100755 --- a/modules/structs/attachment.go +++ b/modules/structs/attachment.go @@ -6,6 +6,8 @@ package structs // import "code.gitea.io/gitea/modules/structs" import ( "time" + + "code.gitea.io/gitea/modules/timeutil" ) // Attachment a generic attachment @@ -27,3 +29,48 @@ type Attachment struct { type EditAttachmentOptions struct { Name string `json:"name"` } + +type Dataset struct { + ID int64 `json:"id"` + Title string `json:"title"` + Status int32 `json:"status"` + Category string `json:"category"` + Description string `json:"description"` + DownloadTimes int64 `json:"downloadTimes"` + UseCount int64 `json:"useCount"` + NumStars int `json:"numStars"` + Recommend bool `json:"recommend"` + License string `json:"license"` + Task string `json:"task"` + ReleaseID int64 `json:"releaseId"` + UserID int64 `json:"userId"` + RepoID int64 `json:"repoId"` + Repo *RepositoryShow `json:"repo"` + CreatedUnix timeutil.TimeStamp `json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `json:"updatedUnix"` + + Attachments []*AttachmentShow `json:"attachments"` +} + +type RepositoryShow struct { + OwnerName string `json:"ownerName"` + Name string `json:"name"` +} + +type AttachmentShow struct { + ID int64 `json:"id"` + UUID string `json:"uuid"` + DatasetID int64 `json:"datasetId"` + ReleaseID int64 `json:"releaseId"` + UploaderID int64 `json:"uploaderId"` + CommentID int64 `json:"commentId"` + Name string `json:"name"` + Description string `json:"description"` + DownloadCount int64 `json:"downloadCount"` + UseNumber int64 `json:"useNumber"` + Size int64 `json:"size"` + IsPrivate bool `json:"isPrivate"` + DecompressState int32 `json:"decompressState"` + Type int `json:"type"` + CreatedUnix timeutil.TimeStamp `json:"createdUnix"` +} diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 0d2576e7e..e0409b340 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -1,5 +1,10 @@ package structs +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/timeutil" +) + type CreateGrampusTrainJobOption struct { DisplayJobName string `json:"display_job_name" binding:"Required"` JobName string `json:"job_name" binding:"Required" ` @@ -40,3 +45,38 @@ type CreateTrainJobOption struct { PreTrainModelUrl string `json:"pre_train_model_url"` SpecId int64 `json:"spec_id" binding:"Required"` } + +type Cloudbrain struct { + ID int64 `json:"id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + Type int `json:"type"` + DisplayJobName string `json:"display_job_name"` + Status string `json:"status"` + CreatedUnix timeutil.TimeStamp `json:"created_unix"` + RepoID int64 `json:"repo_id"` + Duration int64 `json:"duration"` //运行时长 单位秒 + TrainJobDuration string `json:"train_job_duration"` + ImageID string `json:"image_id"` //grampus image_id + Image string `json:"image"` + Uuid string `json:"uuid"` //数据集id + DatasetName string `json:"dataset_name"` + ComputeResource string `json:"compute_resource"` //计算资源，例如npu + AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name + BranchName string `json:"branch_name"` //分支名称 + Parameters string `json:"parameters"` //传给modelarts的param参数 + BootFile string `json:"boot_file"` //启动文件 + Description string `json:"description"` //描述 + ModelName string `json:"model_name"` //模型名称 + ModelVersion string `json:"model_version"` //模型版本 + CkptName string `json:"ckpt_name"` //权重文件名称 + + VersionCount int `json:"-"` //任务的当前版本数量，不包括删除的 + IsLatestVersion string `json:"-"` //是否是最新版本，1是，0否 + CommitID string `json:"-"` //提交的仓库代码id + PreVersionName string `json:"-"` //父版本名称 + StartTime timeutil.TimeStamp `json:"start_time"` + EndTime timeutil.TimeStamp `json:"end_time"` + + Spec *models.Specification `json:"spec"` +} From a2332f6219487b2f7908d9c9cf8b06cb3893f637 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 27 Oct 2022 09:19:54 +0800 Subject: [PATCH 34/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/resource_specification.go | 26 ------ modules/convert/cloudbrain.go | 103 +++++++++++++++++++++ modules/structs/attachment.go | 66 +++++++------ modules/structs/cloudbrain.go | 72 +++++++------- routers/api/v1/api.go | 29 +++++- routers/api/v1/repo/attachments.go | 25 +++++ routers/api/v1/repo/cloudbrain.go | 4 +- routers/api/v1/repo/datasets.go | 13 ++- .../cloudbrain/resource/resource_specification.go | 21 +++-- 9 files changed, 248 insertions(+), 111 deletions(-) create mode 100644 modules/convert/cloudbrain.go create mode 100644 routers/api/v1/repo/attachments.go diff --git a/models/resource_specification.go b/models/resource_specification.go index d69cd7208..2da8d015d 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -194,32 +194,6 @@ func (Specification) TableName() string { return "resource_specification" } -func (s *Specification) ToShow() *SpecificationShow { - return &SpecificationShow{ - ID: s.ID, - AccCardsNum: s.AccCardsNum, - AccCardType: s.AccCardType, - CpuCores: s.CpuCores, - MemGiB: s.MemGiB, - GPUMemGiB: s.GPUMemGiB, - ShareMemGiB: s.ShareMemGiB, - ComputeResource: s.ComputeResource, - UnitPrice: s.UnitPrice, - } -} - -type SpecificationShow struct { - ID int64 `json:"id"` - AccCardsNum int `json:"acc_cards_num"` - AccCardType string `json:"acc_card_type"` - CpuCores int `json:"cpu_cores"` - MemGiB float32 `json:"mem_gi_b"` - GPUMemGiB float32 `json:"gpu_mem_gi_b"` - ShareMemGiB float32 `json:"share_mem_gi_b"` - ComputeResource string `json:"compute_resource"` - UnitPrice int `json:"unit_price"` -} - func InsertResourceSpecification(r ResourceSpecification) (int64, error) { return x.Insert(&r) } diff --git a/modules/convert/cloudbrain.go b/modules/convert/cloudbrain.go new file mode 100644 index 000000000..7df38ee05 --- /dev/null +++ b/modules/convert/cloudbrain.go @@ -0,0 +1,103 @@ +package convert + +import ( + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" +) + +func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain { + return &api.Cloudbrain{ + ID: task.ID, + JobID: task.JobID, + JobType: task.JobType, + Type: task.Type, + DisplayJobName: task.DisplayJobName, + Status: task.Status, + CreatedUnix: int64(task.CreatedUnix), + RepoID: task.RepoID, + Duration: task.Duration, + TrainJobDuration: task.TrainJobDuration, + ImageID: task.ImageID, + Image: task.Image, + Uuid: task.Uuid, + DatasetName: task.DatasetName, + ComputeResource: task.ComputeResource, + AiCenter: task.AiCenter, + BranchName: task.BranchName, + Parameters: task.Parameters, + BootFile: task.BootFile, + Description: task.Description, + ModelName: task.ModelName, + + ModelVersion: task.ModelVersion, + CkptName: task.CkptName, + + StartTime: int64(task.StartTime), + EndTime: int64(task.EndTime), + + Spec: ToSpecification(task.Spec), + } +} +func ToAttachment(attachment *models.Attachment) *api.AttachmentShow { + return &api.AttachmentShow{ + ID: attachment.ID, + UUID: attachment.UUID, + DatasetID: attachment.DatasetID, + ReleaseID: attachment.ReleaseID, + UploaderID: attachment.UploaderID, + CommentID: attachment.CommentID, + Name: attachment.Name, + Description: attachment.Description, + DownloadCount: attachment.DownloadCount, + UseNumber: attachment.UseNumber, + Size: attachment.Size, + IsPrivate: attachment.IsPrivate, + DecompressState: attachment.DecompressState, + Type: attachment.Type, + CreatedUnix: int64(attachment.CreatedUnix), + } +} + +func ToDataset(dataset *models.Dataset) *api.Dataset { + var convertAttachments []*api.AttachmentShow + for _, attachment := range dataset.Attachments { + convertAttachments = append(convertAttachments, ToAttachment(attachment)) + } + return &api.Dataset{ + ID: dataset.ID, + Title: dataset.Title, + Status: dataset.Status, + Category: dataset.Category, + Description: dataset.Description, + DownloadTimes: dataset.DownloadTimes, + UseCount: dataset.UseCount, + NumStars: dataset.NumStars, + Recommend: dataset.Recommend, + License: dataset.License, + Task: dataset.Task, + ReleaseID: dataset.ReleaseID, + UserID: dataset.UserID, + RepoID: dataset.RepoID, + Repo: &api.RepositoryShow{ + OwnerName: dataset.Repo.OwnerName, + Name: dataset.Repo.Name, + }, + CreatedUnix: int64(dataset.CreatedUnix), + UpdatedUnix: int64(dataset.UpdatedUnix), + Attachments: convertAttachments, + } +} + +func ToSpecification(s *models.Specification) *api.SpecificationShow { + return &api.SpecificationShow{ + ID: s.ID, + AccCardsNum: s.AccCardsNum, + AccCardType: s.AccCardType, + CpuCores: s.CpuCores, + MemGiB: s.MemGiB, + GPUMemGiB: s.GPUMemGiB, + ShareMemGiB: s.ShareMemGiB, + ComputeResource: s.ComputeResource, + UnitPrice: s.UnitPrice, + } +} diff --git a/modules/structs/attachment.go b/modules/structs/attachment.go index 3ccc2742a..4a0f8feec 100755 --- a/modules/structs/attachment.go +++ b/modules/structs/attachment.go @@ -6,8 +6,6 @@ package structs // import "code.gitea.io/gitea/modules/structs" import ( "time" - - "code.gitea.io/gitea/modules/timeutil" ) // Attachment a generic attachment @@ -31,23 +29,23 @@ type EditAttachmentOptions struct { } type Dataset struct { - ID int64 `json:"id"` - Title string `json:"title"` - Status int32 `json:"status"` - Category string `json:"category"` - Description string `json:"description"` - DownloadTimes int64 `json:"downloadTimes"` - UseCount int64 `json:"useCount"` - NumStars int `json:"numStars"` - Recommend bool `json:"recommend"` - License string `json:"license"` - Task string `json:"task"` - ReleaseID int64 `json:"releaseId"` - UserID int64 `json:"userId"` - RepoID int64 `json:"repoId"` - Repo *RepositoryShow `json:"repo"` - CreatedUnix timeutil.TimeStamp `json:"createdUnix"` - UpdatedUnix timeutil.TimeStamp `json:"updatedUnix"` + ID int64 `json:"id"` + Title string `json:"title"` + Status int32 `json:"status"` + Category string `json:"category"` + Description string `json:"description"` + DownloadTimes int64 `json:"downloadTimes"` + UseCount int64 `json:"useCount"` + NumStars int `json:"numStars"` + Recommend bool `json:"recommend"` + License string `json:"license"` + Task string `json:"task"` + ReleaseID int64 `json:"releaseId"` + UserID int64 `json:"userId"` + RepoID int64 `json:"repoId"` + Repo *RepositoryShow `json:"repo"` + CreatedUnix int64 `json:"createdUnix"` + UpdatedUnix int64 `json:"updatedUnix"` Attachments []*AttachmentShow `json:"attachments"` } @@ -58,19 +56,19 @@ type RepositoryShow struct { } type AttachmentShow struct { - ID int64 `json:"id"` - UUID string `json:"uuid"` - DatasetID int64 `json:"datasetId"` - ReleaseID int64 `json:"releaseId"` - UploaderID int64 `json:"uploaderId"` - CommentID int64 `json:"commentId"` - Name string `json:"name"` - Description string `json:"description"` - DownloadCount int64 `json:"downloadCount"` - UseNumber int64 `json:"useNumber"` - Size int64 `json:"size"` - IsPrivate bool `json:"isPrivate"` - DecompressState int32 `json:"decompressState"` - Type int `json:"type"` - CreatedUnix timeutil.TimeStamp `json:"createdUnix"` + ID int64 `json:"id"` + UUID string `json:"uuid"` + DatasetID int64 `json:"datasetId"` + ReleaseID int64 `json:"releaseId"` + UploaderID int64 `json:"uploaderId"` + CommentID int64 `json:"commentId"` + Name string `json:"name"` + Description string `json:"description"` + DownloadCount int64 `json:"downloadCount"` + UseNumber int64 `json:"useNumber"` + Size int64 `json:"size"` + IsPrivate bool `json:"isPrivate"` + DecompressState int32 `json:"decompressState"` + Type int `json:"type"` + CreatedUnix int64 `json:"createdUnix"` } diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index e0409b340..b29d7e722 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -1,10 +1,5 @@ package structs -import ( - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/timeutil" -) - type CreateGrampusTrainJobOption struct { DisplayJobName string `json:"display_job_name" binding:"Required"` JobName string `json:"job_name" binding:"Required" ` @@ -47,36 +42,43 @@ type CreateTrainJobOption struct { } type Cloudbrain struct { - ID int64 `json:"id"` - JobID string `json:"job_id"` - JobType string `json:"job_type"` - Type int `json:"type"` - DisplayJobName string `json:"display_job_name"` - Status string `json:"status"` - CreatedUnix timeutil.TimeStamp `json:"created_unix"` - RepoID int64 `json:"repo_id"` - Duration int64 `json:"duration"` //运行时长 单位秒 - TrainJobDuration string `json:"train_job_duration"` - ImageID string `json:"image_id"` //grampus image_id - Image string `json:"image"` - Uuid string `json:"uuid"` //数据集id - DatasetName string `json:"dataset_name"` - ComputeResource string `json:"compute_resource"` //计算资源，例如npu - AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name - BranchName string `json:"branch_name"` //分支名称 - Parameters string `json:"parameters"` //传给modelarts的param参数 - BootFile string `json:"boot_file"` //启动文件 - Description string `json:"description"` //描述 - ModelName string `json:"model_name"` //模型名称 - ModelVersion string `json:"model_version"` //模型版本 - CkptName string `json:"ckpt_name"` //权重文件名称 + ID int64 `json:"id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + Type int `json:"type"` + DisplayJobName string `json:"display_job_name"` + Status string `json:"status"` + CreatedUnix int64 `json:"created_unix"` + RepoID int64 `json:"repo_id"` + Duration int64 `json:"duration"` //运行时长 单位秒 + TrainJobDuration string `json:"train_job_duration"` + ImageID string `json:"image_id"` //grampus image_id + Image string `json:"image"` + Uuid string `json:"uuid"` //数据集id + DatasetName string `json:"dataset_name"` + ComputeResource string `json:"compute_resource"` //计算资源，例如npu + AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name + BranchName string `json:"branch_name"` //分支名称 + Parameters string `json:"parameters"` //传给modelarts的param参数 + BootFile string `json:"boot_file"` //启动文件 + Description string `json:"description"` //描述 + ModelName string `json:"model_name"` //模型名称 + ModelVersion string `json:"model_version"` //模型版本 + CkptName string `json:"ckpt_name"` //权重文件名称 + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` - VersionCount int `json:"-"` //任务的当前版本数量，不包括删除的 - IsLatestVersion string `json:"-"` //是否是最新版本，1是，0否 - CommitID string `json:"-"` //提交的仓库代码id - PreVersionName string `json:"-"` //父版本名称 - StartTime timeutil.TimeStamp `json:"start_time"` - EndTime timeutil.TimeStamp `json:"end_time"` + Spec *SpecificationShow `json:"spec"` +} - Spec *models.Specification `json:"spec"` +type SpecificationShow struct { + ID int64 `json:"id"` + AccCardsNum int `json:"acc_cards_num"` + AccCardType string `json:"acc_card_type"` + CpuCores int `json:"cpu_cores"` + MemGiB float32 `json:"mem_gi_b"` + GPUMemGiB float32 `json:"gpu_mem_gi_b"` + ShareMemGiB float32 `json:"share_mem_gi_b"` + ComputeResource string `json:"compute_resource"` + UnitPrice int `json:"unit_price"` } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 857d810f8..9f8ddd5b5 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -242,6 +242,15 @@ func reqRepoWriter(unitTypes ...models.UnitType) macaron.Handler { } } +func reqWeChat() macaron.Handler { + return func(ctx *context.Context) { + if setting.WechatAuthSwitch && ctx.User.WechatOpenId == "" { + ctx.JSON(http.StatusForbidden, models.BaseErrorMessageApi("settings.no_wechat_bind")) + return + } + } +} + // reqRepoReader user should have specific read permission or be a repo admin or a site admin func reqRepoReader(unitType models.UnitType) macaron.Handler { return func(ctx *context.Context) { @@ -526,6 +535,16 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken()) + m.Group("/attachments", func() { + + m.Get("/:uuid", repo.GetAttachment) + m.Get("/get_chunks", repo.GetSuccessChunks) + m.Get("/new_multipart", repo.NewMultipart) + m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) + m.Post("/complete_multipart", repo.CompleteMultipart) + + }, reqToken()) + // Notifications m.Group("/notifications", func() { m.Combo(""). @@ -982,15 +1001,17 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqRepoReader(models.UnitTypeModelManage)) m.Group("/cloudbrain", func() { m.Group("/train-job", func() { - m.Post("/create", bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) + m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) + + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat, bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) }) m.Group("/inference-job", func() { - m.Post("/create", bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) - + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat, bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) + m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) }) - }, reqRepoReader(models.UnitTypeCloudBrain)) + }, reqToken()) m.Group("/modelarts", func() { m.Group("/notebook", func() { //m.Get("/:jobid", repo.GetModelArtsNotebook) diff --git a/routers/api/v1/repo/attachments.go b/routers/api/v1/repo/attachments.go new file mode 100644 index 000000000..cb36ba2ee --- /dev/null +++ b/routers/api/v1/repo/attachments.go @@ -0,0 +1,25 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/context" + routeRepo "code.gitea.io/gitea/routers/repo" +) + +func GetSuccessChunks(ctx *context.APIContext) { + routeRepo.GetSuccessChunks(ctx.Context) +} + +func NewMultipart(ctx *context.APIContext) { + routeRepo.NewMultipart(ctx.Context) +} +func GetMultipartUploadUrl(ctx *context.APIContext) { + routeRepo.GetMultipartUploadUrl(ctx.Context) +} + +func CompleteMultipart(ctx *context.APIContext) { + routeRepo.CompleteMultipart(ctx.Context) + +} +func GetAttachment(ctx *context.APIContext) { + routeRepo.GetAttachment(ctx.Context) +} diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 380de0ddb..7092a1f02 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" api "code.gitea.io/gitea/modules/structs" @@ -71,7 +73,7 @@ func CloudBrainShow(ctx *context.APIContext) { task.Image = task.EngineName } - ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: task}) + ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)}) } diff --git a/routers/api/v1/repo/datasets.go b/routers/api/v1/repo/datasets.go index 0539a22e9..0223cf740 100644 --- a/routers/api/v1/repo/datasets.go +++ b/routers/api/v1/repo/datasets.go @@ -4,6 +4,10 @@ import ( "fmt" "strings" + "code.gitea.io/gitea/modules/convert" + + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -88,15 +92,20 @@ func datasetMultiple(ctx *context.APIContext, opts *models.SearchDatasetOptions) ctx.JSON(200, map[string]interface{}{ "code": 1, "message": err.Error(), - "data": []*models.Dataset{}, + "data": []*api.Dataset{}, "count": 0, }) return } + var convertDatasets []*api.Dataset + for _, dataset := range datasets { + convertDatasets = append(convertDatasets, convert.ToDataset(dataset)) + } + ctx.JSON(200, map[string]interface{}{ "code": 0, "message": "", - "data": datasets, + "data": convertDatasets, "count": count, }) } diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index c0b20b21a..93abb2923 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -1,20 +1,23 @@ package resource import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/routers/response" "code.gitea.io/gitea/services/admin/operate_log" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" - "time" ) func AddResourceSpecification(doerId int64, req models.ResourceSpecificationReq) error { @@ -210,14 +213,14 @@ func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.S return specs, err } -func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*models.SpecificationShow, error) { +func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*api.SpecificationShow, error) { specs, err := FindAvailableSpecs(userId, opts) if err != nil { return nil, err } - result := make([]*models.SpecificationShow, len(specs)) + result := make([]*api.SpecificationShow, len(specs)) for i, v := range specs { - result[i] = v.ToShow() + result[i] = convert.ToSpecification(v) } return result, nil } From 6cea45e40a7fc9cf3aad5c57d2120d4e40982f25 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 28 Oct 2022 09:17:03 +0800 Subject: [PATCH 35/94] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E4=BF=AE=E6=94=B9=E6=88=90=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E7=9A=84=E6=96=B9=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/repo/modelmanage.go | 25 ++++++++++++++++++++++++- routers/repo/ai_model_manage.go | 12 +++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index b32726997..b404558a1 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -8,6 +8,15 @@ import ( routerRepo "code.gitea.io/gitea/routers/repo" ) +type FileInfo struct { + FileName string `json:"fileName"` + ModTime string `json:"modTime"` + IsDir bool `json:"isDir"` + Size int64 `json:"size"` + ParenDir string `json:"parenDir"` + UUID string `json:"uuid"` +} + func CreateNewModel(ctx *context.APIContext) { log.Info("CreateNewModel by api.") routerRepo.SaveModel(ctx.Context) @@ -35,7 +44,21 @@ func QueryModelListForPredict(ctx *context.APIContext) { func QueryModelFileForPredict(ctx *context.APIContext) { log.Info("QueryModelFileForPredict by api.") - routerRepo.QueryModelFileForPredict(ctx.Context) + id := ctx.Query("id") + result := routerRepo.QueryModelFileByID(id) + re := make([]FileInfo, len(result)) + for _, file := range result { + tmpFile := FileInfo{ + FileName: file.FileName, + ModTime: file.ModTime, + IsDir: file.IsDir, + Size: file.Size, + ParenDir: file.ParenDir, + UUID: file.UUID, + } + re = append(re, tmpFile) + } + ctx.JSON(http.StatusOK, re) } func CreateModelConvert(ctx *context.APIContext) { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 6385cc674..45c21d719 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -910,23 +910,25 @@ func QueryModelFileForPredict(ctx *context.Context) { if id == "" { id = ctx.Query("ID") } + ctx.JSON(http.StatusOK, QueryModelFileByID(id)) +} + +func QueryModelFileByID(id string) []storage.FileInfo { model, err := models.QueryModelById(id) if err == nil { if model.Type == models.TypeCloudBrainTwo { prefix := model.Path[len(setting.Bucket)+1:] fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix) - ctx.JSON(http.StatusOK, fileinfos) + return fileinfos } else if model.Type == models.TypeCloudBrainOne { prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix) - ctx.JSON(http.StatusOK, fileinfos) + return fileinfos } } else { log.Error("no such model!", err.Error()) - ctx.ServerError("no such model:", err) - return } - + return nil } func QueryOneLevelModelFile(ctx *context.Context) { From 187eec79b2f8cdc961944f1fb20f0b405832f510 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 31 Oct 2022 09:45:41 +0800 Subject: [PATCH 36/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 1 + routers/api/v1/repo/modelmanage.go | 5 ++++ routers/repo/ai_model_manage.go | 49 +++++++++++++++++++++++++------------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 9f8ddd5b5..5ecbf3a5f 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -990,6 +990,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/show_model_api", repo.ShowModelManageApi) m.Delete("/delete_model", repo.DeleteModel) m.Get("/downloadall", repo.DownloadModel) + m.Get("/query_model_byId", repo.QueryModelById) m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) m.Post("/create_model_convert", repo.CreateModelConvert) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index b404558a1..253d51e00 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -37,6 +37,11 @@ func DownloadModel(ctx *context.APIContext) { routerRepo.DownloadMultiModelFile(ctx.Context) } +func QueryModelById(ctx *context.APIContext) { + log.Info("QueryModelById by api.") + routerRepo.QueryModelById(ctx.Context) +} + func QueryModelListForPredict(ctx *context.APIContext) { log.Info("QueryModelListForPredict by api.") routerRepo.QueryModelListForPredict(ctx.Context) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 45c21d719..ca90ff432 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -33,13 +33,13 @@ const ( STATUS_ERROR = 2 ) -func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) (string, error) { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) if err != nil { aiTask, err = models.GetRepoCloudBrainByJobID(ctx.Repo.Repository.ID, jobId) if err != nil { log.Info("query task error." + err.Error()) - return err + return "", err } else { log.Info("query gpu train task.") } @@ -55,7 +55,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio if len(aimodels) > 0 { for _, model := range aimodels { if model.Version == version { - return errors.New(ctx.Tr("repo.model.manage.create_error")) + return "", errors.New(ctx.Tr("repo.model.manage.create_error")) } if model.New == MODEL_LATEST { lastNewModelId = model.ID @@ -113,7 +113,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio err = models.SaveModelToDb(model) if err != nil { - return err + return "", err } if len(lastNewModelId) > 0 { //udpate status and version count @@ -136,7 +136,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio log.Info("save model end.") notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, id, name, models.ActionCreateNewModelTask) - return nil + return id, nil } func asyncToCopyModel(aiTask *models.Cloudbrain, id string, modelSelectedFile string) { @@ -206,29 +206,33 @@ func SaveModel(ctx *context.Context) { engine := ctx.QueryInt("engine") modelSelectedFile := ctx.Query("modelSelectedFile") log.Info("engine=" + fmt.Sprint(engine) + " modelSelectedFile=" + modelSelectedFile) - + re := map[string]string{ + "code": "-1", + } if JobId == "" || VersionName == "" { - ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) + re["msg"] = "JobId or VersionName is null." + ctx.JSON(200, re) return } if modelSelectedFile == "" { - ctx.Error(500, fmt.Sprintf("Not selected model file.")) + re["msg"] = "Not selected model file." + ctx.JSON(200, re) return } - if name == "" || version == "" { - ctx.Error(500, fmt.Sprintf("name or version is null.")) + re["msg"] = "name or version is null." + ctx.JSON(200, re) return } - - err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) - + id, err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) if err != nil { log.Info("save model error." + err.Error()) - ctx.Error(500, fmt.Sprintf("save model error. %v", err)) - return + re["msg"] = err.Error() + } else { + re["code"] = "0" + re["id"] = id } - ctx.Status(200) + ctx.JSON(200, re) log.Info("save model end.") } @@ -632,6 +636,19 @@ func ShowModelInfo(ctx *context.Context) { ctx.HTML(200, tplModelInfo) } +func QueryModelById(ctx *context.Context) { + id := ctx.Query("id") + model, err := models.QueryModelById(id) + if err == nil { + model.IsCanOper = isOper(ctx, model.UserId) + model.IsCanDelete = isCanDelete(ctx, model.UserId) + removeIpInfo(model) + ctx.JSON(http.StatusOK, model) + } else { + ctx.JSON(http.StatusNotFound, nil) + } +} + func ShowSingleModel(ctx *context.Context) { name := ctx.Query("name") From a4873ff88a40081e6fc3ec20d9cf1e589ad28104 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 31 Oct 2022 15:30:05 +0800 Subject: [PATCH 37/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 16 +- routers/api/v1/repo/cloudbrain.go | 10 +- routers/repo/modelarts.go | 2 +- services/cloudbrain/cloudbrainTask/train.go | 421 +++++++++++++++++++++++++++- 4 files changed, 427 insertions(+), 22 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 4158174cb..e94dd4ad9 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -348,7 +348,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc return nil } -func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) { +func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) { createTime := timeutil.TimeStampNow() var jobResult *models.CreateTrainJobResult var createErr error @@ -408,17 +408,17 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error }) if errTemp != nil { log.Error("InsertCloudbrainTemp failed: %v", errTemp.Error()) - return errTemp + return "", errTemp } } - return createErr + return "", createErr } - jobId := strconv.FormatInt(jobResult.JobID, 10) + jobID := strconv.FormatInt(jobResult.JobID, 10) createErr = models.CreateCloudbrain(&models.Cloudbrain{ Status: TransTrainJobStatus(jobResult.Status), UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, - JobID: jobId, + JobID: jobID, JobName: req.JobName, DisplayJobName: req.DisplayJobName, JobType: string(models.JobTypeTrain), @@ -456,10 +456,10 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error if createErr != nil { log.Error("CreateCloudbrain(%s) failed:%v", req.DisplayJobName, createErr.Error()) - return createErr + return "", createErr } - notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobId, req.DisplayJobName, models.ActionCreateTrainTask) - return nil + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateTrainTask) + return jobID, nil } func GenerateModelConvertTrainJob(req *GenerateTrainJobReq) (*models.CreateTrainJobResult, error) { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 7092a1f02..649e33956 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -78,11 +78,17 @@ func CloudBrainShow(ctx *context.APIContext) { } func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { + if option.Type == cloudbrainTask.TaskTypeCloudbrainOne { + cloudbrainTask.CloudbrainOneTrainJobCreate(ctx.Context, option) + } + if option.Type == cloudbrainTask.TaskTypeModelArts { + cloudbrainTask.ModelArtsTrainJobNpuCreate(ctx.Context, option) + } - if option.Type == 2 { + if option.Type == cloudbrainTask.TaskTypeGrampusGPU { cloudbrainTask.GrampusTrainJobGpuCreate(ctx.Context, option) } - if option.Type == 3 { + if option.Type == cloudbrainTask.TaskTypeGrampusNPU { cloudbrainTask.GrampusTrainJobNpuCreate(ctx.Context, option) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index d187b211a..e4e73f716 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1230,7 +1230,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - err = modelarts.GenerateTrainJob(ctx, req) + _, err = modelarts.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) trainJobNewDataPrepare(ctx) diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 6f836557c..5e2cce1ff 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -10,6 +10,7 @@ import ( "os" "path" "regexp" + "strconv" "strings" "code.gitea.io/gitea/modules/timeutil" @@ -39,6 +40,277 @@ import ( var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) +const TaskTypeCloudbrainOne = 0 +const TaskTypeModelArts = 1 +const TaskTypeGrampusGPU = 2 +const TaskTypeGrampusNPU = 3 + +func CloudbrainOneTrainJobCreate(ctx *context.Context, option api.CreateTrainJobOption) { + + displayJobName := option.DisplayJobName + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + image := strings.TrimSpace(option.Image) + uuids := option.Attachment + jobType := string(models.JobTypeTrain) + + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + branchName := option.BranchName + repo := ctx.Repo.Repository + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + defer lock.UnLock() + spec, datasetInfos, datasetNames, err := checkParameters(ctx, option, lock, repo) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + command, err := getTrainJobCommand(option) + if err != nil { + log.Error("getTrainJobCommand failed: %v", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath) + if errStr != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr(errStr))) + return + } + + commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName) + + req := cloudbrain.GenerateCloudBrainTaskReq{ + Ctx: ctx, + DisplayJobName: displayJobName, + JobName: jobName, + Image: image, + Command: command, + Uuids: uuids, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, + CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), + Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + JobType: jobType, + Description: option.Description, + BranchName: branchName, + BootFile: option.BootFile, + Params: option.Params, + CommitID: commitID, + BenchmarkTypeID: 0, + BenchmarkChildTypeID: 0, + ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), + Spec: spec, + } + + if option.ModelName != "" { //使用预训练模型训练 + req.ModelName = option.ModelName + req.LabelName = option.LabelName + req.CkptName = option.CkptName + req.ModelVersion = option.ModelVersion + req.PreTrainModelPath = setting.Attachment.Minio.RealPath + option.PreTrainModelUrl + req.PreTrainModelUrl = option.PreTrainModelUrl + + } + + jobId, err := cloudbrain.GenerateTask(req) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) +} +func ModelArtsTrainJobNpuCreate(ctx *context.Context, option api.CreateTrainJobOption) { + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + displayJobName := option.DisplayJobName + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + uuid := option.Attachment + description := option.Description + workServerNumber := option.WorkServerNumber + engineID, _ := strconv.Atoi(option.ImageID) + bootFile := strings.TrimSpace(option.BootFile) + params := option.Params + repo := ctx.Repo.Repository + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" + branchName := option.BranchName + isLatestVersion := modelarts.IsLatestVersion + VersionCount := modelarts.VersionCountOne + EngineName := option.Image + + errStr := checkMultiNode(ctx.User.ID, option.WorkServerNumber) + if errStr != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr(errStr))) + return + } + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + defer lock.UnLock() + + spec, _, _, err := checkParameters(ctx, option, lock, repo) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + //todo: del the codeLocalPath + _, err = ioutil.ReadDir(codeLocalPath) + if err == nil { + os.RemoveAll(codeLocalPath) + } + + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(branchName) + + if err := downloadCode(repo, codeLocalPath, branchName); err != nil { + log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + + //todo: upload code (send to file_server todo this work?) + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Failed to obsMkdir_output")) + return + } + + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Failed to obsMkdir_log")) + return + } + + parentDir := VersionOutputPath + "/" + if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { + // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { + log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + + var parameters models.Parameters + param := make([]models.Parameter, 0) + existDeviceTarget := false + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("运行参数错误")) + return + } + + for _, parameter := range parameters.Parameter { + if parameter.Label == modelarts.DeviceTarget { + existDeviceTarget = true + } + if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl { + param = append(param, models.Parameter{ + Label: parameter.Label, + Value: parameter.Value, + }) + } + } + } + if !existDeviceTarget { + param = append(param, models.Parameter{ + Label: modelarts.DeviceTarget, + Value: modelarts.Ascend, + }) + } + datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid) + if err != nil { + log.Error("Failed to getDatasUrlListByUUIDS: %v", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Failed to getDatasUrlListByUUIDS:"+err.Error())) + return + } + dataPath := dataUrl + jsondatas, err := json.Marshal(datasUrlList) + if err != nil { + log.Error("Failed to Marshal: %v", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("json error:"+err.Error())) + return + } + if isMultiDataset { + param = append(param, models.Parameter{ + Label: modelarts.MultiDataUrl, + Value: string(jsondatas), + }) + } + if option.ModelName != "" { //使用预训练模型训练 + ckptUrl := "/" + option.PreTrainModelUrl + option.CkptName + param = append(param, models.Parameter{ + Label: modelarts.CkptUrl, + Value: "s3:/" + ckptUrl, + }) + } + + req := &modelarts.GenerateTrainJobReq{ + JobName: jobName, + DisplayJobName: displayJobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + bootFile, + BootFile: bootFile, + TrainUrl: outputObsPath, + WorkServerNumber: workServerNumber, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: getPoolId(), + Uuid: uuid, + Parameters: param, + CommitID: commitID, + IsLatestVersion: isLatestVersion, + BranchName: branchName, + Params: option.Params, + EngineName: EngineName, + VersionCount: VersionCount, + TotalVersionCount: modelarts.TotalVersionCount, + DatasetName: datasetNames, + Spec: spec, + } + if option.ModelName != "" { //使用预训练模型训练 + req.ModelName = option.ModelName + req.LabelName = option.LabelName + req.CkptName = option.CkptName + req.ModelVersion = option.ModelVersion + req.PreTrainModelUrl = option.PreTrainModelUrl + + } + + userCommand, userImageUrl := getUserCommand(engineID, req) + req.UserCommand = userCommand + req.UserImageUrl = userImageUrl + + //将params转换Parameters.Parameter，出错时返回给前端 + var Parameters modelarts.Parameters + if err := json.Unmarshal([]byte(params), &Parameters); err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("json.Unmarshal failed:"+err.Error())) + return + } + + jobId, err := modelarts.GenerateTrainJob(ctx, req) + if err != nil { + log.Error("GenerateTrainJob failed:%v", err.Error()) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) + +} + func GrampusTrainJobGpuCreate(ctx *context.Context, option api.CreateTrainJobOption) { displayJobName := option.DisplayJobName @@ -179,13 +451,19 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock } computeResource := models.GPUResource - if option.Type == 3 { + if isNpuTask(option) { computeResource = models.NPUResource } + //check count limit - count, err := GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeTrain), computeResource) + taskType := option.Type + if isC2NetTask(option) { + taskType = 2 + } + + count, err := GetNotFinalStatusTaskCount(ctx.User.ID, taskType, string(models.JobTypeTrain), computeResource) if err != nil { - log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) + log.Error("GetCountByUserID failed:%v", err, ctx.Data["MsgID"]) return nil, nil, "", fmt.Errorf("system error") } else { if count >= 1 { @@ -195,7 +473,7 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock } //check param - if err := grampusParamCheckCreateTrainJob(option.BootFile, option.BranchName); err != nil { + if err := paramCheckCreateTrainJob(option.BootFile, option.BranchName); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err, ctx.Data["MsgID"]) return nil, nil, "", err } @@ -216,13 +494,26 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock //check specification computeType := models.GPU - if option.Type == 3 { + + if isNpuTask(option) { computeType = models.NPU } + cluster := models.OpenICluster + if isC2NetTask(option) { + cluster = models.C2NetCluster + } + aiCenterCode := "" + if option.Type == TaskTypeCloudbrainOne { + aiCenterCode = models.AICenterOfCloudBrainOne + } else if option.Type == TaskTypeModelArts { + aiCenterCode = models.AICenterOfCloudBrainTwo + } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, option.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeTrain, ComputeResource: computeType, - Cluster: models.C2NetCluster, + Cluster: cluster, + AiCenterCode: aiCenterCode, }) if err != nil || spec == nil { return nil, nil, "", fmt.Errorf("Resource specification is not available.") @@ -234,14 +525,26 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock } //check dataset - datasetInfos, datasetNames, err := models.GetDatasetInfo(option.Attachment, computeType) - if err != nil { - log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - return nil, nil, "", fmt.Errorf(ctx.Tr("cloudbrain.error.dataset_select")) + var datasetInfos map[string]models.DatasetInfo + var datasetNames string + if option.Type != TaskTypeModelArts { + datasetInfos, datasetNames, err = models.GetDatasetInfo(option.Attachment, computeType) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil, nil, "", fmt.Errorf(ctx.Tr("cloudbrain.error.dataset_select")) + } } return spec, datasetInfos, datasetNames, err } +func isNpuTask(option api.CreateTrainJobOption) bool { + return option.Type == TaskTypeModelArts || option.Type == TaskTypeGrampusNPU +} + +func isC2NetTask(option api.CreateTrainJobOption) bool { + return option.Type == TaskTypeGrampusGPU || option.Type == TaskTypeGrampusNPU +} + func GrampusTrainJobNpuCreate(ctx *context.Context, option api.CreateTrainJobOption) { displayJobName := option.DisplayJobName @@ -412,7 +715,7 @@ func uploadCodeToObs(codePath, jobName, parentDir string) error { return nil } -func grampusParamCheckCreateTrainJob(bootFile string, branchName string) error { +func paramCheckCreateTrainJob(bootFile string, branchName string) error { if !strings.HasSuffix(strings.TrimSpace(bootFile), ".py") { log.Error("the boot file(%s) must be a python file", bootFile) return errors.New("启动文件必须是python文件") @@ -792,3 +1095,99 @@ func SyncTaskStatus(task *models.Cloudbrain) error { return nil } + +func getTrainJobCommand(option api.CreateTrainJobOption) (string, error) { + var command string + bootFile := strings.TrimSpace(option.BootFile) + params := option.Params + + if !strings.HasSuffix(bootFile, ".py") { + log.Error("bootFile(%s) format error", bootFile) + return command, errors.New("bootFile format error") + } + + var parameters models.Parameters + var param string + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + return command, err + } + + for _, parameter := range parameters.Parameter { + param += " --" + parameter.Label + "=" + parameter.Value + } + } + if option.CkptName != "" { + param += " --ckpt_url" + "=" + "/pretrainmodel/" + option.CkptName + } + + command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + option.DisplayJobName + "-" + cloudbrain.LogFile + + return command, nil +} + +func checkMultiNode(userId int64, serverNum int) string { + if serverNum == 1 { + return "" + } + modelarts.InitMultiNode() + var isServerNumValid = false + if modelarts.MultiNodeConfig != nil { + for _, info := range modelarts.MultiNodeConfig.Info { + if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, userId); isInOrg { + if isInNodes(info.Node, serverNum) { + isServerNumValid = true + break + } + + } + } + } + if isServerNumValid { + return "" + } else { + return "repo.modelarts.no_node_right" + } +} + +func isInNodes(nodes []int, num int) bool { + for _, node := range nodes { + if node == num { + return true + } + } + return false + +} + +func getUserCommand(engineId int, req *modelarts.GenerateTrainJobReq) (string, string) { + userImageUrl := "" + userCommand := "" + if engineId < 0 { + tmpCodeObsPath := strings.Trim(req.CodeObsPath, "/") + tmpCodeObsPaths := strings.Split(tmpCodeObsPath, "/") + lastCodeDir := "code" + if len(tmpCodeObsPaths) > 0 { + lastCodeDir = tmpCodeObsPaths[len(tmpCodeObsPaths)-1] + } + userCommand = "/bin/bash /home/work/run_train.sh 's3://" + req.CodeObsPath + "' '" + lastCodeDir + "/" + req.BootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + req.DataUrl + "' --'train_url'='s3://" + req.TrainUrl + "'" + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + log.Info("json parse err." + err.Error()) + } else { + for _, engine := range versionInfos.Version { + if engine.ID == engineId { + userImageUrl = engine.Url + break + } + } + } + for _, param := range req.Parameters { + userCommand += " --'" + param.Label + "'='" + param.Value + "'" + } + return userCommand, userImageUrl + } + return userCommand, userImageUrl +} From 670c13abe6a0ac07cec1c8f83fd8abf19846cb51 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 1 Nov 2022 16:20:27 +0800 Subject: [PATCH 38/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/convert/cloudbrain.go | 8 ++++++ modules/structs/tagger.go | 7 +++++ routers/api/v1/repo/mlops.go | 63 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 modules/structs/tagger.go create mode 100644 routers/api/v1/repo/mlops.go diff --git a/modules/convert/cloudbrain.go b/modules/convert/cloudbrain.go index 7df38ee05..1487f468e 100644 --- a/modules/convert/cloudbrain.go +++ b/modules/convert/cloudbrain.go @@ -101,3 +101,11 @@ func ToSpecification(s *models.Specification) *api.SpecificationShow { UnitPrice: s.UnitPrice, } } + +func ToTagger(user *models.User) *api.Tagger { + return &api.Tagger{ + Name: user.Name, + RelAvatarURL: user.RelAvatarLink(), + Email: user.Email, + } +} diff --git a/modules/structs/tagger.go b/modules/structs/tagger.go new file mode 100644 index 000000000..8933c8c5c --- /dev/null +++ b/modules/structs/tagger.go @@ -0,0 +1,7 @@ +package structs + +type Tagger struct { + Name string `json:"name"` + Email string `json:"email"` + RelAvatarURL string `json:"relAvatarURL"` +} diff --git a/routers/api/v1/repo/mlops.go b/routers/api/v1/repo/mlops.go new file mode 100644 index 000000000..3392ebfec --- /dev/null +++ b/routers/api/v1/repo/mlops.go @@ -0,0 +1,63 @@ +package repo + +import ( + "net/http" + + "code.gitea.io/gitea/models" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/modules/log" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/routers/api/v1/utils" +) + +//标注任务可分配人员 +func ListTagger(ctx *context.APIContext) { + + taggers := make([]*api.Tagger, 0) + userRemember := make(map[string]string) + collaborators, err := ctx.Repo.Repository.GetCollaborators(utils.GetListOptions(ctx)) + if err != nil { + log.Warn("ListCollaborators", err) + ctx.JSON(http.StatusOK, taggers) + return + } + for _, collaborator := range collaborators { + taggers = append(taggers, convert.ToTagger(collaborator.User)) + userRemember[collaborator.User.Name] = "" + } + + teams, err := ctx.Repo.Repository.GetRepoTeams() + if err != nil { + log.Warn("ListTeams", err) + ctx.JSON(http.StatusOK, taggers) + return + } + + for _, team := range teams { + for _, user := range team.Members { + if _, ok := userRemember[user.Name]; !ok { + taggers = append(taggers, convert.ToTagger(user)) + } + } + } + ctx.JSON(http.StatusOK, taggers) + +} +func GetRight(ctx *context.APIContext) { + right := "none" + + if ctx.IsUserRepoReaderSpecific(models.UnitTypeCode) { + right = "read" + } + + if ctx.IsUserRepoWriter([]models.UnitType{models.UnitTypeCode}) || ctx.IsUserRepoAdmin() { + right = "write" + } + + ctx.JSON(http.StatusOK, map[string]string{ + "right": right, + }) + +} From 7efc877097a6299523972f8ace0fc9df656a9dbf Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 1 Nov 2022 17:16:28 +0800 Subject: [PATCH 39/94] =?UTF-8?q?=E5=A2=9E=E5=8A=A06.8=E7=9A=84=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 3 +++ routers/api/v1/repo/modelmanage.go | 41 +++++++++++++++++++++--------- routers/repo/ai_model_manage.go | 52 +++++++++++++++++++++++++++++--------- 3 files changed, 72 insertions(+), 24 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 5ecbf3a5f..6e2c74b0b 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -993,6 +993,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_model_byId", repo.QueryModelById) m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) + m.Get("/query_train_job", repo.QueryTrainJobList) + m.Get("/query_train_model", repo.QueryTrainModelList) + m.Get("/query_train_job_version", repo.QueryTrainJobVersionList) m.Post("/create_model_convert", repo.CreateModelConvert) m.Get("/show_model_convert_page") m.Get("/:id", repo.GetCloudbrainModelConvertTask) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index 253d51e00..2c1fd9f01 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" routerRepo "code.gitea.io/gitea/routers/repo" ) @@ -47,22 +48,38 @@ func QueryModelListForPredict(ctx *context.APIContext) { routerRepo.QueryModelListForPredict(ctx.Context) } +func QueryTrainModelList(ctx *context.APIContext) { + result, err := routerRepo.QueryTrainModelFileById(ctx.Context) + if err != nil { + log.Info("query error." + err.Error()) + } + re := convertFileFormat(result) + ctx.JSON(http.StatusOK, re) +} + +func convertFileFormat(result []storage.FileInfo) []FileInfo { + re := make([]FileInfo, 0) + if result != nil { + for _, file := range result { + tmpFile := FileInfo{ + FileName: file.FileName, + ModTime: file.ModTime, + IsDir: file.IsDir, + Size: file.Size, + ParenDir: file.ParenDir, + UUID: file.UUID, + } + re = append(re, tmpFile) + } + } + return re +} + func QueryModelFileForPredict(ctx *context.APIContext) { log.Info("QueryModelFileForPredict by api.") id := ctx.Query("id") result := routerRepo.QueryModelFileByID(id) - re := make([]FileInfo, len(result)) - for _, file := range result { - tmpFile := FileInfo{ - FileName: file.FileName, - ModTime: file.ModTime, - IsDir: file.IsDir, - Size: file.Size, - ParenDir: file.ParenDir, - UUID: file.UUID, - } - re = append(re, tmpFile) - } + re := convertFileFormat(result) ctx.JSON(http.StatusOK, re) } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index ca90ff432..99f032572 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -528,23 +528,33 @@ func QueryTrainJobList(ctx *context.Context) { } -func QueryTrainModelList(ctx *context.Context) { - log.Info("query train job list. start.") - jobName := ctx.Query("jobName") - taskType := ctx.QueryInt("type") - VersionName := ctx.Query("versionName") - if VersionName == "" { - VersionName = ctx.Query("VersionName") +func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) { + JobID := ctx.Query("jobId") + VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) + if err == nil { + if count == 1 { + task := VersionListTasks[0] + jobName := task.JobName + taskType := task.Type + VersionName := task.VersionName + modelDbResult, err := getModelFromObjectSave(jobName, taskType, VersionName) + return modelDbResult, err + } } + log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + return nil, errors.New("Not found task.") +} + +func getModelFromObjectSave(jobName string, taskType int, VersionName string) ([]storage.FileInfo, error) { if taskType == models.TypeCloudBrainTwo { objectkey := path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionName) + "/" modelDbResult, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, objectkey) log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + return nil, err } else { - ctx.JSON(200, modelDbResult) - return + return modelDbResult, nil } } else if taskType == models.TypeCloudBrainOne { modelSrcPrefix := setting.CBCodePathPrefix + jobName + "/model/" @@ -552,12 +562,30 @@ func QueryTrainModelList(ctx *context.Context) { modelDbResult, err := storage.GetAllObjectByBucketAndPrefixMinio(bucketName, modelSrcPrefix) if err != nil { log.Info("get TypeCloudBrainOne TrainJobListModel failed:", err) + return nil, err } else { - ctx.JSON(200, modelDbResult) - return + return modelDbResult, nil } } - ctx.JSON(200, "") + return nil, errors.New("Not support.") +} + +func QueryTrainModelList(ctx *context.Context) { + log.Info("query train job list. start.") + jobName := ctx.Query("jobName") + taskType := ctx.QueryInt("type") + VersionName := ctx.Query("versionName") + if VersionName == "" { + VersionName = ctx.Query("VersionName") + } + modelDbResult, err := getModelFromObjectSave(jobName, taskType, VersionName) + if err != nil { + log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + ctx.JSON(200, "") + } else { + ctx.JSON(200, modelDbResult) + return + } } func DownloadSingleModelFile(ctx *context.Context) { From 4253a45af450e1634fd7759b4af0755a64067581 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 2 Nov 2022 08:55:06 +0800 Subject: [PATCH 40/94] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6e2c74b0b..049fafcaf 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -993,9 +993,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_model_byId", repo.QueryModelById) m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) - m.Get("/query_train_job", repo.QueryTrainJobList) m.Get("/query_train_model", repo.QueryTrainModelList) - m.Get("/query_train_job_version", repo.QueryTrainJobVersionList) m.Post("/create_model_convert", repo.CreateModelConvert) m.Get("/show_model_convert_page") m.Get("/:id", repo.GetCloudbrainModelConvertTask) From 0cd36f95056c759cdc072970a2e527db4075f411 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 2 Nov 2022 14:35:27 +0800 Subject: [PATCH 41/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 049fafcaf..485d26090 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -528,10 +528,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/images", func() { - m.Get("public", repo.GetPublicImages) - m.Get("custom", repo.GetCustomImages) - m.Get("star", repo.GetStarImages) - m.Get("npu", repo.GetNpuImages) + m.Get("/public", repo.GetPublicImages) + m.Get("/custom", repo.GetCustomImages) + m.Get("/star", repo.GetStarImages) + m.Get("/npu", repo.GetNpuImages) }, reqToken()) From a260eb539eebef210f2843de1223ccf7e8dfcb97 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 2 Nov 2022 14:41:51 +0800 Subject: [PATCH 42/94] =?UTF-8?q?fix=20bugs=20=09#3061=20=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E5=90=AF=E6=99=BAnpu=E8=AE=AD=E7=BB=83=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E9=A1=B5=E9=9D=A2=E7=9A=84=E5=9F=BA=E4=BA=8E=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E6=98=BE=E7=A4=BA=E7=A9=BA=20=09#3057=20=E3=80=90?= =?UTF-8?q?=E9=80=9A=E7=9F=A5=E3=80=91=E5=B7=B2=E8=AF=BB=E6=B6=88=E6=81=AF?= =?UTF-8?q?tab=E9=A1=B5=E7=BF=BB=E9=A1=B5=E5=8A=9F=E8=83=BD=E4=B8=8D?= =?UTF-8?q?=E5=8F=AF=E7=94=A8=20=09#3058=20=E3=80=90=E5=9B=BD=E9=99=85?= =?UTF-8?q?=E5=8C=96=E3=80=91=E8=8B=B1=E6=96=87=E7=95=8C=E9=9D=A2=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E4=BD=93=E9=AA=8C=E8=8F=9C=E5=8D=95=E4=BB=8D=E7=84=B6?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E7=9A=84=E4=B8=AD=E6=96=87=20=09#1945=20?= =?UTF-8?q?=E3=80=90=E4=B8=AA=E4=BA=BA=E4=B8=AD=E5=BF=83=E3=80=91=E5=A4=B4?= =?UTF-8?q?=E5=83=8F=E5=88=87=E5=88=B0=E5=85=B6=E5=AE=83=E7=BB=84=E7=BB=87?= =?UTF-8?q?=E5=86=8D=E5=88=87=E5=9B=9E=E8=87=AA=E5=B7=B1=E8=B7=B3=E8=BD=AC?= =?UTF-8?q?=E5=88=B0=E9=A6=96=E9=A1=B5=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/user/notification.go | 4 ++-- templates/base/head_navbar.tmpl | 4 ++-- templates/base/head_navbar_fluid.tmpl | 4 ++-- templates/base/head_navbar_home.tmpl | 4 ++-- templates/base/head_navbar_pro.tmpl | 4 ++-- templates/user/dashboard/navbar.tmpl | 2 +- 8 files changed, 13 insertions(+), 11 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e7d730027..3013fcdaa 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1036,6 +1036,7 @@ cloudbrain.time.starttime=Start run time cloudbrain.time.endtime=End run time cloudbrain.datasetdownload=Dataset download url model_manager = Model +model_experience = Model Experience model_noright=You have no right to do the operation. model_rename=Duplicate model name, please modify model name. diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 972865484..e4abf5f11 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1036,6 +1036,7 @@ datasets.desc=数据集功能 cloudbrain_helper=使用GPU/NPU资源，开启Notebook、模型训练任务等 model_manager = 模型 +model_experience = 模型体验 model_noright=您没有操作权限。 model_rename=模型名称重复，请修改模型名称 diff --git a/routers/user/notification.go b/routers/user/notification.go index 0fd6df007..d66e893a4 100644 --- a/routers/user/notification.go +++ b/routers/user/notification.go @@ -132,11 +132,11 @@ func getNotifications(c *context.Context) { } c.Data["Title"] = c.Tr("notifications") - //c.Data["Keyword"] = keyword + c.Data["Type"] = keyword c.Data["Status"] = status c.Data["Notifications"] = notifications - pager.SetDefaultParams(c) + pager.AddParam(c, "q", "Type") c.Data["Page"] = pager } diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index f9908f6f0..50521516f 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -38,7 +38,7 @@ {{.i18n.Tr "repo.model_manager"}}
- 模型体验 + {{.i18n.Tr "repo.model_experience"}}
diff --git a/templates/base/head_navbar_fluid.tmpl b/templates/base/head_navbar_fluid.tmpl index 010568e8d..b2fd60787 100644 --- a/templates/base/head_navbar_fluid.tmpl +++ b/templates/base/head_navbar_fluid.tmpl @@ -38,7 +38,7 @@ {{.i18n.Tr "repo.model_manager"}}
- 模型体验 + {{.i18n.Tr "repo.model_experience"}}
- + {{.SignedUser.Name}} From 041b350bc2074bf4650899c0672fe2afe5c2b6ce Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 2 Nov 2022 14:48:40 +0800 Subject: [PATCH 43/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 485d26090..68902ec68 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -724,12 +724,12 @@ func RegisterRoutes(m *macaron.Macaron) { m.Combo("/repositories/:id", reqToken()).Get(repo.GetByID) - m.Group("/:username/:reponame/datasets", func() { + m.Group("/datasets/:username/:reponame", func() { m.Get("/current_repo", repo.CurrentRepoDatasetMultiple) m.Get("/my_datasets", repo.MyDatasetsMultiple) m.Get("/public_datasets", repo.PublicDatasetMultiple) m.Get("/my_favorite", repo.MyFavoriteDatasetMultiple) - }, reqToken()) + }, reqToken(), repoAssignment()) m.Group("/repos", func() { m.Get("/search", repo.Search) From 6834f3e065019dddaad97f9694859483459d8f91 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 2 Nov 2022 17:26:40 +0800 Subject: [PATCH 44/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/train.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 5e2cce1ff..c3eb256f2 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/urfs_client/urchin" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/notification" @@ -382,7 +384,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, option api.CreateTrainJobOpt //prepare command preTrainModelPath := getPreTrainModelPath(option.PreTrainModelUrl, option.CkptName) - command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", allFileName, preTrainModelPath, option.CkptName) + command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", allFileName, preTrainModelPath, option.CkptName, "") if err != nil { log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Create task failed, internal error")) @@ -611,7 +613,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, option api.CreateTrainJobOpt //prepare command preTrainModelPath := getPreTrainModelPath(option.PreTrainModelUrl, option.CkptName) - command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, allFileName, preTrainModelPath, option.CkptName) + command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, allFileName, preTrainModelPath, option.CkptName, grampus.GetNpuModelRemoteObsUrl(jobName)) if err != nil { log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) @@ -862,15 +864,18 @@ func getPreTrainModelPath(pretrainModelDir string, fileName string) string { } -func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, pretrainModelPath, pretrainModelFileName string) (string, error) { +func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, pretrainModelPath, pretrainModelFileName, modelRemoteObsUrl string) (string, error) { var command string + //prepare workDir := grampus.NpuWorkDir - if processorType == grampus.ProcessorTypeGPU { + if processorType == grampus.ProcessorTypeNPU { + command += "pwd;cd " + workDir + grampus.CommandPrepareScriptNpu + } else if processorType == grampus.ProcessorTypeGPU { workDir = grampus.GpuWorkDir + command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScriptGpu, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) } - command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { //no need to download code & dataset by internet @@ -885,7 +890,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //no need to process } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand + commandUnzip := "cd " + workDir + "code;unzip -q master.zip;rm -f master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand command += commandUnzip } @@ -919,7 +924,8 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo var commandCode string if processorType == grampus.ProcessorTypeNPU { - commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py /tmp/log/train.log" + paramCode + ";" + paramCode += " --model_url=" + modelRemoteObsUrl + commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py " + grampus.NpuLocalLogUrl + paramCode + ";" } else if processorType == grampus.ProcessorTypeGPU { if pretrainModelFileName != "" { paramCode += " --ckpt_url" + "=" + workDir + "pretrainmodel/" + pretrainModelFileName @@ -935,8 +941,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //upload models if processorType == grampus.ProcessorTypeNPU { - commandUpload := "cd " + workDir + setting.Grampus.SyncScriptProject + "/;./uploader_for_npu " + setting.Bucket + " " + outputRemotePath + " " + workDir + "output/;" - command += commandUpload + // no need to upload } else if processorType == grampus.ProcessorTypeGPU { commandUpload := "cd " + workDir + setting.Grampus.SyncScriptProject + "/;./uploader_for_gpu " + setting.Grampus.Env + " " + outputRemotePath + " " + workDir + "output/;" command += commandUpload @@ -948,7 +953,6 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo return command, nil } - func processPretrainModelParameter(pretrainModelPath string, pretrainModelFileName string, commandDownload string) string { commandDownloadTemp := commandDownload if pretrainModelPath != "" { @@ -1083,6 +1087,11 @@ func SyncTaskStatus(task *models.Cloudbrain) error { task.CorrectCreateUnix() if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) + if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource { + if len(result.JobInfo.Tasks[0].CenterID) == 1 { + urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID)) + } + } } err = models.UpdateJob(task) if err != nil { From 8842ba0349fa643ce5cf3aa96cf0ded2e05a16e1 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 10:11:15 +0800 Subject: [PATCH 45/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/structs/cloudbrain.go | 2 +- routers/api/v1/api.go | 4 +-- routers/repo/grampus.go | 49 +---------------------------- services/cloudbrain/cloudbrainTask/train.go | 9 ++++-- 4 files changed, 10 insertions(+), 54 deletions(-) diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index b29d7e722..d12d05a21 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -32,7 +32,7 @@ type CreateTrainJobOption struct { BootFile string `json:"boot_file" binding:"Required"` BranchName string `json:"branch_name" binding:"Required"` Params string `json:"run_para_list" binding:"Required"` - WorkServerNumber int `json:"work_server_number" binding:"Required"` + WorkServerNumber int `json:"work_server_number"` ModelName string `json:"model_name"` ModelVersion string `json:"model_version"` CkptName string `json:"ckpt_name"` diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 2f5f059b6..d6a8adc44 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1011,11 +1011,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/train-job", func() { m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) - m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat, bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) }) m.Group("/inference-job", func() { - m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat, bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) }) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 35f9a11bf..0620350f6 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -509,28 +509,6 @@ func GrampusTrainJobVersionCreate(ctx *context.Context, form auth.CreateGrampusT } -func checkSpecialPool(ctx *context.Context, resourceType string) string { - grampus.InitSpecialPool() - if grampus.SpecialPools != nil { - for _, pool := range grampus.SpecialPools.Pools { - - if pool.IsExclusive && pool.Type == resourceType { - - org, _ := models.GetOrgByName(pool.Org) - if org != nil { - isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID) - if !isOrgMember { - return ctx.Tr("repo.grampus.no_operate_right") - } - } - } - - } - - } - return "" -} - func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) { ctx.Data["IsCreate"] = true grampusTrainJobNpuCreate(ctx, form) @@ -1003,7 +981,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo if processorType == grampus.ProcessorTypeNPU { //no need to process } else if processorType == grampus.ProcessorTypeGPU { - unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) + unZipDatasetCommand := cloudbrainTask.GenerateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;rm -f master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand command += commandUnzip } @@ -1077,31 +1055,6 @@ func processPretrainModelParameter(pretrainModelPath string, pretrainModelFileNa return commandDownloadTemp } -func generateDatasetUnzipCommand(datasetName string) string { - var unZipDatasetCommand string - - datasetNameArray := strings.Split(datasetName, ";") - if len(datasetNameArray) == 1 { //单数据集 - unZipDatasetCommand = "unzip -q '" + datasetName + "';" - if strings.HasSuffix(datasetNameArray[0], ".tar.gz") { - unZipDatasetCommand = "tar --strip-components=1 -zxvf '" + datasetName + "';" - } - unZipDatasetCommand += "rm -f '" + datasetName + "';" - - } else { //多数据集 - for _, datasetNameTemp := range datasetNameArray { - if strings.HasSuffix(datasetNameTemp, ".tar.gz") { - unZipDatasetCommand = unZipDatasetCommand + "tar -zxvf '" + datasetNameTemp + "';" - } else { - unZipDatasetCommand = unZipDatasetCommand + "unzip -q '" + datasetNameTemp + "' -d './" + strings.TrimSuffix(datasetNameTemp, ".zip") + "';" - } - unZipDatasetCommand += "rm -f '" + datasetNameTemp + "';" - } - - } - return unZipDatasetCommand -} - func downloadZipCode(ctx *context.Context, codePath, branchName string) error { archiveType := git.ZIP archivePath := codePath diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index c3eb256f2..b59cbc804 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -889,7 +889,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo if processorType == grampus.ProcessorTypeNPU { //no need to process } else if processorType == grampus.ProcessorTypeGPU { - unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) + unZipDatasetCommand := GenerateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;rm -f master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand command += commandUnzip } @@ -962,7 +962,7 @@ func processPretrainModelParameter(pretrainModelPath string, pretrainModelFileNa return commandDownloadTemp } -func generateDatasetUnzipCommand(datasetName string) string { +func GenerateDatasetUnzipCommand(datasetName string) string { var unZipDatasetCommand string datasetNameArray := strings.Split(datasetName, ";") @@ -971,6 +971,7 @@ func generateDatasetUnzipCommand(datasetName string) string { if strings.HasSuffix(datasetNameArray[0], ".tar.gz") { unZipDatasetCommand = "tar --strip-components=1 -zxvf '" + datasetName + "';" } + unZipDatasetCommand += "rm -f '" + datasetName + "';" } else { //多数据集 for _, datasetNameTemp := range datasetNameArray { @@ -979,6 +980,7 @@ func generateDatasetUnzipCommand(datasetName string) string { } else { unZipDatasetCommand = unZipDatasetCommand + "unzip -q '" + datasetNameTemp + "' -d './" + strings.TrimSuffix(datasetNameTemp, ".zip") + "';" } + unZipDatasetCommand += "rm -f '" + datasetNameTemp + "';" } } @@ -1071,7 +1073,8 @@ func SyncTaskStatus(task *models.Cloudbrain) error { } oldStatus := task.Status task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) - if task.Status != result.JobInfo.Status || result.JobInfo.Status == models.GrampusStatusRunning { + + if task.Status != oldStatus || task.Status == models.GrampusStatusRunning { task.Duration = result.JobInfo.RunSec if task.Duration < 0 { task.Duration = 0 From c34021b030ad2f2d1de6b6942f0c8905bcbee1bc Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 10:46:28 +0800 Subject: [PATCH 46/94] =?UTF-8?q?=E8=A1=A5=E5=85=85gitRepo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index d6a8adc44..c5325c5ab 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1011,11 +1011,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/train-job", func() { m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) - m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), context.ReferencesGitRepo(false), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) }) m.Group("/inference-job", func() { - m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), context.ReferencesGitRepo(false), repo.CreateCloudBrainInferenceTask) m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) }) From 28889f60b321923afde2f5cf23ed286bd67269ba Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 11:04:15 +0800 Subject: [PATCH 47/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4diam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/api.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c5325c5ab..92ec10cdd 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -977,15 +977,23 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/:id/log", repo.CloudbrainGetLog) m.Get("/:id/download_log_file", repo.CloudbrainDownloadLogFile) m.Group("/train-job", func() { + + m.Post("/create", reqToken(), reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), context.ReferencesGitRepo(false), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) + m.Group("/:jobid", func() { m.Get("", repo.GetModelArtsTrainJobVersion) + m.Get("/detail", reqToken(), reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) m.Get("/model_list", repo.CloudBrainModelList) m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.CloudBrainStop) }) }) m.Group("/inference-job", func() { + m.Post("/create", reqToken(), reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), context.ReferencesGitRepo(false), repo.CreateCloudBrainInferenceTask) + m.Group("/:jobid", func() { m.Get("", repo.GetCloudBrainInferenceJob) + m.Get("/detail", reqToken(), reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.DelCloudBrainJob) m.Get("/result_list", repo.InferencJobResultList) }) @@ -1007,19 +1015,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/:id/modelartlog", repo.TrainJobForModelConvertGetLog) m.Get("/:id/model_list", repo.CloudBrainModelConvertList) }, reqRepoReader(models.UnitTypeModelManage)) - m.Group("/cloudbrain", func() { - m.Group("/train-job", func() { - m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) - - m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), context.ReferencesGitRepo(false), bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) - - }) - m.Group("/inference-job", func() { - m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat(), bind(api.CreateTrainJobOption{}), context.ReferencesGitRepo(false), repo.CreateCloudBrainInferenceTask) - m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) - }) - - }, reqToken()) m.Group("/modelarts", func() { m.Group("/notebook", func() { //m.Get("/:jobid", repo.GetModelArtsNotebook) From 636c6909d39d3cd18d8bd31a6254b3edd384eb23 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 11:12:24 +0800 Subject: [PATCH 48/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/repo/cloudbrain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 9c99f6e48..c933c97d3 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -16,6 +16,8 @@ import ( "strings" "time" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" @@ -72,6 +74,7 @@ func CloudBrainShow(ctx *context.APIContext) { } else if task.Type == models.TypeC2Net { task.Image = task.EngineName } + task.AiCenter = cloudbrainService.GetAiCenterShow(task.AiCenter, ctx.Context) ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)}) From beeb8f06304a6c2c30d58eb75591f0dc6fb3045f Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 12:42:51 +0800 Subject: [PATCH 49/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/count.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 34b334445..985706911 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -71,7 +71,7 @@ func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, c } key := jobNewType + "-" + strconv.Itoa(cloudbrainType) - if len(computeResource) > 0 { + if len(computeResource) > 0 && cloudbrainType == models.TypeC2Net { key = key + "-" + computeResource[0] } From 3ebd9d325ac125e32a69bcae470d7da66c5b9689 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 14:29:54 +0800 Subject: [PATCH 50/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/structs/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index d12d05a21..866c85dad 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -22,7 +22,7 @@ type CreateGrampusTrainJobOption struct { } type CreateTrainJobOption struct { - Type int `json:"type" binding:"Required"` + Type int `json:"type"` DisplayJobName string `json:"display_job_name" binding:"Required"` ImageID string `json:"image_id"` Image string `json:"image" binding:"Required"` From bd96fa5515c0799d84f788f7751400896aaaddd5 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 15:02:50 +0800 Subject: [PATCH 51/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/train.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index b59cbc804..8e4673d66 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -530,7 +530,12 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock var datasetInfos map[string]models.DatasetInfo var datasetNames string if option.Type != TaskTypeModelArts { - datasetInfos, datasetNames, err = models.GetDatasetInfo(option.Attachment, computeType) + if isC2NetTask(option) { + datasetInfos, datasetNames, err = models.GetDatasetInfo(option.Attachment, computeType) + } else { + datasetInfos, datasetNames, err = models.GetDatasetInfo(option.Attachment) + } + if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) return nil, nil, "", fmt.Errorf(ctx.Tr("cloudbrain.error.dataset_select")) From 306cf75a8e40941c597e58f428ba4592eebb02c1 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 3 Nov 2022 15:15:46 +0800 Subject: [PATCH 52/94] #3045 update --- models/resource_queue.go | 3 --- models/resource_specification.go | 11 ++++------- services/cloudbrain/resource/resource_specification.go | 1 + 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/models/resource_queue.go b/models/resource_queue.go index fc0dd8cb5..87809bba7 100644 --- a/models/resource_queue.go +++ b/models/resource_queue.go @@ -313,9 +313,6 @@ func SyncGrampusQueues(updateList []ResourceQueue, insertList []ResourceQueue, e if _, err = sess.In("id", deleteSpcIds).Update(&ResourceSpecification{Status: SpecOffShelf}); err != nil { return err } - if _, err = sess.In("spec_id", deleteSpcIds).Delete(&ResourceSceneSpec{}); err != nil { - return err - } } } diff --git a/models/resource_specification.go b/models/resource_specification.go index 2da8d015d..a6f5f1f82 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -168,6 +168,7 @@ type FindSpecsOptions struct { UseShareMemGiB bool //if true,find specs no matter used or not used in scene. if false,only find specs used in scene RequestAll bool + SpecStatus int } type Specification struct { @@ -269,10 +270,6 @@ func ResourceSpecOffShelf(id int64) (int64, error) { } sess.Close() }() - //delete scene spec relation - if _, err = sess.Where("spec_id = ?", id).Delete(&ResourceSceneSpec{}); err != nil { - return 0, err - } param := ResourceSpecification{ Status: SpecOffShelf, @@ -317,9 +314,6 @@ func SyncGrampusSpecs(updateList []ResourceSpecification, insertList []ResourceS if _, err = sess.Cols("status", "is_available").In("id", deleteIds).Update(&ResourceSpecification{Status: SpecOffShelf, IsAvailable: false}); err != nil { return err } - if _, err = sess.In("spec_id", deleteIds).Delete(&ResourceSceneSpec{}); err != nil { - return err - } } //update exists specs @@ -384,6 +378,9 @@ func FindSpecs(opts FindSpecsOptions) ([]*Specification, error) { if opts.UseShareMemGiB { cond = cond.And(builder.Eq{"resource_specification.share_mem_gi_b": opts.ShareMemGiB}) } + if opts.SpecStatus > 0 { + cond = cond.And(builder.Eq{"resource_specification.status": opts.SpecStatus}) + } r := make([]*Specification, 0) s := x.Where(cond). Join("INNER", "resource_queue", "resource_queue.id = resource_specification.queue_id") diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index b68abbb88..9388a1efb 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -197,6 +197,7 @@ func AddSpecOperateLog(doerId int64, operateType string, newValue, oldValue *mod } func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.Specification, error) { + opts.SpecStatus = models.SpecOnShelf r, err := models.FindSpecs(opts) if err != nil { log.Error("FindAvailableSpecs error.%v", err) From 54ff3a22dd80b5648bc61e1f9f47c413491360ee Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 15:46:58 +0800 Subject: [PATCH 53/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 92ec10cdd..813b77c8c 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -749,6 +749,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken()) m.Group("/:username/:reponame", func() { + m.Get("/right", reqToken(), repo.GetRight) + m.Get("/tagger", reqToken(), repo.ListTagger) m.Combo("").Get(reqAnyRepoReader(), repo.Get). Delete(reqToken(), reqOwner(), repo.Delete). Patch(reqToken(), reqAdmin(), bind(api.EditRepoOption{}), context.RepoRef(), repo.Edit) From 974701ce3162c3154e3e535dee96835ada6e3796 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 3 Nov 2022 16:32:16 +0800 Subject: [PATCH 54/94] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/repo/mlops.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/routers/api/v1/repo/mlops.go b/routers/api/v1/repo/mlops.go index 3392ebfec..43969330d 100644 --- a/routers/api/v1/repo/mlops.go +++ b/routers/api/v1/repo/mlops.go @@ -36,12 +36,20 @@ func ListTagger(ctx *context.APIContext) { } for _, team := range teams { + team.GetMembers(&models.SearchMembersOptions{}) for _, user := range team.Members { if _, ok := userRemember[user.Name]; !ok { taggers = append(taggers, convert.ToTagger(user)) + userRemember[user.Name] = "" } } } + if !ctx.Repo.Owner.IsOrganization() { + if _, ok := userRemember[ctx.Repo.Owner.Name]; !ok { + taggers = append(taggers, convert.ToTagger(ctx.Repo.Owner)) + + } + } ctx.JSON(http.StatusOK, taggers) } From 44e19cd8ef7b0ab0b3ae144abdb0fbf64e224e35 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Fri, 4 Nov 2022 09:41:14 +0800 Subject: [PATCH 55/94] =?UTF-8?q?#3045=20=E7=AE=97=E5=8A=9B=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E7=AE=A1=E7=90=86=E6=94=AF=E6=8C=81=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E7=9F=AD=E6=9C=9F=E5=9B=9E=E6=94=B6=E7=9A=84=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web_src/vuepages/pages/resources/components/SceneDialog.vue | 10 ++++++---- web_src/vuepages/pages/resources/scene/index.vue | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/web_src/vuepages/pages/resources/components/SceneDialog.vue b/web_src/vuepages/pages/resources/components/SceneDialog.vue index 3b77be6e2..cadfb1db1 100644 --- a/web_src/vuepages/pages/resources/components/SceneDialog.vue +++ b/web_src/vuepages/pages/resources/components/SceneDialog.vue @@ -69,7 +69,9 @@
- + + +
@@ -167,7 +169,7 @@ export default { const params = { cluster: this.dataInfo.Cluster, queue: this.dataInfo.QueueId === '-1' ? '' : this.dataInfo.QueueId, - status: 2, + // status: 2, page: 1, }; return getResSpecificationList(params).then(res => { @@ -177,11 +179,11 @@ export default { const data = list.map((item) => { const Queue = item.Queue; const Spec = item.Spec; - // const NGPU = `${Queue.ComputeResource}:${Spec.AccCardsNum === 0 ? '0' : Spec.AccCardsNum + '*' + getListValueWithKey(this.accCardTypeList, Queue.AccCardType)}`; const NGPU = `${Queue.ComputeResource}:${Spec.AccCardsNum + '*' + getListValueWithKey(this.accCardTypeList, Queue.AccCardType)}`; + const statusStr = Spec.Status != '2' ? `(${getListValueWithKey(this.statusList, Spec.Status.toString())})` : ''; return { k: Spec.ID, - v: `${NGPU}, CPU:${Spec.CpuCores}, ${this.$t('resourcesManagement.gpuMem')}:${Spec.GPUMemGiB}GB, ${this.$t('resourcesManagement.mem')}:${Spec.MemGiB}GB, ${this.$t('resourcesManagement.shareMem')}:${Spec.ShareMemGiB}GB, ${this.$t('resourcesManagement.unitPrice')}:${Spec.UnitPrice}${this.$t('resourcesManagement.point_hr')}`, + v: `${NGPU}, CPU:${Spec.CpuCores}, ${this.$t('resourcesManagement.gpuMem')}:${Spec.GPUMemGiB}GB, ${this.$t('resourcesManagement.mem')}:${Spec.MemGiB}GB, ${this.$t('resourcesManagement.shareMem')}:${Spec.ShareMemGiB}GB, ${this.$t('resourcesManagement.unitPrice')}:${Spec.UnitPrice}${this.$t('resourcesManagement.point_hr')}${statusStr}`, } }); this.specsList.splice(0, Infinity, ...data); diff --git a/web_src/vuepages/pages/resources/scene/index.vue b/web_src/vuepages/pages/resources/scene/index.vue index 30c53b742..a706b358b 100644 --- a/web_src/vuepages/pages/resources/scene/index.vue +++ b/web_src/vuepages/pages/resources/scene/index.vue @@ -64,7 +64,7 @@ header-align="center" min-width="180"> @@ -100,7 +100,7 @@