From 8ccb01533b944e8de880a243ab7dd166a62bbb61 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 17 Nov 2021 17:57:50 +0800 Subject: [PATCH 1/2] update --- models/cloudbrain.go | 58 ++++++++++++++------------------------ routers/api/v1/api.go | 2 -- routers/api/v1/repo/modelarts.go | 60 ---------------------------------------- routers/repo/modelarts.go | 55 ------------------------------------ 4 files changed, 20 insertions(+), 155 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 74689976e..0f9cde079 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -30,7 +30,6 @@ const ( JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" - JobVersionName JobType = "V0001" ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 @@ -71,28 +70,28 @@ type Cloudbrain struct { VersionID int64 `xorm:"INDEX DEFAULT 0"` VersionName string `xorm:"INDEX"` - Uuid string + Uuid string //数据集id DatasetName string - VersionCount int `xorm:"INDEX DEFAULT 1"` - IsLatestVersion string - CommitID string - FatherVersionName string - ComputeResource string - EngineID int64 - - TrainUrl string - BranchName string - Parameters string - BootFile string - DataUrl string - LogUrl string - PreVersionId int64 - FlavorCode string + VersionCount int `xorm:"INDEX DEFAULT 1"` //任务的当前版本数量，不包括删除的 + IsLatestVersion string //是否是最新版本，1是，0否 + CommitID string //提交的仓库代码id + FatherVersionName string //父版本名称 + ComputeResource string //计算资源，例如npu + EngineID int64 //引擎id + + TrainUrl string //输出的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id Description string - WorkServerNumber int - FlavorName string - EngineName string - TotalVersionCount int + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 User *User `xorm:"-"` Repo *Repository `xorm:"-"` @@ -1159,23 +1158,6 @@ func deleteJobVersion(e Engine, job *Cloudbrain) error { return err } -// func DeleteJobVersion(job *Cloudbrain, jobID string, versionName string) error { -// return deleteJobVersion(x, job, jobID, versionName) -// } - -// func deleteJobVersion(e Engine, job *Cloudbrain, jobID string, versionName string) error { -// var sess *xorm.Session -// sess = e.Where("job_id = ? AND version_name !=?", jobID, versionName) -// _, err := sess.Delete(job) -// return err -// } - -// func deleteJobVersion(e Engine, jobID string, versionName string) error { -// deleteCloudbrainSql := "delete from cloudbrain where job_id=" + jobID + "and version_name=" + versionName -// _, err := e.Exec(deleteCloudbrainSql) -// return err -// } - func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobName: jobName} return getRepoCloudBrain(cb) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index d66aa698d..dde17c059 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -880,8 +880,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/log", repo.TrainJobGetLog) m.Post("/del_version", repo.DelTrainJobVersion) m.Post("/stop_version", repo.StopTrainJobVersion) - m.Get("/model_list", repo.ModelList) - m.Get("/model_download", repo.ModelDownload) // m.Group("/:version-name", func() { // m.Get("", repo.GetModelArtsTrainJobVersion) // }) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index c5bc625d6..aea529da2 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -8,14 +8,12 @@ package repo import ( "net/http" "strconv" - "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" ) func GetModelArtsNotebook(ctx *context.APIContext) { @@ -300,61 +298,3 @@ func StopTrainJobVersion(ctx *context.APIContext) { "StatusOK": 0, }) } - -func ModelList(ctx *context.APIContext) { - var ( - err error - ) - - var jobID = ctx.Params(":jobid") - var versionName = ctx.Query("version_name") - parentDir := ctx.Query("parentDir") - dirArray := strings.Split(parentDir, "/") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - return - } - VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) - models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath) - if err != nil { - log.Info("get TrainJobListModel failed:", err) - ctx.ServerError("GetObsListObject:", err) - return - } - - ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "VersionName": versionName, - "StatusOK": 0, - "Path": dirArray, - "Dirs": models, - // "task": task, - "PageIsCloudBrain": true, - }) -} - -func ModelDownload(ctx *context.APIContext) { - var ( - err error - ) - - var jobID = ctx.Params(":jobid") - versionName := ctx.Query("version_name") - parentDir := ctx.Query("parent_dir") - fileName := ctx.Query("file_name") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - return - } - VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) - - url, err := storage.GetObsCreateVersionSignedUrl(task.JobName, parentDir, fileName, VersionOutputPath) - if err != nil { - log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrl", err) - return - } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) -} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index e8cac4568..d69f7a1b2 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1008,48 +1008,6 @@ func TrainJobShow(ctx *context.Context) { return } - // attach, err := models.GetAttachmentByUUID(task.Uuid) - // if err != nil { - // log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error()) - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - // return - // } - - // result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) - // if err != nil { - // log.Error("GetJob(%s) failed:%v", jobID, err.Error()) - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - // return - // } - - // if result != nil { - // result.CreateTime = time.Unix(int64(result.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05") - // if result.Duration != 0 { - // result.TrainJobDuration = addZero(result.Duration/3600000) + ":" + addZero(result.Duration%3600000/60000) + ":" + addZero(result.Duration%60000/1000) - - // } else { - // result.TrainJobDuration = "00:00:00" - // } - // result.Status = modelarts.TransTrainJobStatus(result.IntStatus) - // err = models.SetTrainJobStatusByJobID(jobID, result.Status, result.Duration, string(result.TrainJobDuration)) - // if err != nil { - // ctx.ServerError("UpdateJob failed", err) - // return - // } - - // result.DatasetName = task.DatasetName - // } - - // resultLogFile, resultLog, err := trainJobGetLog(jobID) - // if err != nil { - // log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - // return - // } - - // ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] - // ctx.Data["log"] = resultLog - // ctx.Data["task"] = task ctx.Data["jobID"] = jobID // ctx.Data["result"] = result ctx.Data["version_list_task"] = VersionListTasks @@ -1156,19 +1114,6 @@ func TrainJobDel(ctx *context.Context) { return } - // task, err := models.GetCloudbrainByJobID(jobID) - // if err != nil { - // log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) - // return - // } - - // err = models.DeleteJob(task) - // if err != nil { - // ctx.ServerError("DeleteJob failed", err) - // return - // } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") } From c4441fed5f4df3f5c830d6a51c3a923618654604 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 17 Nov 2021 18:23:15 +0800 Subject: [PATCH 2/2] update --- routers/api/v1/api.go | 2 ++ routers/api/v1/repo/modelarts.go | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index dde17c059..d66aa698d 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -880,6 +880,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/log", repo.TrainJobGetLog) m.Post("/del_version", repo.DelTrainJobVersion) m.Post("/stop_version", repo.StopTrainJobVersion) + m.Get("/model_list", repo.ModelList) + m.Get("/model_download", repo.ModelDownload) // m.Group("/:version-name", func() { // m.Get("", repo.GetModelArtsTrainJobVersion) // }) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index aea529da2..ba4015d3b 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -8,12 +8,14 @@ package repo import ( "net/http" "strconv" + "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" ) func GetModelArtsNotebook(ctx *context.APIContext) { @@ -298,3 +300,61 @@ func StopTrainJobVersion(ctx *context.APIContext) { "StatusOK": 0, }) } + +func ModelList(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) + models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + "Path": dirArray, + "Dirs": models, + "task": task, + "PageIsCloudBrain": true, + }) +} + +func ModelDownload(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + versionName := ctx.Query("version_name") + parentDir := ctx.Query("parent_dir") + fileName := ctx.Query("file_name") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) + + url, err := storage.GetObsCreateVersionSignedUrl(task.JobName, parentDir, fileName, VersionOutputPath) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +}