From 001c924b62c2daa67f96b3464849736867445917 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 29 Oct 2021 08:55:54 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E4=BB=A3=E7=A0=81bug=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/repo_statistic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 4d23e79f2..9bc70eaff 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -23,7 +23,7 @@ func RepoStatisticAuto() { func RepoStatisticDaily(date string) { log.Info("%s", date) log.Info("begin Repo Statistic") - t, _ := time.Parse("2006-01-02", "date") + t, _ := time.Parse("2006-01-02", date) if err := models.DeleteRepoStatDaily(date); err != nil { log.Error("DeleteRepoStatDaily failed: %v", err.Error()) return From 236e43983f3c7b1904a69345005b64be4010faa2 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Fri, 29 Oct 2021 10:47:28 +0800 Subject: [PATCH 2/7] opt --- models/cloudbrain.go | 4 +-- modules/modelarts/modelarts.go | 6 ++-- routers/api/v1/repo/modelarts.go | 2 ++ routers/repo/modelarts.go | 45 ++++------------------------ templates/repo/modelarts/index.tmpl | 2 +- templates/repo/modelarts/notebook/index.tmpl | 4 +-- templates/repo/modelarts/trainjob/index.tmpl | 6 ++-- 7 files changed, 19 insertions(+), 50 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index d83e38d32..be14c633e 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -60,7 +60,7 @@ type Cloudbrain struct { ContainerIp string CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Duration int `xorm:"INDEX duration"` + Duration int64 `xorm:"INDEX duration"` TrainJobDuration string DeletedAt time.Time `xorm:"deleted"` CanDebug bool `xorm:"-"` @@ -933,7 +933,7 @@ func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err erro return } -func SetTrainJobStatusByJobID(jobID string, status string, duration int, trainjobduration string) (err error) { +func SetTrainJobStatusByJobID(jobID string, status string, duration int64, trainjobduration string) (err error) { cb := &Cloudbrain{JobID: jobID, Status: string(status), Duration: duration, TrainJobDuration: trainjobduration} _, err = x.Cols("status", "duration", "train_job_duration").Where("cloudbrain.job_id=?", jobID).Update(cb) return diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 63baa910f..a0d068b56 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -221,7 +221,7 @@ func TransTrainJobStatus(status int) string { case 0: return "UNKNOWN" case 1: - return "CREATING" + return "INIT" case 2: return "IMAGE_CREATING" case 3: @@ -237,13 +237,13 @@ func TransTrainJobStatus(status int) string { case 8: return "RUNNING" case 9: - return "STOPPED" + return "KILLING" case 10: return "COMPLETED" case 11: return "FAILED" case 12: - return "STOPPED" + return "KILLED" case 13: return "CANCELED" case 14: diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 1ec4fa919..f38cb1ad7 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -64,6 +64,8 @@ func GetModelArtsTrainJob(ctx *context.APIContext) { } job.Status = modelarts.TransTrainJobStatus(result.IntStatus) + job.Duration = result.Duration + job.TrainJobDuration = result.TrainJobDuration err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 3203bc872..396457077 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -506,43 +506,7 @@ func TrainJobIndex(ctx *context.Context) { page = 1 } - tasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - RepoID: repo.ID, - Type: models.TypeCloudBrainTrainJob, - }) - if err != nil { - ctx.ServerError("Cloudbrain", err) - return - } - - for i := range tasks { - TrainJobDetail, err := modelarts.GetTrainJob(tasks[i].Cloudbrain.JobID, strconv.FormatInt(tasks[i].Cloudbrain.VersionID, 10)) - if TrainJobDetail != nil { - TrainJobDetail.CreateTime = time.Unix(int64(TrainJobDetail.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05") - if TrainJobDetail.Duration != 0 { - TrainJobDetail.TrainJobDuration = addZero(TrainJobDetail.Duration/3600000) + ":" + addZero(TrainJobDetail.Duration%3600000/60000) + ":" + addZero(TrainJobDetail.Duration%60000/1000) - - } else { - TrainJobDetail.TrainJobDuration = "00:00:00" - } - } - if err != nil { - log.Error("GetJob(%s) failed:%v", tasks[i].Cloudbrain.JobID, err.Error()) - return - } - err = models.SetTrainJobStatusByJobID(tasks[i].Cloudbrain.JobID, modelarts.TransTrainJobStatus(TrainJobDetail.IntStatus), int(TrainJobDetail.Duration), string(TrainJobDetail.TrainJobDuration)) - // err = models.UpdateJob(tasks[i].Cloudbrain) - if err != nil { - ctx.ServerError("UpdateJob failed", err) - return - } - } - - trainTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, @@ -560,7 +524,7 @@ func TrainJobIndex(ctx *context.Context) { ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true - ctx.Data["Tasks"] = trainTasks + ctx.Data["Tasks"] = tasks ctx.HTML(200, tplModelArtsTrainJobIndex) } @@ -901,12 +865,13 @@ func TrainJobShow(ctx *context.Context) { } else { result.TrainJobDuration = "00:00:00" } - err = models.SetTrainJobStatusByJobID(jobID, modelarts.TransTrainJobStatus(result.IntStatus), int(result.Duration), string(result.TrainJobDuration)) + result.Status = modelarts.TransTrainJobStatus(result.IntStatus) + err = models.SetTrainJobStatusByJobID(jobID, result.Status, result.Duration, string(result.TrainJobDuration)) if err != nil { ctx.ServerError("UpdateJob failed", err) return } - result.Status = modelarts.TransTrainJobStatus(result.IntStatus) + result.DatasetName = attach.Name } diff --git a/templates/repo/modelarts/index.tmpl b/templates/repo/modelarts/index.tmpl index bcfef05da..5ceccdd9a 100755 --- a/templates/repo/modelarts/index.tmpl +++ b/templates/repo/modelarts/index.tmpl @@ -415,7 +415,7 @@ $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; const repoPath = job.dataset.repopath; - if (job.textContent.trim() == 'STOPPED') { + if (job.textContent.trim() == 'STOPPED' || job.textContent.trim() == 'START_FAILED' || job.textContent.trim() == 'CREATE_FAILED') { return } diff --git a/templates/repo/modelarts/notebook/index.tmpl b/templates/repo/modelarts/notebook/index.tmpl index 2a4115723..841954214 100755 --- a/templates/repo/modelarts/notebook/index.tmpl +++ b/templates/repo/modelarts/notebook/index.tmpl @@ -423,12 +423,12 @@ // 加载任务状态 var timeid = window.setInterval(loadJobStatus, 15000); - // $(document).ready(loadJobStatus); + $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; const repoPath = job.dataset.repopath; - if (job.textContent.trim() == 'STOPPED') { + if (job.textContent.trim() == 'STOPPED' || job.textContent.trim() == 'START_FAILED' || job.textContent.trim() == 'CREATE_FAILED') { return } diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 0d15638ad..56df053ad 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -459,12 +459,14 @@ // 加载任务状态 var timeid = window.setInterval(loadJobStatus, 15000); - // $(document).ready(loadJobStatus); + $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; const repoPath = job.dataset.repopath; - if (job.textContent.trim() == 'STOPPED') { + if (job.textContent.trim() == 'IMAGE_FAILED' || job.textContent.trim() == 'SUBMIT_FAILED' || job.textContent.trim() == 'DELETE_FAILED' + || job.textContent.trim() == 'KILLED' || job.textContent.trim() == 'COMPLETED' || job.textContent.trim() == 'FAILED' + || job.textContent.trim() == 'CANCELED' || job.textContent.trim() == 'LOST') { return } From bfc61b4a3dfacfb4c9aa9d4837999cca4d5c83e8 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 29 Oct 2021 10:48:49 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E5=90=8D=E7=A7=B0=E5=92=8C=E6=98=AF=E5=90=A6=E7=A7=81=E6=9C=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/repo_statistic.go | 6 ++++-- routers/repo/repo_statistic.go | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/models/repo_statistic.go b/models/repo_statistic.go index 9863c331a..adef672e0 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -9,8 +9,10 @@ import ( // RepoStatistic statistic info of all repository type RepoStatistic struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"unique(s) NOT NULL"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"unique(s) NOT NULL"` + Name string `xorm:"INDEX"` + IsPrivate bool Date string `xorm:"unique(s) NOT NULL"` NumWatches int64 `xorm:"NOT NULL DEFAULT 0"` NumWatchesAdded int64 `xorm:"NOT NULL DEFAULT 0"` diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 9bc70eaff..2de22a1ee 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -89,6 +89,8 @@ func RepoStatisticDaily(date string) { repoStat := models.RepoStatistic{ RepoID: repo.ID, Date: date, + Name: repo.Name, + IsPrivate: repo.IsPrivate, NumWatches: int64(repo.NumWatches), NumStars: int64(repo.NumStars), NumDownloads: repo.CloneCnt, From 2e6ff484897747274e9146f14f1516eafe79e6b4 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 29 Oct 2021 16:52:10 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=AC=A1=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/repo.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index b0bb608d0..437521d5a 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -531,6 +531,8 @@ func Download(ctx *context.Context) { } } + ctx.Repo.Repository.IncreaseCloneCnt() + ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } From 49a4bf09686b17906b89d802cc1d85157de938c4 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 29 Oct 2021 16:54:54 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=9C=89=E4=B8=AA=E6=8C=87=E6=A0=87?= =?UTF-8?q?=E6=9D=83=E9=87=8D=E5=92=8C=E6=96=87=E6=A1=A3=E4=B8=8D=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index eb0bab836..eb0e41c90 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1295,7 +1295,7 @@ func SetRadarMapConfig() { RadarMap.Impact = sec.Key("impact").MustFloat64(0.3) RadarMap.ImpactWatch = sec.Key("impact_watch").MustFloat64(0.1) - RadarMap.ImpactStar = sec.Key("impact_star").MustFloat64(0.3) + RadarMap.ImpactStar = sec.Key("impact_star").MustFloat64(0.2) RadarMap.ImpactFork = sec.Key("impact_fork").MustFloat64(0.3) RadarMap.ImpactCodeDownload = sec.Key("impact_code_download").MustFloat64(0.2) RadarMap.ImpactComments = sec.Key("impact_comments").MustFloat64(0.1) From e9cc609b7d36adcccf9df575e360d5b8a0d1dd24 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Fri, 29 Oct 2021 18:24:05 +0800 Subject: [PATCH 6/7] download model --- modules/storage/obs.go | 47 +++++++------ routers/repo/cloudbrain.go | 42 +----------- routers/repo/dir.go | 11 +-- routers/repo/modelarts.go | 78 ++++++++++++++-------- routers/routes/routes.go | 2 +- templates/repo/modelarts/notebook/index.tmpl | 4 +- templates/repo/modelarts/trainjob/index.tmpl | 4 +- .../repo/modelarts/trainjob/models/dir_list.tmpl | 27 ++++++++ .../repo/modelarts/trainjob/models/index.tmpl | 29 ++++++++ 9 files changed, 141 insertions(+), 103 deletions(-) create mode 100755 templates/repo/modelarts/trainjob/models/dir_list.tmpl create mode 100755 templates/repo/modelarts/trainjob/models/index.tmpl diff --git a/modules/storage/obs.go b/modules/storage/obs.go index e5e02952f..697ced6c5 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -5,7 +5,6 @@ package storage import ( - "fmt" "io" "path" "strconv" @@ -18,6 +17,15 @@ import ( "code.gitea.io/gitea/modules/setting" ) +type FileInfo struct { + FileName string `json:"FileName"` + ModTime string `json:"ModTime"` + IsDir bool `json:"IsDir"` + Size int64 `json:"Size"` + ParenDir string `json:"ParenDir"` + UUID string `json:"UUID"` +} + //check if has the object //todo：修改查询方式 func ObsHasObject(path string) (bool, error) { @@ -141,8 +149,7 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { - fmt.Printf("Code:%s\n", obsError.Code) - fmt.Printf("Message:%s\n", obsError.Message) + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err @@ -160,40 +167,38 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { - fmt.Printf("Code:%s\n", obsError.Code) - fmt.Printf("Message:%s\n", obsError.Message) + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err } } -func GetObsListObject(jobName string) ([]string, error) { - // jobName = "liuzx202110271830856" +func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath), "/") - log.Info("input.Prefix:", input.Prefix) + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") output, err := ObsCli.ListObjects(input) - log.Info("output.Prefix:", output) - ModelListArr := make([]string, 0) + fileInfos := make([]FileInfo, 0) if err == nil { - fmt.Printf("RequestId:%s\n", output.RequestId) for index, val := range output.Contents { - fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n", + log.Info("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n", index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size) str1 := strings.Split(val.Key, "/") - ModelList := str1[len(str1)-1] - ModelListArr = append(ModelListArr, ModelList) - log.Info("ModelListArr.Prefix:", ModelListArr) + fileName := str1[len(str1)-1] + log.Info("", fileName) + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir:false, + } + fileInfos = append(fileInfos, fileInfo) } - return ModelListArr, err + return fileInfos, err } else { if obsError, ok := err.(obs.ObsError); ok { - fmt.Println(obsError.Code) - fmt.Println(obsError.Message) - } else { - fmt.Println(err) + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) } return nil, err } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 782a51000..d5bc85a30 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -510,7 +510,7 @@ func CloudBrainShowModels(ctx *context.Context) { return } - var fileInfos []FileInfo + var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"]) @@ -583,50 +583,10 @@ func CloudBrainDownloadModel(ctx *context.Context) { http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } -// func TrainJobloadModel(ctx *context.Context) { -// parentDir := ctx.Query("parentDir") -// fileName := ctx.Query("fileName") -// jobName := ctx.Query("jobName") -// filePath := "jobs/" + jobName + "/model/" + parentDir -// url, err := storage.Attachments.PresignedGetURL(filePath, fileName) -// if err != nil { -// log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"]) -// ctx.ServerError("PresignedGetURL", err) -// return -// } - -// http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) -// } - -func TrainJobListModel(ctx *context.Context) { - ctx.Data["PageIsCloudBrain"] = true - jobID := ctx.Params(":jobid") - task, err := models.GetCloudbrainByJobID(jobID) - if err != nil { - log.Error("no such job!", ctx.Data["err"]) - ctx.ServerError("no such job:", err) - return - } - TrainJobListModel, err := storage.GetObsListObject(task.JobName) - log.Info("TrainJobListModel", TrainJobListModel) - fmt.Println("TrainJobListModel:", TrainJobListModel) - if err != nil { - log.Info("get TrainJobListModel failed:", err) - return - } - ctx.Data["task"] = task - ctx.Data["JobID"] = jobID - ctx.Data["ListModel"] = TrainJobListModel - ctx.HTML(200, tplModelArtsTrainJobListModel) -} - func TrainJobDownloadModel(ctx *context.Context) { JobName := ctx.Query("JobName") fileName := ctx.Query("file_name") - // JobName = "liuzx202110271830856" - // fileName = "Untitled.ipynb" - body, err := storage.ObsModelDownload(JobName, fileName) if err != nil { log.Info("download error.") diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 612019d46..406f3dc73 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -21,15 +21,6 @@ const ( tplDirIndex base.TplName = "repo/datasets/dirs/index" ) -type FileInfo struct { - FileName string `json:"FileName"` - ModTime string `json:"ModTime"` - IsDir bool `json:"IsDir"` - Size int64 `json:"Size"` - ParenDir string `json:"ParenDir"` - UUID string `json:"UUID"` -} - type RespGetDirs struct { ResultCode string `json:"resultCode"` FileInfos string `json:"fileInfos"` @@ -59,7 +50,7 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) { return } - var fileInfos []FileInfo + var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:", err.Error()) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 396457077..2c8f8f617 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -3,7 +3,6 @@ package repo import ( "encoding/json" "errors" - "fmt" "io" "net/http" "os" @@ -12,18 +11,18 @@ import ( "strings" "time" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/obs" - "code.gitea.io/gitea/modules/storage" - "github.com/unknwon/com" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + + "github.com/unknwon/com" ) const ( @@ -39,7 +38,7 @@ const ( tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" - tplModelArtsTrainJobListModel base.TplName = "repo/modelarts/trainjob/list_model" + tplModelArtsTrainJobShowModels base.TplName = "repo/modelarts/trainjob/models/index" ) // MustEnableDataset check if repository enable internal cb @@ -492,13 +491,13 @@ func NotebookDel(ctx *context.Context) { func TrainJobIndex(ctx *context.Context) { MustEnableModelArts(ctx) - can, err := canUserCreateTrainJob(ctx.User.ID) - if err != nil { - ctx.ServerError("canUserCreateTrainJob", err) - return - } - - ctx.Data["CanCreate"] = can + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //ctx.Data["CanCreate"] = can repo := ctx.Repo.Repository page := ctx.QueryInt("page") @@ -531,17 +530,17 @@ func TrainJobIndex(ctx *context.Context) { func TrainJobNew(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true - can, err := canUserCreateTrainJob(ctx.User.ID) - if err != nil { - ctx.ServerError("canUserCreateTrainJob", err) - return - } - - if !can { - log.Error("the user can not create train-job") - ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) - return - } + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //if !can { + // log.Error("the user can not create train-job") + // ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) + // return + //} t := time.Now() var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -1060,3 +1059,30 @@ func getConfigList(perPage, page int, sortBy, order, searchContent, configType s return list, nil } + +func TrainJobShowModels(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + jobID := ctx.Params(":jobid") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("no such job!", ctx.Data["msgID"]) + ctx.ServerError("no such job:", err) + return + } + + models, err := storage.GetObsListObject(task.JobName, parentDir) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.Data["Path"] = dirArray + ctx.Data["Dirs"] = models + ctx.Data["task"] = task + ctx.Data["JobID"] = jobID + ctx.HTML(200, tplModelArtsTrainJobShowModels) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 90d204a82..7e7d0642a 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -991,7 +991,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog) - m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobListModel) + m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobShowModels) m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel) }) m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) diff --git a/templates/repo/modelarts/notebook/index.tmpl b/templates/repo/modelarts/notebook/index.tmpl index 841954214..29c132580 100755 --- a/templates/repo/modelarts/notebook/index.tmpl +++ b/templates/repo/modelarts/notebook/index.tmpl @@ -423,7 +423,7 @@ // 加载任务状态 var timeid = window.setInterval(loadJobStatus, 15000); - $(document).ready(loadJobStatus); + $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; @@ -435,7 +435,7 @@ $.get(`/api/v1/repos/${repoPath}/modelarts/notebook/${jobID}`, (data) => { const jobID = data.JobID const status = data.JobStatus - if (status != job.textContent.trim() || status=='RUNNING') { + if (status != job.textContent.trim()) { //$('#' + jobID).text(status) //if (status == 'STOPPED') { window.location.reload() diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 56df053ad..4945e4781 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -459,7 +459,7 @@ // 加载任务状态 var timeid = window.setInterval(loadJobStatus, 15000); - $(document).ready(loadJobStatus); + $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; @@ -473,7 +473,7 @@ $.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => { const jobID = data.JobID const status = data.JobStatus - if (status != job.textContent.trim() || status=='RUNNING') { + if (status != job.textContent.trim()) { //$('#' + jobID).text(status) //if (status == 'STOPPED') { window.location.reload() diff --git a/templates/repo/modelarts/trainjob/models/dir_list.tmpl b/templates/repo/modelarts/trainjob/models/dir_list.tmpl new file mode 100755 index 000000000..a9683de77 --- /dev/null +++ b/templates/repo/modelarts/trainjob/models/dir_list.tmpl @@ -0,0 +1,27 @@ +{{if .Dirs}} + + + {{range .Dirs}} + + + + + + {{end}} + +
+ + + + {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} + + + + + {{.Size | FileSize}} + + + {{.ModTime}} +
+ +{{end}} diff --git a/templates/repo/modelarts/trainjob/models/index.tmpl b/templates/repo/modelarts/trainjob/models/index.tmpl new file mode 100755 index 000000000..3b53ad78e --- /dev/null +++ b/templates/repo/modelarts/trainjob/models/index.tmpl @@ -0,0 +1,29 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+
+

+ {{ range $index, $item := .Path }}{{ $item }}/{{ end }} +

+
+
+
+ +
+
+
+
+ {{template "repo/cloudbrain/models/dir_list" .}} +
+
+
+
+
+
+ + + +{{template "base/footer" .}} From f16f5f1041e568164a9058ce7231e49ec579621e Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Fri, 29 Oct 2021 19:33:58 +0800 Subject: [PATCH 7/7] download-cb2-model --- modules/storage/obs.go | 40 +++++++++++++--------- routers/repo/cloudbrain.go | 28 --------------- routers/repo/modelarts.go | 13 +++++++ .../repo/modelarts/trainjob/models/dir_list.tmpl | 2 +- .../repo/modelarts/trainjob/models/index.tmpl | 2 +- 5 files changed, 39 insertions(+), 46 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 697ced6c5..bd73281d0 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -181,17 +181,28 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) if err == nil { - for index, val := range output.Contents { - log.Info("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n", - index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size) + for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") - fileName := str1[len(str1)-1] - log.Info("", fileName) + var isDir bool + var fileName,nextParentDir string + if strings.HasSuffix(val.Key, "/") { + fileName = str1[len(str1)-2] + isDir = true + nextParentDir = fileName + if fileName == parentDir || (fileName + "/") == setting.OutPutPath { + continue + } + } else { + fileName = str1[len(str1)-1] + isDir = false + } + fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, Size: val.Size, - IsDir:false, + IsDir:isDir, + ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) } @@ -227,20 +238,17 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file return output.SignedUrl, nil } -func GetObsCreateSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { - +func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") + input.Expires = 60 * 60 - input.Method = obs.HttpMethodPut - - input.QueryParams = map[string]string{ - "partNumber": com.ToStr(partNumber, 10), - "uploadId": uploadId, - //"partSize": com.ToStr(partSize,10), - } + input.Method = obs.HttpMethodGet + reqParams := make(map[string]string) + reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" + input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d5bc85a30..a49d4026f 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -583,34 +583,6 @@ func CloudBrainDownloadModel(ctx *context.Context) { http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } -func TrainJobDownloadModel(ctx *context.Context) { - JobName := ctx.Query("JobName") - fileName := ctx.Query("file_name") - - body, err := storage.ObsModelDownload(JobName, fileName) - if err != nil { - log.Info("download error.") - } else { - defer body.Close() - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - p := make([]byte, 1024) - var readErr error - var readCount int - // 读取对象内容 - for { - readCount, readErr = body.Read(p) - if readCount > 0 { - ctx.Resp.Write(p[:readCount]) - //fmt.Printf("%s", p[:readCount]) - } - if readErr != nil { - break - } - } - } -} - func GetRate(ctx *context.Context) { var jobID = ctx.Params(":jobid") job, err := models.GetCloudbrainByJobID(jobID) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 2c8f8f617..166fe6704 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1086,3 +1086,16 @@ func TrainJobShowModels(ctx *context.Context) { ctx.Data["JobID"] = jobID ctx.HTML(200, tplModelArtsTrainJobShowModels) } + +func TrainJobDownloadModel(ctx *context.Context) { + parentDir := ctx.Query("parentDir") + fileName := ctx.Query("fileName") + jobName := ctx.Query("jobName") + url, err := storage.GetObsCreateSignedUrl(jobName, parentDir, fileName) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +} diff --git a/templates/repo/modelarts/trainjob/models/dir_list.tmpl b/templates/repo/modelarts/trainjob/models/dir_list.tmpl index a9683de77..9e92681e5 100755 --- a/templates/repo/modelarts/trainjob/models/dir_list.tmpl +++ b/templates/repo/modelarts/trainjob/models/dir_list.tmpl @@ -6,7 +6,7 @@ - + {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} diff --git a/templates/repo/modelarts/trainjob/models/index.tmpl b/templates/repo/modelarts/trainjob/models/index.tmpl index 3b53ad78e..9ca62dc2c 100755 --- a/templates/repo/modelarts/trainjob/models/index.tmpl +++ b/templates/repo/modelarts/trainjob/models/index.tmpl @@ -16,7 +16,7 @@
- {{template "repo/cloudbrain/models/dir_list" .}} + {{template "repo/modelarts/trainjob/models/dir_list" .}}