From 83e61aad76b8121b5cb10ea9b45d6fd3b4428b96 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 6 Jun 2022 15:15:12 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 5 +++-- models/dataset.go | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index ea8f1645f..a93082bf0 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -41,8 +41,9 @@ type Attachment struct { UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added CommentID int64 Name string - Description string `xorm:"TEXT"` - DownloadCount int64 `xorm:"DEFAULT 0"` + Description string `xorm:"TEXT"` + DownloadCount int64 `xorm:"DEFAULT 0"` + UseCount int64 Size int64 `xorm:"DEFAULT 0"` IsPrivate bool `xorm:"DEFAULT false"` DecompressState int32 `xorm:"DEFAULT 0"` diff --git a/models/dataset.go b/models/dataset.go index d4a7748d3..baf7731e4 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -25,6 +25,7 @@ type Dataset struct { Category string Description string `xorm:"TEXT"` DownloadTimes int64 + UseCount int64 NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` Recommend bool `xorm:"INDEX NOT NULL DEFAULT false"` License string From 7315dd24e251e5ca235d4602d5a7042f0958c7d9 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 7 Jun 2022 11:52:21 +0800 Subject: [PATCH 02/15] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 15 ++++++++++++--- models/cloudbrain.go | 4 ++++ models/dataset.go | 17 ++++++++++++++--- models/repo_list.go | 2 ++ routers/admin/dataset.go | 4 ++++ routers/home.go | 4 ++++ 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index a93082bf0..fdf930ecf 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -41,9 +41,9 @@ type Attachment struct { UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added CommentID int64 Name string - Description string `xorm:"TEXT"` - DownloadCount int64 `xorm:"DEFAULT 0"` - UseCount int64 + Description string `xorm:"TEXT"` + DownloadCount int64 `xorm:"DEFAULT 0"` + UseCount int64 `xorm:"DEFAULT 0"` Size int64 `xorm:"DEFAULT 0"` IsPrivate bool `xorm:"DEFAULT false"` DecompressState int32 `xorm:"DEFAULT 0"` @@ -108,6 +108,15 @@ func (a *Attachment) IncreaseDownloadCount() error { return nil } +func IncreaseAttachmentUseCount(uuid string) error { + // Update use count. + if _, err := x.Exec("UPDATE `attachment` SET use_count=use_count+1 WHERE uuid=?", uuid); err != nil { + return fmt.Errorf("increase attachment use count: %v", err) + } + + return nil +} + func (a *Attachment) UpdateDatasetUpdateUnix() error { // Update download count. if _, err := x.Exec("UPDATE `dataset` SET updated_unix="+fmt.Sprint(time.Now().Unix())+" WHERE id=?", a.DatasetID); err != nil { diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 810e68d30..451a36c81 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1395,6 +1395,8 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { if _, err = x.NoAutoTime().Insert(cloudbrain); err != nil { return err } + + go IncreaseDatasetUseCount(cloudbrain.Uuid) return nil } @@ -1629,6 +1631,8 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { return err } + go IncreaseDatasetUseCount(new.Uuid) + return nil } func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { diff --git a/models/dataset.go b/models/dataset.go index baf7731e4..aa29984f0 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -25,9 +25,9 @@ type Dataset struct { Category string Description string `xorm:"TEXT"` DownloadTimes int64 - UseCount int64 - NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` - Recommend bool `xorm:"INDEX NOT NULL DEFAULT false"` + UseCount int64 `xorm:"DEFAULT 0"` + NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` + Recommend bool `xorm:"INDEX NOT NULL DEFAULT false"` License string Task string ReleaseID int64 `xorm:"INDEX"` @@ -351,6 +351,17 @@ func UpdateDataset(ctx DBContext, rel *Dataset) error { return err } +func IncreaseDatasetUseCount(uuid string) { + + IncreaseAttachmentUseCount(uuid) + + attachment, _ := GetAttachmentByUUID(uuid) + if attachment != nil { + x.Exec("UPDATE `dataset` SET use_count=use_count+1 WHERE id=?", attachment.DatasetID) + } + +} + // GetDatasetByID returns Dataset with given ID. func GetDatasetByID(id int64) (*Dataset, error) { rel := new(Dataset) diff --git a/models/repo_list.go b/models/repo_list.go index 253cc968c..92654c11c 100755 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -218,6 +218,8 @@ const ( SearchOrderByForks SearchOrderBy = "num_forks ASC" SearchOrderByForksReverse SearchOrderBy = "num_forks DESC" SearchOrderByDownloadTimes SearchOrderBy = "download_times DESC" + SearchOrderByUseCount SearchOrderBy = "use_count ASC" + SearchOrderByUseCountReverse SearchOrderBy = "use_count DESC" SearchOrderByHot SearchOrderBy = "(num_watches + num_stars + num_forks + clone_cnt) DESC" SearchOrderByActive SearchOrderBy = "(num_issues + num_pulls + num_commit) DESC" SearchOrderByWatches SearchOrderBy = "num_watches DESC" diff --git a/routers/admin/dataset.go b/routers/admin/dataset.go index 6b29b06ff..d1a8f2780 100644 --- a/routers/admin/dataset.go +++ b/routers/admin/dataset.go @@ -61,6 +61,10 @@ func Datasets(ctx *context.Context) { orderBy = models.SearchOrderByForksReverse case "fewestforks": orderBy = models.SearchOrderByForks + case "mostusecount": + orderBy = models.SearchOrderByUseCountReverse + case "fewestusecount": + orderBy = models.SearchOrderByUseCount default: ctx.Data["SortType"] = "recentupdate" orderBy = models.SearchOrderByRecentUpdated diff --git a/routers/home.go b/routers/home.go index 38acffb2f..a34302876 100755 --- a/routers/home.go +++ b/routers/home.go @@ -322,6 +322,10 @@ func ExploreDatasets(ctx *context.Context) { orderBy = models.SearchOrderByStarsReverse case "feweststars": orderBy = models.SearchOrderByStars + case "mostusecount": + orderBy = models.SearchOrderByUseCountReverse + case "fewestusecount": + orderBy = models.SearchOrderByUseCount case "default": orderBy = models.SearchOrderByDefault default: From 007b224ab6fdd85fbc57b285c50ca4b9048fa004 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 8 Jun 2022 15:32:58 +0800 Subject: [PATCH 03/15] fix issue --- options/locale/locale_en-US.ini | 3 +- options/locale/locale_zh-CN.ini | 3 +- templates/explore/datasets.tmpl | 342 ++++++++++++++++++++++--------------- templates/repo/datasets/index.tmpl | 4 +- 4 files changed, 207 insertions(+), 145 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index c52a369ce..0b33879a2 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1446,7 +1446,8 @@ issues.filter_sort.feweststars = Fewest stars issues.filter_sort.mostforks = Most forks issues.filter_sort.fewestforks = Fewest forks issues.filter_sort.downloadtimes = Most downloaded -issues.filter_sort.moststars = Most star +issues.filter_sort.mostusecount = Most Quote +issues.filter_sort.fewestusecount=Fewest Quote issues.action_open = Open issues.action_close = Close issues.action_label = Label diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index cb1c7565a..3402de834 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1452,12 +1452,13 @@ issues.filter_sort.mostcomment=最多评论 issues.filter_sort.leastcomment=最少评论 issues.filter_sort.nearduedate=到期日从近到远 issues.filter_sort.farduedate=到期日从远到近 -issues.filter_sort.moststars=点赞由多到少 issues.filter_sort.feweststars=点赞由少到多 issues.filter_sort.mostforks=派生由多到少 issues.filter_sort.fewestforks=派生由少到多 issues.filter_sort.downloadtimes=下载次数 issues.filter_sort.moststars=收藏数量 +issues.filter_sort.mostusecount=最多引用 +issues.filter_sort.fewestusecount=最少引用 issues.action_open=开启 issues.action_close=关闭 issues.action_label=标签 diff --git a/templates/explore/datasets.tmpl b/templates/explore/datasets.tmpl index 5739ebb10..57942c24a 100644 --- a/templates/explore/datasets.tmpl +++ b/templates/explore/datasets.tmpl @@ -1,133 +1,164 @@ {{template "base/head" .}}
{{template "explore/dataset_search" .}}
- + {{template "explore/dataset_left" .}} - +

- {{.i18n.Tr "datasets"}} + {{.i18n.Tr "datasets"}}

- {{.i18n.Tr "repo.issues.filter_sort"}} - + {{.i18n.Tr "repo.issues.filter_sort"}} +
- {{.i18n.Tr "repo.issues.filter_sort.default"}} - {{.i18n.Tr "repo.issues.filter_sort.latest"}} - {{.i18n.Tr "repo.issues.filter_sort.oldest"}} - {{.i18n.Tr "repo.issues.filter_sort.recentupdate"}} - {{.i18n.Tr "repo.issues.filter_sort.leastupdate"}} - {{.i18n.Tr "repo.issues.filter_sort.downloadtimes"}} - {{.i18n.Tr "repo.issues.filter_sort.moststars"}} + {{.i18n.Tr "repo.issues.filter_sort.default"}} + {{.i18n.Tr "repo.issues.filter_sort.latest"}} + {{.i18n.Tr "repo.issues.filter_sort.oldest"}} + {{.i18n.Tr "repo.issues.filter_sort.recentupdate"}} + {{.i18n.Tr "repo.issues.filter_sort.leastupdate"}} + {{.i18n.Tr "repo.issues.filter_sort.downloadtimes"}} + {{.i18n.Tr "repo.issues.filter_sort.moststars"}} + {{.i18n.Tr "repo.issues.filter_sort.mostusecount"}} + {{.i18n.Tr "repo.issues.filter_sort.fewestusecount"}}
@@ -141,85 +172,114 @@ {{end}}
- 仅显示平台推荐 + + 仅显示平台推荐
{{range $k, $v :=.Datasets}} -
+
-
- {{.Repo.OwnerName}} / {{.Repo.Alias}} +
+ {{.Title}}{{if .Recommend}}{{end}} {{if $.IsSigned}} - - - {{.DownloadTimes}} + +
- + + + +
${starItems[{{$k}}]} {{else}} - - - {{.DownloadTimes}} + +
- + + + +
${starItems[{{$k}}]} {{end}} +
-
{{.Title}}{{if .Recommend}}{{end}}
{{if or (.Category) (.Task) (.License)}} {{end}} -
+

{{.Description}}

- +
{{if eq .UserID 0}} - + {{else}} - + {{end}} - 创建于：{{TimeSinceUnix1 .CreatedUnix}} + {{TimeSinceUnixShort .CreatedUnix}} + + + {{.UseCount}} + + + + {{.DownloadTimes}} +
{{end}} - +
- +
diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index 74448406f..88e9ff149 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -105,7 +105,7 @@ } .heart-stroke { - stroke: #666; + stroke: #FA8C16; stroke-width: 2; fill: #fff } @@ -148,7 +148,7 @@
-

{{.dataset.Title}}

+

{{.dataset.Title}}

From 3a322ca9678afafe78477e6ebc98c0aaec000ba6 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 8 Jun 2022 15:41:19 +0800 Subject: [PATCH 04/15] no cache when downloading --- routers/repo/cloudbrain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a5dd52956..b034d3d12 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1150,6 +1150,7 @@ func CloudBrainDownloadModel(ctx *context.Context) { ctx.ServerError("PresignedGetURL", err) return } + ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } From 8f12d315d4f2d9787e0e65fc07fba333c2e4216b Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Jun 2022 16:03:39 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 8 ++++---- models/dataset.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index fdf930ecf..bb99ba467 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -43,7 +43,7 @@ type Attachment struct { Name string Description string `xorm:"TEXT"` DownloadCount int64 `xorm:"DEFAULT 0"` - UseCount int64 `xorm:"DEFAULT 0"` + UseNumber int64 `xorm:"DEFAULT 0"` Size int64 `xorm:"DEFAULT 0"` IsPrivate bool `xorm:"DEFAULT false"` DecompressState int32 `xorm:"DEFAULT 0"` @@ -108,9 +108,9 @@ func (a *Attachment) IncreaseDownloadCount() error { return nil } -func IncreaseAttachmentUseCount(uuid string) error { - // Update use count. - if _, err := x.Exec("UPDATE `attachment` SET use_count=use_count+1 WHERE uuid=?", uuid); err != nil { +func IncreaseAttachmentUseNumber(uuid string) error { + // Update use number. + if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid=?", uuid); err != nil { return fmt.Errorf("increase attachment use count: %v", err) } diff --git a/models/dataset.go b/models/dataset.go index aa29984f0..726a5010f 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -213,7 +213,7 @@ func SearchDatasetByCondition(opts *SearchDatasetOptions, cond builder.Cond) (Da defer sess.Close() datasets := make(DatasetList, 0, opts.PageSize) - selectColumnsSql := "distinct dataset.id,dataset.title, dataset.status, dataset.category, dataset.description, dataset.download_times, dataset.license, dataset.task, dataset.release_id, dataset.user_id, dataset.repo_id, dataset.created_unix,dataset.updated_unix,dataset.num_stars,dataset.recommend" + selectColumnsSql := "distinct dataset.id,dataset.title, dataset.status, dataset.category, dataset.description, dataset.download_times, dataset.license, dataset.task, dataset.release_id, dataset.user_id, dataset.repo_id, dataset.created_unix,dataset.updated_unix,dataset.num_stars,dataset.recommend,dataset.use_count" count, err := sess.Distinct("dataset.id").Join("INNER", "repository", "repository.id = dataset.repo_id"). Join("INNER", "attachment", "attachment.dataset_id=dataset.id"). @@ -353,7 +353,7 @@ func UpdateDataset(ctx DBContext, rel *Dataset) error { func IncreaseDatasetUseCount(uuid string) { - IncreaseAttachmentUseCount(uuid) + IncreaseAttachmentUseNumber(uuid) attachment, _ := GetAttachmentByUUID(uuid) if attachment != nil { From b158cdcbeb7e41fe2ac65151976db68fc6a68b0c Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 8 Jun 2022 16:31:17 +0800 Subject: [PATCH 06/15] show metric_statistic backend --- models/cloudbrain.go | 11 ++++++++++ modules/modelarts/resty.go | 41 ++++++++++++++++++++++++++++++++++++++ routers/api/v1/api.go | 1 + routers/api/v1/repo/modelarts.go | 43 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 810e68d30..d0cf0ff84 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1150,6 +1150,17 @@ type LogFile struct { Name string } +type GetTrainJobMetricStatisticResult struct { + TrainJobResult + Interval int `json:"interval"` //查询的时间间隔，单位为分钟 + MetricsInfo []Metrics `json:"metrics"` //监控详情 +} + +type Metrics struct { + Metric string `json:"metric"` //监控指标项 + Value []string `json:"value"` //获取的监控值的序列，元素为String类型 +} + func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { sess := x.NewSession() defer sess.Close() diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index 2f7d08c35..961e02538 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -1119,3 +1119,44 @@ sendjob: return &result, nil } + +func GetTrainJobMetricStatistic(jobID, versionID, podName string) (*models.GetTrainJobMetricStatisticResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetTrainJobMetricStatisticResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic") + + if err != nil { + return nil, fmt.Errorf("resty GetTrainJobMetricStatistic: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetTrainJobMetricStatistic failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("GetTrainJobMetricStatistic failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetTrainJobMetricStatistic(%s) failed", jobID) + return &result, fmt.Errorf("获取任务资源占用情况失败:%s", result.ErrorMsg) + } + + return &result, nil +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 9a05aa8ae..d6d3b001a 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -922,6 +922,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/del_version", repo.DelTrainJobVersion) m.Post("/stop_version", repo.StopTrainJobVersion) m.Get("/model_list", repo.ModelList) + m.Get("/metric_statistics", repo.TrainJobGetMetricStatistic) }) }) m.Group("/inference-job", func() { diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 9e4edea03..c14976282 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -462,3 +462,46 @@ func ResultList(ctx *context.APIContext) { "PageIsCloudBrain": true, }) } + +func TrainJobGetMetricStatistic(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + + result, err := trainJobGetMetricStatistic(jobID, versionName) + if err != nil { + log.Error("trainJobGetMetricStatistic(%s) failed:%v", jobID, err.Error()) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "Interval": result.Interval, + "MetricsInfo": result.MetricsInfo, + }) +} + +func trainJobGetMetricStatistic(jobID string, versionName string) (*models.GetTrainJobMetricStatisticResult, error) { + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) + return nil, err + } + + resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error()) + return nil, err + } + + result, err := modelarts.GetTrainJobMetricStatistic(jobID, strconv.FormatInt(task.VersionID, 10), resultLogFile.LogFileList[0]) + if err != nil { + log.Error("GetTrainJobMetricStatistic(%s) failed:%v", jobID, err.Error()) + return nil, err + } + + return result, err +} From e38fa7a7bc10ade10c5a397d74b9a9f1fafeb26f Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 8 Jun 2022 17:02:11 +0800 Subject: [PATCH 07/15] fix issue --- options/locale/locale_en-US.ini | 4 ++++ options/locale/locale_zh-CN.ini | 4 ++++ templates/repo/datasets/index.tmpl | 43 +++++++++++++++++++++----------------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0b33879a2..7cc0ed00f 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -898,6 +898,10 @@ search_dataset = Search Dataset Files unzip_tooltips = If it has not been decompressed for a long time, please check whether the compressed package has encrypted files or file errors zip_failed = Decompression failed, please check whether the compressed package is encrypted or contact technical support dataset_desc = The description should not exceed 1024 characters +unzip_successed=Unzip Successed +unzip_failed=Unzip Failed +unzip_stared=Unzipping +unzip_status=Unzip Status [repo] owner = Owner repo_name = Repository Name diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 3402de834..1c24dceab 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -903,6 +903,10 @@ search_dataset = 搜索数据集文件 unzip_tooltips = 如果长时间未解压,请检查压缩包是否有加密文件或者文件错误 zip_failed = 解压失败，请检查压缩包是否有加密或者联系技术支持人员。 dataset_desc = 描述字数不超过1024个字符 +unzip_successed=解压成功 +unzip_failed=解压失败 +unzip_stared=解压中 +unzip_status=解压状态 [repo] owner=拥有者 diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index 88e9ff149..8e2c98a4e 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -276,27 +276,32 @@
- {{if .Description}} -
{{.Description}}
{{$.i18n.Tr "dataset.download"}}：{{.DownloadCount}}
- - {{.Name}} - - - {{else}} - -
{{$.i18n.Tr "dataset.download"}}：{{.DownloadCount}}
- - {{.Name}} - +
+ + {{$.i18n.Tr "dataset.unzip_status"}}：{{if eq .DecompressState 1}}{{$.i18n.Tr "dataset.unzip_successed"}}{{else if eq .DecompressState 0}}{{$.i18n.Tr "dataset.unzip_stared"}}{{else}}{{$.i18n.Tr "dataset.unzip_failed"}}{{end}} +   {{$.i18n.Tr "dataset.download"}}：{{.DownloadCount}} + {{if .Description}}  {{$.i18n.Tr "dataset.description"}}：{{.Description}}{{end}} +
+ +
+ {{.DecompressState}} + {{if eq .DecompressState 1}} + + {{else if eq .DecompressState 0}} + + {{else}} + + {{end}} + + {{.Name}} + + +
- {{end}} - -
{{.Size | FileSize}} From 6473f414c21800308b1db46844a62e0dfc6da80c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Jun 2022 17:22:57 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/dataset.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 0e57fe1a0..9d7f4a4d7 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -172,6 +172,10 @@ func DatasetIndex(ctx *context.Context) { for _, attachment := range pageAttachments { uploader, _ := models.GetUserByID(attachment.UploaderID) attachment.Uploader = uploader + if !strings.HasSuffix(attachment.Name, ".zip") { + attachment.DecompressState = 3 //非zip文件 + } + } ctx.Data["Page"] = pager From 13fe2ad4b299e5cd0f42d18fd34480914017a732 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 9 Jun 2022 10:20:02 +0800 Subject: [PATCH 09/15] fix issue --- templates/repo/datasets/index.tmpl | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index 8e2c98a4e..a1feb4310 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -283,18 +283,19 @@   {{$.i18n.Tr "dataset.download"}}：{{.DownloadCount}} {{if .Description}}  {{$.i18n.Tr "dataset.description"}}：{{.Description}}{{end}}
-
- {{.DecompressState}} {{if eq .DecompressState 1}} {{else if eq .DecompressState 0}} - {{else}} + {{else if eq .DecompressState 2}} + {{else}} + {{end}} {{.Name}} @@ -327,18 +328,6 @@ {{.CreatedUnix | TimeSinceUnix1}}
-
{{$.i18n.Tr "dataset.download"}} @@ -355,11 +344,6 @@ @click="setPrivate('{{.UUID}}',true,{{$k}})" v-else="privates[{{$k}}]">{{$.i18n.Tr "dataset.set_private"}} {{end}} - @@ -368,7 +352,6 @@ {{$.i18n.Tr "dataset.copy_url"}} - {{if and ($.CanWrite) (eq .DecompressState 1) }} {{$.i18n.Tr "dataset.annotation"}} From 484c6792c95aad6138b9063287c666cd8c564307 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 9 Jun 2022 10:23:22 +0800 Subject: [PATCH 10/15] fix issue --- templates/repo/datasets/index.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index a1feb4310..9a8f2aaea 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -279,8 +279,9 @@
- {{$.i18n.Tr "dataset.unzip_status"}}：{{if eq .DecompressState 1}}{{$.i18n.Tr "dataset.unzip_successed"}}{{else if eq .DecompressState 0}}{{$.i18n.Tr "dataset.unzip_stared"}}{{else}}{{$.i18n.Tr "dataset.unzip_failed"}}{{end}} -   {{$.i18n.Tr "dataset.download"}}：{{.DownloadCount}} + {{if ne .DecompressState 3}}{{$.i18n.Tr "dataset.unzip_status"}}：{{if eq .DecompressState 1}}{{$.i18n.Tr "dataset.unzip_successed"}}{{else if eq .DecompressState 0}}{{$.i18n.Tr "dataset.unzip_stared"}}{{else}}{{$.i18n.Tr "dataset.unzip_failed"}}{{end}} +   {{end}}{{$.i18n.Tr "dataset.download"}}：{{.DownloadCount}} {{if .Description}}  {{$.i18n.Tr "dataset.description"}}：{{.Description}}{{end}}
From cde34795b1e122fa3e025f4280bfeaf43d742a7b Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 9 Jun 2022 11:30:34 +0800 Subject: [PATCH 11/15] download npu log --- routers/repo/modelarts.go | 39 +++++++++++++++++++++++++++++++++++---- routers/routes/routes.go | 1 + 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 95ca8df62..6a94ef0e7 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2251,7 +2251,6 @@ func ModelDownload(ctx *context.Context) { versionName := ctx.Query("version_name") parentDir := ctx.Query("parent_dir") fileName := ctx.Query("file_name") - log.Info("DownloadSingleModelFile start.") task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) if err != nil { log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) @@ -2259,7 +2258,6 @@ func ModelDownload(ctx *context.Context) { } path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir, fileName), "/") - log.Info("Download path is:%s", path) url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) if err != nil { @@ -2267,6 +2265,7 @@ func ModelDownload(ctx *context.Context) { ctx.ServerError("GetObsCreateSignedUrl", err) return } + ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } @@ -2278,13 +2277,11 @@ func ResultDownload(ctx *context.Context) { versionName := ctx.Query("version_name") parentDir := ctx.Query("parent_dir") fileName := ctx.Query("file_name") - log.Info("DownloadResult start.") task := ctx.Cloudbrain if err != nil { ctx.Data["error"] = err.Error() } path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, "result/", versionName, parentDir, fileName), "/") - log.Info("Download path is:%s", path) url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) if err != nil { @@ -2292,6 +2289,7 @@ func ResultDownload(ctx *context.Context) { ctx.ServerError("GetObsCreateSignedUrl", err) return } + ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } func DeleteJobStorage(jobName string) error { @@ -2390,3 +2388,36 @@ func SetJobCount(ctx *context.Context) { } ctx.Data["jobCount"] = jobCount } + +func TrainJobDownloadLogFile(ctx *context.Context) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + versionName := ctx.Query("version_name") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetCloudbrainByJobIDAndVersionName", err) + return + } + + resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetTrainJobLogFileNames", err) + return + } + + path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, modelarts.LogPath, versionName, resultLogFile.LogFileList[0]), "/") + + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) + if err != nil { + log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err) + return + } + ctx.Resp.Header().Set("Cache-Control", "max-age=0") + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 7ba8fe61a..278c40204 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1136,6 +1136,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobStop) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobDel) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) + m.Get("/download_log_file", cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobDownloadLogFile) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobNewVersion) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) }) From d8e38d04dafb08095a6298ffe3b647cba9c18e04 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 9 Jun 2022 20:02:20 +0800 Subject: [PATCH 12/15] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 8 +++ web_src/js/index.js | 97 +++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 8f168fcf9..695ae601e 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -305,6 +305,7 @@ data-tab="first{{$k}}">{{$.i18n.Tr "repo.modelarts.train_job.config"}} {{$.i18n.Tr "repo.modelarts.log"}} + 资源占用情况 {{$.i18n.Tr "repo.model_download"}}
@@ -501,6 +502,13 @@
+
+ +
+
+
diff --git a/web_src/js/index.js b/web_src/js/index.js index 34b5febc5..c51f6638e 100755 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -5070,3 +5070,100 @@ function initcreateRepo() { } initcreateRepo() + + +function initChartsNpu() { + const url = window.location.href + const urlArr = url.split('/') + let userName = urlArr.slice(-5)[0] + let repoPath = urlArr.slice(-4)[0] + let jobID = urlArr.slice(-1)[0] + + + let options = { + legend: { + data: [] + }, + grid: { + top: '30%', + bottom: '2%', + containLabel: true + }, + tooltip: { + trigger: 'axis', + backgroundColor: 'rgb(51, 56, 84)', + borderColor: 'rgb(51, 51, 51)', + borderWidth: 0, + textStyle: { + color: '#fff' + }, + axisPointer: { + type: 'line' + } + }, + xAxis: { + type: 'category', + data: [], + boundaryGap: false, + axisLabel: { + interval: 'auto' + }, + name: '时间(min)' + }, + yAxis: { + min: 0, + max: 100, + show: true, + name: '占有率(%)', + axisLine: { + show: true + }, + axisTick: { show: true } + }, + + series: [] + }; + $('.metric_chart').click(function (e) { + let versionName = $(this).data('version') + console.log("11111", versionName) + let myCharts = echarts.init(document.getElementById(`metric-${versionName}`)) + $.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each`, (res) => { + let filterDta = res.MetricsInfo.filter((item) => { + + return !(['recvBytesRate', 'diskWriteRate', 'sendBytesRate', 'diskReadRate'].includes(item.metric)) + }) + let legenData = filterDta.map((item) => { + return item.metric + }) + let seriesData = filterDta.map((item) => { + let seriesOption = { + name: item.metric, + type: 'line', + symbol: 'circle', + symbolSize: 10, + smooth: true, + showSymbol: false, + lineStyle: { + width: 2, + shadowColor: 'rgba(0,0,0,0.3)', + shadowBlur: 10, + shadowOffsetY: 8 + }, + data: item.value + } + return seriesOption + }) + let xLength = res.MetricsInfo[0].value.length + console.log(legenData) + options.xAxis.data = Array.from({ length: xLength }, (_, index) => index + 1) + options.legend.data = legenData + options.series = seriesData + options && myCharts.setOption(options); + + }) + options && myCharts.setOption(options); + + }) +} + +initChartsNpu() \ No newline at end of file From 65ca5c5641e9146119a22ada2694a9aec9d6b774 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Fri, 10 Jun 2022 09:52:18 +0800 Subject: [PATCH 13/15] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 8f168fcf9..2308d7cf7 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -220,6 +220,7 @@
{{.displayJobName}}
+ {{range $k ,$v := .version_list_task}}
-
+ +
From 61d01bba9aa33f96c11b8827da4921bb24fd9390 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Fri, 10 Jun 2022 11:20:25 +0800 Subject: [PATCH 14/15] debug --- modules/storage/obs.go | 14 ++++++++++++++ routers/repo/modelarts.go | 11 +++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 03349864a..33730b72c 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -564,3 +564,17 @@ func ObsCreateObject(path string) error { return nil } + +func GetObsLogFileName(prefix string) (string, error) { + input := &obs.ListObjectsInput{} + input.Bucket = setting.Bucket + input.Prefix = prefix + + output, err := ObsCli.ListObjects(input) + if err != nil { + log.Error("PutObject failed:", err.Error()) + return "", err + } + + return output.Contents[0].Key, nil +} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 6a94ef0e7..44476ab1e 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2403,16 +2403,15 @@ func TrainJobDownloadLogFile(ctx *context.Context) { return } - resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10)) + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, modelarts.LogPath, versionName), "/") + "/job" + key, err := storage.GetObsLogFileName(prefix) if err != nil { - log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetTrainJobLogFileNames", err) + log.Error("GetObsLogFileName(%s) failed:%v", jobID, err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsLogFileName", err) return } - path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, modelarts.LogPath, versionName, resultLogFile.LogFileList[0]), "/") - - url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key) if err != nil { log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err) From af7c36e8f996e1e4651519b31744c126fcb4ba1a Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 10 Jun 2022 14:17:18 +0800 Subject: [PATCH 15/15] fix issue --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + templates/repo/modelarts/trainjob/show.tmpl | 13 ++++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 7cc0ed00f..06ef92403 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1145,6 +1145,7 @@ modelarts.infer_job.model_version = Model/Version modelarts.infer_job.select_model = Select Model modelarts.infer_job.boot_file_helper=The startup file is the entry file for your program execution and must end in.py.Such as inference.py, main.py, example/inference. Py, case/main.py. modelarts.infer_job.tooltip = The model has been deleted and cannot be viewed. +modelarts.download_log=Download log file debug_task_not_created = Debug task has not been created diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 1c24dceab..75d2fe7fb 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1155,6 +1155,7 @@ modelarts.infer_job.model_version = 模型/版本 modelarts.infer_job.select_model = 选择模型 modelarts.infer_job.boot_file_helper=启动文件是您程序执行的入口文件，必须是以.py结尾的文件。比如inference.py、main.py、example/inference.py、case/main.py。 modelarts.infer_job.tooltip = 该模型已删除，无法查看。 +modelarts.download_log=下载日志文件 debug_task_not_created = 未创建过调试任务 diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 2308d7cf7..56518834b 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -177,6 +177,12 @@ border: 1px solid #dfe1e6; } + .ti-download-file { + display: flex; + align-items: center; + margin: 0.5rem 0; + } + .disabled { cursor: default; pointer-events: none; @@ -481,11 +487,12 @@