From d84a497a995e1f5958e750ed7255e17deae58a9f Mon Sep 17 00:00:00 2001 From: chenshihai Date: Mon, 26 Sep 2022 09:59:22 +0800 Subject: [PATCH 01/32] =?UTF-8?q?#2193=20=E9=95=9C=E5=83=8F=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E5=90=8C=E6=AD=A5=E6=8F=90=E7=A4=BA=E8=AF=AD=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20#2642=20=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0?= =?UTF-8?q?=E4=BA=91=E8=84=91=E4=BB=BB=E5=8A=A1=E5=88=97=E8=A1=A8=E4=BA=91?= =?UTF-8?q?=E8=84=91=E4=BE=A7=E5=90=8D=E7=A7=B0=E5=88=97=E6=82=AC=E6=B5=AE?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E6=96=87=E5=AD=97=E4=B8=8D=E5=AF=B9=20#2870?= =?UTF-8?q?=20=E4=B8=AA=E4=BA=BA=E4=B8=AD=E5=BF=83=E7=9A=84=E4=BA=91?= =?UTF-8?q?=E8=84=91=E4=BB=BB=E5=8A=A1=E5=88=97=E8=A1=A8=EF=BC=8C=E7=AD=9B?= =?UTF-8?q?=E9=80=89=E5=90=AF=E6=99=BA=E9=9B=86=E7=BE=A4=E6=89=80=E6=9C=89?= =?UTF-8?q?=E7=9A=84npu=E8=B0=83=E8=AF=95=E4=BB=BB=E5=8A=A1=EF=BC=8C?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 2 ++ options/locale/locale_zh-CN.ini | 4 +++- templates/admin/cloudbrain/list.tmpl | 9 +++++---- templates/user/dashboard/cloudbrains.tmpl | 25 ++++++++++++------------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5923dfd5f..115f5aa36 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1364,7 +1364,9 @@ fork_guest_user = Sign in to fork this repository. copy_link = Copy copy_link_success = Link has been copied copy_link_error = Use ⌘C or Ctrl-C to copy +copy = Copy copied = Copied OK +copied_error = Copied error unwatch = Unwatch watch = Watch unstar = Unstar diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 29e9811d5..8efdff9c0 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1380,7 +1380,9 @@ fork_guest_user=登录并 派生 这个项目。 copy_link=复制链接 copy_link_success=已复制链接 copy_link_error=请按下 ⌘-C 或 Ctrl-C 复制 +copy=复制 copied=复制成功 +copied_error=复制失败 unwatch=取消关注 watch=关注 unstar=取消点赞 @@ -2013,7 +2015,7 @@ settings.githooks=管理 Git 钩子 settings.basic_settings=基本设置 settings.mirror_settings=镜像设置 settings.sync_mirror=同步 -settings.mirror_sync_in_progress=镜像同步正在进行中，请稍后后再试。 +settings.mirror_sync_in_progress=镜像同步正在进行中，请稍后再试。 settings.email_notifications.enable=启用邮件通知 settings.email_notifications.onmention=只在被提到时邮件通知 settings.email_notifications.disable=停用邮件通知 diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index 14951dd69..89649a307 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -205,11 +205,12 @@
- {{.JobName}} diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl index 17b40315c..defed27bc 100755 --- a/templates/user/dashboard/cloudbrains.tmpl +++ b/templates/user/dashboard/cloudbrains.tmpl @@ -15,11 +15,11 @@ data-all-compute="{{.i18n.Tr "admin.cloudbrain.all_computing_resources"}}" data-all-status="{{.i18n.Tr "admin.cloudbrain.all_status"}}">
{{template "admin/cloudbrain/search_dashboard" .}} -
+
{{template "base/alert" .}} -
+
-
+
@@ -377,19 +377,18 @@
{{end}} - {{end}} -
-
- - -
-
+ {{end}}
-
+
+
+
+
+ +
From 2866436b36dbbf4a16b39089478715b0bba596ca Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 27 Sep 2022 11:13:02 +0800 Subject: [PATCH 02/32] fix-bug --- models/cloudbrain_static.go | 1 + routers/admin/cloudbrains.go | 4 ++-- routers/api/v1/repo/cloudbrain_dashboard.go | 18 +++++++++++++----- routers/repo/cloudbrain.go | 15 +++++++++++++-- routers/user/home.go | 5 ++--- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 58a93d476..185f64b77 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -34,6 +34,7 @@ type TaskDetail struct { CardDuration string `json:"CardDuration"` AiCenter string `json:"AiCenter"` FlavorName string `json:"FlavorName"` + WorkServerNum int64 `json:"WorkServerNum"` Spec *Specification `json:"Spec"` } diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index fcb878627..d04db032c 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -98,8 +98,8 @@ func CloudBrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) + // ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) + // ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 9d237d8e3..86c04a470 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -738,7 +738,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { tasks := []models.TaskDetail{} for i, task := range ciTasks { ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - var taskDetail models.TaskDetail taskDetail.ID = ciTasks[i].Cloudbrain.ID taskDetail.JobID = ciTasks[i].Cloudbrain.JobID @@ -758,11 +757,20 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias } - taskDetail.CardNum, taskDetail.CardType, _ = repo.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) + if ciTasks[i].Cloudbrain.WorkServerNumber >= 1 { + taskDetail.WorkServerNum = int64(ciTasks[i].Cloudbrain.WorkServerNumber) + } else { + taskDetail.WorkServerNum = 1 + } + // taskDetail.CardNum, taskDetail.CardType, _ = repo.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) + // if ciTasks[i].Cloudbrain.Spec != nil { + // taskDetail.CardNum = ciTasks[i].Cloudbrain.Spec.AccCardsNum + // taskDetail.CardType = ciTasks[i].Cloudbrain.Spec.AccCardType + // taskDetail.AiCenter = ciTasks[i].Cloudbrain.Spec.AiCenterName + // } + // taskDetail.AiCenter = repo.GetCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) + // taskDetail.FlavorName, _ = repo.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) - taskDetail.AiCenter = repo.GetCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) - taskDetail.FlavorName, _ = repo.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) - taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) if ciTasks[i].Cloudbrain.DeletedAt != nilTime || ciTasks[i].Repo == nil { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 73aa9b791..d28f8178b 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2841,8 +2841,19 @@ func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { return "" } func GetCloudbrainCardDuration(task models.Cloudbrain) string { - cardNum, _, _ := GetCloudbrainCardNumAndType(task) - cardDuration := models.ConvertDurationToStr(int64(cardNum) * task.Duration) + spec, err := resource.GetCloudbrainSpec(task.ID) + if err != nil { + log.Info("error:" + err.Error()) + return "" + } + cardNum := spec.AccCardsNum + var workServerNumber int64 + if task.WorkServerNumber >= 1 { + workServerNumber = int64(task.WorkServerNumber) + } else { + workServerNumber = 1 + } + cardDuration := models.ConvertDurationToStr(workServerNumber * int64(cardNum) * task.Duration) return cardDuration } func GetCloudbrainWaitTime(task models.Cloudbrain) string { diff --git a/routers/user/home.go b/routers/user/home.go index 78e6c00e9..fe13f7531 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -23,7 +23,6 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/routers/repo" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" @@ -841,8 +840,8 @@ func Cloudbrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) + // ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) + // ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } From d1601eade82dc9fa48030f28d3f8063a78fcd09b Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 28 Sep 2022 17:54:38 +0800 Subject: [PATCH 03/32] update --- models/cloudbrain_static.go | 31 ++- routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 318 ++++++++++++++++++---------- 3 files changed, 233 insertions(+), 117 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 185f64b77..48df111a0 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -45,16 +45,45 @@ func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) return x.SQL(countSql).Count() } func GetTodayCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count FROM " + + countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) return x.SQL(countSql).Count() } +func GetTodayRunningCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + " and (status='" + string(JobRunning) + "'" + + " or status='" + string(ModelArtsTrainJobInit) + "')" + return x.SQL(countSql).Count() +} +func GetTodayWaitingCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + " and status='" + string(JobWaiting) + "'" + return x.SQL(countSql).Count() +} + func GetCreatorCount() (int64, error) { countSql := "SELECT count(distinct user_id) FROM public.cloudbrain" return x.SQL(countSql).Count() } +func GetCloudbrainTypeCount() ([]map[string]string, error) { + countSql := "SELECT type,count(*) num FROM public.cloudbrain group by type order by num desc" + return x.QueryString(countSql) +} + +func GetCloudbrainStatusCount() ([]map[string]string, error) { + countSql := "SELECT status,count(*) num FROM public.cloudbrain group by status order by num desc" + return x.QueryString(countSql) +} + +func GetCloudbrainTpyeDurationSum() ([]map[string]string, error) { + countSql := "SELECT type,sum(duration) FROM public.cloudbrain group by type order by sum(duration) desc" + return x.QueryString(countSql) +} + func GetRecordBeginTime() ([]*CloudbrainInfo, error) { sess := x.NewSession() defer sess.Close() diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 0b941b400..e2743489a 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -577,6 +577,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/downloadAll", repo.DownloadCloudBrainBoard) m.Group("/cloudbrain", func() { m.Get("/overview", repo.GetAllCloudbrainsOverview) + m.Get("/overview_duration", repo.GetOverviewDuration) m.Get("/distribution", repo.GetAllCloudbrainsPeriodDistribution) m.Get("/trend", repo.GetAllCloudbrainsTrend) m.Get("/trend_detail_data", repo.GetAllCloudbrainsTrendDetail) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 86c04a470..3fcc10826 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -53,20 +53,140 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { log.Error("Can not query todayCreatorCount.", err) return } + cloudbrainTypeCount, err := models.GetCloudbrainTypeCount() + log.Info("cloudbrainTypeCount:", cloudbrainTypeCount) + if err != nil { + log.Error("Can not query cloudbrainTypeCount.", err) + return + } + + cloudbrainTpyeDurationSum, err := models.GetCloudbrainTpyeDurationSum() + log.Info("cloudbrainTpyeDurationSum:", cloudbrainTpyeDurationSum) + if err != nil { + log.Error("Can not query cloudbrainTpyeDurationSum.", err) + return + } + + todayCloudbrainCount, err := models.GetTodayCloudbrainCount(beginTime, endTime) + log.Info("todayCloudbrainCount:", todayCloudbrainCount) + if err != nil { + log.Error("Can not query todayCloudbrainCount.", err) + return + } + + todayRunningCount, err := models.GetTodayRunningCount(beginTime, endTime) + log.Info("todayRunningCount:", todayRunningCount) + if err != nil { + log.Error("Can not query todayRunningCount.", err) + return + } + + todayWaitingCount, err := models.GetTodayWaitingCount(beginTime, endTime) + log.Info("todayWaittingCount:", todayWaitingCount) + if err != nil { + log.Error("Can not query todayWaitingCount.", err) + return + } + + todayCompletedCount := todayCloudbrainCount - todayRunningCount - todayWaitingCount + log.Info("todayCompletedCount:", todayCompletedCount) + creatorCount, err := models.GetCreatorCount() if err != nil { log.Error("Can not query creatorCount.", err) return } - todayStatusResult := make(map[string]int) - cloudBrainNum := make(map[int]int) - cloudBrainOneDuration := int64(0) - cloudBrainTwoDuration := int64(0) - intelligentNetDuration := int64(0) - todayNewJobCount := 0 + // todayStatusResult := make(map[string]int) + // cloudBrainNum := make(map[int]int) + // cloudBrainOneDuration := int64(0) + // cloudBrainTwoDuration := int64(0) + // intelligentNetDuration := int64(0) + // todayNewJobCount := 0 + + // if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { + // todayNewJobCount += 1 + // if _, ok := todayStatusResult[cloudbrain.Status]; !ok { + // todayStatusResult[cloudbrain.Status] = 1 + // } else { + // todayStatusResult[cloudbrain.Status] += 1 + // } + // } + + // if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { + // cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 + // } else { + // cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 + // } + + // if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + // cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration + // } + // if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + // cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration + // } + // if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + // intelligentNetDuration = intelligentNetDuration + cloudbrain.Cloudbrain.Duration + // } + + // } + // count = len(cloudbrains) + // page += 1 + // } + + // statusNameList := []string{string(models.ModelArtsTrainJobCompleted), string(models.JobFailed), string(models.ModelArtsTrainJobInit), + // string(models.JobRunning), string(models.ModelArtsStartFailed), string(models.JobStopped), string(models.JobSucceeded), + // string(models.JobWaiting), string(models.ModelArtsTrainJobKilled)} + // for _, v := range statusNameList { + // if _, ok := todayStatusResult[v]; !ok { + // todayStatusResult[v] = 0 + // } + // } + + // cloudBrainTypeList := []int{0, 1, 2} + // for _, v := range cloudBrainTypeList { + // if _, ok := cloudBrainNum[v]; !ok { + // cloudBrainNum[v] = 0 + // } + // } + + // todayRunningCount := todayStatusResult[string(models.JobRunning)] + // todayCompletedCount := todayStatusResult[string(models.ModelArtsTrainJobCompleted)] + todayStatusResult[string(models.JobFailed)] + + // todayStatusResult[string(models.ModelArtsStartFailed)] + todayStatusResult[string(models.JobStopped)] + todayStatusResult[string(models.JobSucceeded)] + todayStatusResult[string(models.ModelArtsTrainJobKilled)] + // todayWaitingCount := todayStatusResult[string(models.ModelArtsTrainJobInit)] + todayStatusResult[string(models.JobWaiting)] + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "recordBeginTime": recordBeginTime, + "updateTime": now.Unix(), + // "cloudBrainNum": cloudBrainNum, + // "cloudBrainOneDuration": cloudBrainOneDuration, + // "cloudBrainTwoDuration": cloudBrainTwoDuration, + // "intelligentNetDuration": intelligentNetDuration, + "todayCreatorCount": todayCreatorCount, + "creatorCount": creatorCount, + "todayRunningCount": todayRunningCount, + "todayCompletedCount": todayCompletedCount, + "todayWaitingCount": todayWaitingCount, + "todayNewJobCount": todayCloudbrainCount, + "cloudbrainTypeCount": cloudbrainTypeCount, + }) +} +func GetOverviewDuration(ctx *context.Context) { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix + now := time.Now() + endTime := now page := 1 pagesize := 1000 count := pagesize + worker_server_num := 1 + cardNum := 1 + durationSum := int64(0) + durationSumAll := int64(0) for count == pagesize && count != 0 { cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -81,72 +201,30 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { ctx.ServerError("Get cloudbrains failed:", err) return } + models.LoadSpecs4CloudbrainInfo(cloudbrains) for _, cloudbrain := range cloudbrains { - if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { - todayNewJobCount += 1 - if _, ok := todayStatusResult[cloudbrain.Status]; !ok { - todayStatusResult[cloudbrain.Status] = 1 - } else { - todayStatusResult[cloudbrain.Status] += 1 - } - } - - if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { - cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 + duration := cloudbrain.Duration + if cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.WorkServerNumber } else { - cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 - } - - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration + worker_server_num = 1 } - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration - } - if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - intelligentNetDuration = intelligentNetDuration + cloudbrain.Cloudbrain.Duration + if cloudbrain.Spec != nil { + cardNum = cloudbrain.Spec.AccCardsNum + } else { + cardNum = 1 } - } - count = len(cloudbrains) - page += 1 - } - - statusNameList := []string{string(models.ModelArtsTrainJobCompleted), string(models.JobFailed), string(models.ModelArtsTrainJobInit), - string(models.JobRunning), string(models.ModelArtsStartFailed), string(models.JobStopped), string(models.JobSucceeded), - string(models.JobWaiting), string(models.ModelArtsTrainJobKilled)} - for _, v := range statusNameList { - if _, ok := todayStatusResult[v]; !ok { - todayStatusResult[v] = 0 - } - } - - cloudBrainTypeList := []int{0, 1, 2} - for _, v := range cloudBrainTypeList { - if _, ok := cloudBrainNum[v]; !ok { - cloudBrainNum[v] = 0 + durationSum = duration * int64(worker_server_num) * int64(cardNum) + durationSumAll = durationSum + durationSumAll + count = len(cloudbrains) + page += 1 } } - - todayRunningCount := todayStatusResult[string(models.JobRunning)] - todayCompletedCount := todayStatusResult[string(models.ModelArtsTrainJobCompleted)] + todayStatusResult[string(models.JobFailed)] + - todayStatusResult[string(models.ModelArtsStartFailed)] + todayStatusResult[string(models.JobStopped)] + todayStatusResult[string(models.JobSucceeded)] + todayStatusResult[string(models.ModelArtsTrainJobKilled)] - todayWaitingCount := todayStatusResult[string(models.ModelArtsTrainJobInit)] + todayStatusResult[string(models.JobWaiting)] - + log.Info("durationSumAll:", durationSumAll) ctx.JSON(http.StatusOK, map[string]interface{}{ - "recordBeginTime": recordBeginTime, - "updateTime": now.Unix(), - "cloudBrainNum": cloudBrainNum, - "cloudBrainOneDuration": cloudBrainOneDuration, - "cloudBrainTwoDuration": cloudBrainTwoDuration, - "intelligentNetDuration": intelligentNetDuration, - "todayCreatorCount": todayCreatorCount, - "creatorCount": creatorCount, - "todayRunningCount": todayRunningCount, - "todayCompletedCount": todayCompletedCount, - "todayWaitingCount": todayWaitingCount, - "todayNewJobCount": todayNewJobCount, + "durationSum": durationSum, }) } @@ -605,64 +683,72 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } func GetCloudbrainsStatusAnalysis(ctx *context.Context) { - recordCloudbrain, err := models.GetRecordBeginTime() + // recordCloudbrain, err := models.GetRecordBeginTime() + // if err != nil { + // log.Error("Can not get recordCloudbrain", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + // return + // } + // recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix + // endTime := time.Now() + // cloudBrainStatusResult := make(map[string]int) + // cloudBrainStatusAnalysis := make(map[string]int) + // totalCount := 0 + + // page := 1 + // pagesize := 1000 + // count := pagesize + // for count == pagesize && count != 0 { + // cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ + // ListOptions: models.ListOptions{ + // Page: page, + // PageSize: pagesize, + // }, + // Type: models.TypeCloudBrainAll, + // BeginTimeUnix: int64(recordBeginTime), + // EndTimeUnix: endTime.Unix(), + // }) + // if err != nil { + // ctx.ServerError("Get cloudbrains failed:", err) + // return + // } + // for _, cloudbrain := range cloudbrains { + // if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { + // cloudBrainStatusResult[cloudbrain.Status] = 1 + // } else { + // cloudBrainStatusResult[cloudbrain.Status] += 1 + // } + // } + // count = len(cloudbrains) + // totalCount = totalCount + count + // page += 1 + // } + + // var jobStatuses []string + // jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), + // string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), + // string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) + // jobStatusesCount := 0 + // for _, v := range jobStatuses { + // if _, ok := cloudBrainStatusResult[v]; !ok { + // cloudBrainStatusAnalysis[v] = 0 + // } else { + // cloudBrainStatusAnalysis[v] = cloudBrainStatusResult[v] + // } + // jobStatusesCount = jobStatusesCount + cloudBrainStatusResult[v] + // } + // cloudBrainStatusAnalysis["OTHER"] = totalCount - jobStatusesCount + + cloudbrainStatusCount, err := models.GetCloudbrainStatusCount() + log.Info("cloudbrainStatusCount:", cloudbrainStatusCount) if err != nil { - log.Error("Can not get recordCloudbrain", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + log.Error("Can not query cloudbrainStatusCount.", err) return } - recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix - endTime := time.Now() - cloudBrainStatusResult := make(map[string]int) - cloudBrainStatusAnalysis := make(map[string]int) - totalCount := 0 - - page := 1 - pagesize := 1000 - count := pagesize - for count == pagesize && count != 0 { - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: pagesize, - }, - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - for _, cloudbrain := range cloudbrains { - if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { - cloudBrainStatusResult[cloudbrain.Status] = 1 - } else { - cloudBrainStatusResult[cloudbrain.Status] += 1 - } - } - count = len(cloudbrains) - totalCount = totalCount + count - page += 1 - } - - var jobStatuses []string - jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), - string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), - string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) - jobStatusesCount := 0 - for _, v := range jobStatuses { - if _, ok := cloudBrainStatusResult[v]; !ok { - cloudBrainStatusAnalysis[v] = 0 - } else { - cloudBrainStatusAnalysis[v] = cloudBrainStatusResult[v] - } - jobStatusesCount = jobStatusesCount + cloudBrainStatusResult[v] - } - cloudBrainStatusAnalysis["OTHER"] = totalCount - jobStatusesCount ctx.JSON(http.StatusOK, map[string]interface{}{ - "cloudBrainStatusResult": cloudBrainStatusAnalysis, + // "cloudBrainStatusResult": cloudBrainStatusAnalysis, + "cloudbrainStatusCount": cloudbrainStatusCount, }) } From 9b4e39532261a0d1abeb1cd887280a07f99cefee Mon Sep 17 00:00:00 2001 From: chenshihai Date: Thu, 29 Sep 2022 14:20:35 +0800 Subject: [PATCH 04/32] =?UTF-8?q?#2981=20=E3=80=90=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E5=90=8E=E5=8F=B0=E3=80=91=E8=B5=84=E6=BA=90=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E9=A1=B5=E9=9D=A2=E5=A2=9E=E5=8A=A0=E6=A8=AA?= =?UTF-8?q?=E5=90=91=E6=BB=9A=E5=8A=A8=E6=9D=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web_src/vuepages/pages/resources/specification/index.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web_src/vuepages/pages/resources/specification/index.vue b/web_src/vuepages/pages/resources/specification/index.vue index 58d95d74f..17ca4870c 100644 --- a/web_src/vuepages/pages/resources/specification/index.vue +++ b/web_src/vuepages/pages/resources/specification/index.vue @@ -21,8 +21,8 @@
-
- +
+ From 1a4760384f234411f750053344164b4dcee444d8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sat, 8 Oct 2022 10:16:07 +0800 Subject: [PATCH 05/32] update kanban --- routers/api/v1/repo/cloudbrain_dashboard.go | 22 +++++++++++++++++----- routers/repo/cloudbrain.go | 7 ++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 3fcc10826..e14963639 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -187,6 +187,9 @@ func GetOverviewDuration(ctx *context.Context) { cardNum := 1 durationSum := int64(0) durationSumAll := int64(0) + cloudBrainOneSum := int64(0) + cloudBrainTwoSum := int64(0) + c2NetSum := int64(0) for count == pagesize && count != 0 { cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -204,7 +207,6 @@ func GetOverviewDuration(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(cloudbrains) for _, cloudbrain := range cloudbrains { - duration := cloudbrain.Duration if cloudbrain.WorkServerNumber >= 1 { worker_server_num = cloudbrain.WorkServerNumber } else { @@ -215,16 +217,26 @@ func GetOverviewDuration(ctx *context.Context) { } else { cardNum = 1 } - + duration := cloudbrain.Duration durationSum = duration * int64(worker_server_num) * int64(cardNum) - durationSumAll = durationSum + durationSumAll + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetSum += durationSum + } + + durationSumAll += durationSum count = len(cloudbrains) page += 1 } } - log.Info("durationSumAll:", durationSumAll) ctx.JSON(http.StatusOK, map[string]interface{}{ - "durationSum": durationSum, + "cloudBrainOneSum": cloudBrainOneSum, + "cloudBrainTwoSum": cloudBrainTwoSum, + "c2NetSum": c2NetSum, + "durationSumAll": durationSumAll, }) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d28f8178b..8051abdb2 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2841,12 +2841,17 @@ func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { return "" } func GetCloudbrainCardDuration(task models.Cloudbrain) string { + cardNum := int(0) spec, err := resource.GetCloudbrainSpec(task.ID) if err != nil { log.Info("error:" + err.Error()) return "" } - cardNum := spec.AccCardsNum + if spec != nil { + cardNum = spec.AccCardsNum + } else { + cardNum = 1 + } var workServerNumber int64 if task.WorkServerNumber >= 1 { workServerNumber = int64(task.WorkServerNumber) From 42a60d946c25684fd8281a1915e32e01d267e8c6 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Sat, 8 Oct 2022 10:48:38 +0800 Subject: [PATCH 06/32] fix-2940 --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/repo/cloudbrain.go | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b83fb6369..a20797df3 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1058,6 +1058,7 @@ modify_image=Modify Image image_exist=Image name has been used, please use a new one. image_committing=Image is submitting, please try again later. image_commit_fail=Failed to submit image, please try again later. +image_over_20g=Failed to submit image, the size of image can not be over 20GB. image_not_exist=Image does not exits. image_edit_fail=Failed to edit image, please try again later. image_delete_fail=Failed to delete image, please try again later. diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index c6f7e51fd..5a9d02dff 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1059,6 +1059,7 @@ modify_image=修改镜像 image_exist=镜像Tag已被使用，请修改镜像Tag。 image_committing=镜像正在提交中，请稍后再试。 image_commit_fail=提交镜像失败，请稍后再试。 +image_over_20g=提交镜像失败，镜像大小不能超过20GB。 image_not_exist=镜像不存在。 image_edit_fail=编辑镜像失败，请稍后再试。 image_delete_fail=删除镜像失败，请稍后再试。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 73aa9b791..9311ddeac 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1147,6 +1147,8 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain } else if models.IsErrorImageCommitting(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing"))) + } else if isOver20GError(err) { + ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_over_20g"))) } else { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail"))) } @@ -1156,6 +1158,10 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain ctx.JSON(200, models.BaseOKMessage) } +func isOver20GError(err error) bool { + return strings.Contains(err.Error(), "over max image size 20GB") +} + func checkTopics(Topics string) ([]string, string) { var topics = make([]string, 0) var topicsStr = strings.TrimSpace(Topics) From 3d5bb467ad82df91cf19a527b59d0fe660930fd6 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sat, 8 Oct 2022 11:01:17 +0800 Subject: [PATCH 07/32] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 34 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index e14963639..17e30f101 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -186,10 +186,18 @@ func GetOverviewDuration(ctx *context.Context) { worker_server_num := 1 cardNum := 1 durationSum := int64(0) - durationSumAll := int64(0) + cardDuSum := int64(0) + cloudBrainOneSum := int64(0) cloudBrainTwoSum := int64(0) c2NetSum := int64(0) + cDCenterSum := int64(0) + + cloudBrainOneDuration := int64(0) + cloudBrainTwoDuration := int64(0) + c2NetDuration := int64(0) + cDCenterDuration := int64(0) + for count == pagesize && count != 0 { cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -220,23 +228,37 @@ func GetOverviewDuration(ctx *context.Context) { duration := cloudbrain.Duration durationSum = duration * int64(worker_server_num) * int64(cardNum) if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration += duration cloudBrainOneSum += durationSum } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration cloudBrainTwoSum += durationSum } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration c2NetSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDCenterSum += durationSum } - durationSumAll += durationSum + durationSum += duration + cardDuSum += durationSum count = len(cloudbrains) page += 1 } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "cloudBrainOneSum": cloudBrainOneSum, - "cloudBrainTwoSum": cloudBrainTwoSum, - "c2NetSum": c2NetSum, - "durationSumAll": durationSumAll, + "cloudBrainOneCardDuSum": cloudBrainOneSum, + "cloudBrainTwoCardDuSum": cloudBrainTwoSum, + "c2NetCardDuSum": c2NetSum, + "cDNetCardDuSum": cDCenterSum, + "cardDuSum": cardDuSum, + + "cloudBrainOneDuration": cloudBrainOneDuration, + "cloudBrainTwoDuration": cloudBrainTwoDuration, + "c2NetDuration": c2NetDuration, + "cDCenterDuration": cDCenterDuration, + "durationSum": durationSum, }) } From 565f05319e99839deca4859399d20bc6b8f0b145 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sat, 8 Oct 2022 11:59:01 +0800 Subject: [PATCH 08/32] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 17e30f101..6c0f4d777 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -181,7 +181,7 @@ func GetOverviewDuration(ctx *context.Context) { now := time.Now() endTime := now page := 1 - pagesize := 1000 + pagesize := 10000 count := pagesize worker_server_num := 1 cardNum := 1 From 436f288c16b412b56128f29beda03279bbdfa82c Mon Sep 17 00:00:00 2001 From: liuzx Date: Sat, 8 Oct 2022 15:43:48 +0800 Subject: [PATCH 09/32] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 34 +++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 6c0f4d777..ae84e4c95 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -19,6 +19,7 @@ type DateCloudbrainNum struct { CloudOneJobTypeRes map[string]int `json:"cloudOneJobTypeRes"` CloudTwoJobTypeRes map[string]int `json:"cloudTwoJobTypeRes"` IntelligentNetJobTypeRes map[string]int `json:"intelligentNetJobTypeRes"` + CDCenterJobTypeRes map[string]int `json:"cDCenterJobTypeRes"` CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"` CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"` } @@ -612,6 +613,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { cloudOneJobTypeRes := make(map[string]int) cloudTwoJobTypeRes := make(map[string]int) intelligentNetJobTypeRes := make(map[string]int) + cDCenterJobTypeRes := make(map[string]int) cloudBrainPeriodNum := make(map[int]int) cloudBrainComputeResource := make(map[string]int) beginTimeTemp := beginTime.Unix() @@ -620,7 +622,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } page := 1 - pagesize := 1000 + pagesize := 10000 count := pagesize //Each time a maximum of 1000 pieces of data are detected to the memory, batch processing for count == pagesize && count != 0 { @@ -660,6 +662,13 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { intelligentNetJobTypeRes[cloudbrain.JobType] += 1 } } + if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + if _, ok := cDCenterJobTypeRes[cloudbrain.JobType]; !ok { + cDCenterJobTypeRes[cloudbrain.JobType] = 1 + } else { + cDCenterJobTypeRes[cloudbrain.JobType] += 1 + } + } if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 @@ -690,7 +699,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { intelligentNetJobTypeRes[v] = 0 } } - cloudBrainTypeList := []int{0, 1, 2} + cloudBrainTypeList := []int{0, 1, 2, 3} for _, v := range cloudBrainTypeList { if _, ok := cloudBrainPeriodNum[v]; !ok { cloudBrainPeriodNum[v] = 0 @@ -711,6 +720,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { "cloudOneJobTypeRes": cloudOneJobTypeRes, "cloudTwoJobTypeRes": cloudTwoJobTypeRes, "intelligentNetJobTypeRes": intelligentNetJobTypeRes, + "cDCenterJobTypeRes": cDCenterJobTypeRes, "cloudBrainPeriodNum": cloudBrainPeriodNum, "cloudBrainComputeResource": cloudBrainComputeResource, }) @@ -1086,11 +1096,12 @@ func GetRunningTop(ctx *context.Context) { }) } -func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*models.CloudbrainInfo) (map[string]int, map[string]int, map[string]int, map[int]int, map[string]int) { +func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*models.CloudbrainInfo) (map[string]int, map[string]int, map[string]int, map[string]int, map[int]int, map[string]int) { cloudOneJobTypeRes := make(map[string]int) cloudTwoJobTypeRes := make(map[string]int) intelligentNetJobTypeRes := make(map[string]int) + cDCenterJobTypeRes := make(map[string]int) cloudBrainPeriodNum := make(map[int]int) cloudBrainComputeResource := make(map[string]int) for _, cloudbrain := range cloudbrains { @@ -1133,7 +1144,7 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m } jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} - cloudBrainTypeList := []int{0, 1, 2} + cloudBrainTypeList := []int{0, 1, 2, 3} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -1144,6 +1155,9 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m if _, ok := intelligentNetJobTypeRes[v]; !ok { intelligentNetJobTypeRes[v] = 0 } + if _, ok := cDCenterJobTypeRes[v]; !ok { + cDCenterJobTypeRes[v] = 0 + } } for _, v := range cloudBrainTypeList { if _, ok := cloudBrainPeriodNum[v]; !ok { @@ -1151,7 +1165,7 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m } } cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] - return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource + return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource } func getDayCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { @@ -1168,12 +1182,13 @@ func getDayCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudbra } dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { - cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006/01/02"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CDCenterJobTypeRes: cDCenterJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) @@ -1203,12 +1218,13 @@ func getMonthCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudb return nil, err } for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { - cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) monthCloudbrainNum = append(monthCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006/01"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CDCenterJobTypeRes: cDCenterJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) @@ -1241,7 +1257,7 @@ func getDayCloudbrainInfo(beginTime time.Time, endTime time.Time) ([]DateCloudbr dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { - _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) + _, _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) dayCloudbrainInfo = append(dayCloudbrainInfo, DateCloudbrainInfo{ Date: endTimeTemp.Format("2006/01/02"), CloudBrainPeriodNum: cloudBrainPeriodNum, @@ -1272,7 +1288,7 @@ func getMonthCloudbrainInfo(beginTime time.Time, endTime time.Time) ([]DateCloud dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) || (endTimeTemp.Before(beginTime) && beginTime.Before(endTime)) { - _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) + _, _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) dayCloudbrainInfo = append(dayCloudbrainInfo, DateCloudbrainInfo{ Date: endTimeTemp.Format("2006/01"), CloudBrainPeriodNum: cloudBrainPeriodNum, From 4c74252b3a9e399a7d5b4c6a5b939915df4d010a Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Sat, 8 Oct 2022 15:57:32 +0800 Subject: [PATCH 10/32] fix issue --- templates/repo/modelarts/notebook/show.tmpl | 17 ++++++++++++++--- web_src/js/components/images/selectImages.vue | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index f3929ec60..0f632eab2 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -207,6 +207,18 @@ padding-left: 1rem; padding-top: 0.5rem; } + .dataset_nowrap_two_line{ + word-wrap: break-word; + word-break: break-all; + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + line-clamp: 2; + -webkit-line-clamp: 2; + text-overflow: -o-ellipsis-lastline; + max-height: 50px; + }
{{template "repo/header" .}} @@ -437,15 +449,14 @@ {{range $.datasetDownload}} - - + {{if eq .IsDelete true}} {{.DatasetName}}({{$.i18n.Tr "dataset.file_deleted"}}) {{else}} {{.DatasetName}} {{end}} - {{.DatasetDownloadLink}} + {{.DatasetDownloadLink}} {{$.i18n.Tr "dataset.download_copy"}} {{end}} diff --git a/web_src/js/components/images/selectImages.vue b/web_src/js/components/images/selectImages.vue index add4553d0..5bbad1a6b 100755 --- a/web_src/js/components/images/selectImages.vue +++ b/web_src/js/components/images/selectImages.vue @@ -47,6 +47,7 @@ type="text" :placeholder="i18n.image_search_placeholder" v-model="search" + @keydown.enter.stop.prevent="searchName" />
From 05ac77a6d95cbf39ae20b44f49b2f2fc011f82c8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sat, 8 Oct 2022 16:59:11 +0800 Subject: [PATCH 11/32] update --- models/cloudbrain.go | 3 +- routers/api/v1/repo/cloudbrain_dashboard.go | 43 +++++++++++++++-------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index bcaea544a..d088a9127 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2281,8 +2281,7 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", - "created_unix", "start_time", "end_time").Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index ae84e4c95..6dbdb01a8 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -186,13 +186,14 @@ func GetOverviewDuration(ctx *context.Context) { count := pagesize worker_server_num := 1 cardNum := 1 - durationSum := int64(0) + // durationSum := int64(0) + durationAllSum := int64(0) cardDuSum := int64(0) - cloudBrainOneSum := int64(0) - cloudBrainTwoSum := int64(0) - c2NetSum := int64(0) - cDCenterSum := int64(0) + cloudBrainOneCardDuSum := int64(0) + cloudBrainTwoCardDuSum := int64(0) + c2NetCardDuSum := int64(0) + cDNetCardDuSum := int64(0) cloudBrainOneDuration := int64(0) cloudBrainTwoDuration := int64(0) @@ -216,50 +217,50 @@ func GetOverviewDuration(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(cloudbrains) for _, cloudbrain := range cloudbrains { - if cloudbrain.WorkServerNumber >= 1 { - worker_server_num = cloudbrain.WorkServerNumber + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber } else { worker_server_num = 1 } - if cloudbrain.Spec != nil { - cardNum = cloudbrain.Spec.AccCardsNum - } else { + if cloudbrain.Cloudbrain.Spec == nil { cardNum = 1 + } else { + cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } duration := cloudbrain.Duration - durationSum = duration * int64(worker_server_num) * int64(cardNum) + durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { cloudBrainOneDuration += duration - cloudBrainOneSum += durationSum + cloudBrainOneCardDuSum += durationSum } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { cloudBrainTwoDuration += duration - cloudBrainTwoSum += durationSum + cloudBrainTwoCardDuSum += durationSum } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { c2NetDuration += duration - c2NetSum += durationSum + c2NetCardDuSum += durationSum } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { cDCenterDuration += duration - cDCenterSum += durationSum + cDNetCardDuSum += durationSum } - durationSum += duration + durationAllSum += duration cardDuSum += durationSum count = len(cloudbrains) page += 1 } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "cloudBrainOneCardDuSum": cloudBrainOneSum, - "cloudBrainTwoCardDuSum": cloudBrainTwoSum, - "c2NetCardDuSum": c2NetSum, - "cDNetCardDuSum": cDCenterSum, + "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, + "cloudBrainTwoCardDuSum": cloudBrainTwoCardDuSum, + "c2NetCardDuSum": c2NetCardDuSum, + "cDNetCardDuSum": cDNetCardDuSum, "cardDuSum": cardDuSum, "cloudBrainOneDuration": cloudBrainOneDuration, "cloudBrainTwoDuration": cloudBrainTwoDuration, "c2NetDuration": c2NetDuration, "cDCenterDuration": cDCenterDuration, - "durationSum": durationSum, + "durationSum": durationAllSum, }) } From 876adf2b7774e2d45a29a03037376ebd17343e57 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sat, 8 Oct 2022 17:46:42 +0800 Subject: [PATCH 12/32] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 6dbdb01a8..9d1e8363b 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1165,7 +1165,7 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m cloudBrainPeriodNum[v] = 0 } } - cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] + cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] + cloudBrainPeriodNum[3] return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource } From 26a1a08aca890af0acecc9e9fd215406c92f8a87 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sun, 9 Oct 2022 10:57:27 +0800 Subject: [PATCH 13/32] update --- routers/admin/cloudbrains.go | 10 ++- routers/api/v1/repo/cloudbrain_dashboard.go | 118 +++++++------------------- routers/repo/cloudbrain.go | 125 ++-------------------------- routers/user/home.go | 3 - 4 files changed, 43 insertions(+), 213 deletions(-) diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index d04db032c..96db935fe 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -98,8 +98,6 @@ func CloudBrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - // ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - // ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) @@ -179,6 +177,7 @@ func DownloadCloudBrains(ctx *context.Context) { log.Warn("Can not get cloud brain info", err) continue } + models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { for k, v := range allValues(row, record, ctx) { @@ -208,8 +207,11 @@ func allValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[str } func getCloudbrainCardType(rs *models.CloudbrainInfo) string { - _, cardType, _ := repo.GetCloudbrainCardNumAndType(rs.Cloudbrain) - return cardType + if rs.Cloudbrain.Spec != nil { + return rs.Cloudbrain.Spec.AccCardType + } else { + return "" + } } func getRepoPathName(rs *models.CloudbrainInfo) string { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 9d1e8363b..3e1cc2753 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -625,7 +625,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { page := 1 pagesize := 10000 count := pagesize - //Each time a maximum of 1000 pieces of data are detected to the memory, batch processing + //Each time a maximum of 10000 pieces of data are detected to the memory, batch processing for count == pagesize && count != 0 { cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -699,6 +699,9 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { if _, ok := intelligentNetJobTypeRes[v]; !ok { intelligentNetJobTypeRes[v] = 0 } + if _, ok := cDCenterJobTypeRes[v]; !ok { + cDCenterJobTypeRes[v] = 0 + } } cloudBrainTypeList := []int{0, 1, 2, 3} for _, v := range cloudBrainTypeList { @@ -714,9 +717,10 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } } - cloudOneJobTypeRes["EVALUATION"] = cloudOneJobTypeRes[string(models.JobTypeBenchmark)] + cloudOneJobTypeRes[string(models.JobTypeSnn4imagenet)] + cloudOneJobTypeRes[string(models.JobTypeBrainScore)] - cloudTwoJobTypeRes["EVALUATION"] = cloudTwoJobTypeRes[string(models.JobTypeBenchmark)] + cloudTwoJobTypeRes[string(models.JobTypeSnn4imagenet)] + cloudTwoJobTypeRes[string(models.JobTypeBrainScore)] - intelligentNetJobTypeRes["EVALUATION"] = intelligentNetJobTypeRes[string(models.JobTypeBenchmark)] + intelligentNetJobTypeRes[string(models.JobTypeSnn4imagenet)] + intelligentNetJobTypeRes[string(models.JobTypeBrainScore)] + cloudOneJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[0] - cloudOneJobTypeRes[string(models.JobTypeTrain)] - cloudOneJobTypeRes[string(models.JobTypeInference)] - cloudOneJobTypeRes[string(models.JobTypeDebug)] + cloudTwoJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[1] - cloudTwoJobTypeRes[string(models.JobTypeTrain)] - cloudTwoJobTypeRes[string(models.JobTypeInference)] - cloudTwoJobTypeRes[string(models.JobTypeDebug)] + intelligentNetJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[2] - intelligentNetJobTypeRes[string(models.JobTypeTrain)] - intelligentNetJobTypeRes[string(models.JobTypeInference)] - intelligentNetJobTypeRes[string(models.JobTypeDebug)] + cDCenterJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[3] - cDCenterJobTypeRes[string(models.JobTypeTrain)] - cDCenterJobTypeRes[string(models.JobTypeInference)] - cDCenterJobTypeRes[string(models.JobTypeDebug)] ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudOneJobTypeRes": cloudOneJobTypeRes, "cloudTwoJobTypeRes": cloudTwoJobTypeRes, @@ -728,62 +732,6 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } func GetCloudbrainsStatusAnalysis(ctx *context.Context) { - // recordCloudbrain, err := models.GetRecordBeginTime() - // if err != nil { - // log.Error("Can not get recordCloudbrain", err) - // ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - // return - // } - // recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix - // endTime := time.Now() - // cloudBrainStatusResult := make(map[string]int) - // cloudBrainStatusAnalysis := make(map[string]int) - // totalCount := 0 - - // page := 1 - // pagesize := 1000 - // count := pagesize - // for count == pagesize && count != 0 { - // cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - // ListOptions: models.ListOptions{ - // Page: page, - // PageSize: pagesize, - // }, - // Type: models.TypeCloudBrainAll, - // BeginTimeUnix: int64(recordBeginTime), - // EndTimeUnix: endTime.Unix(), - // }) - // if err != nil { - // ctx.ServerError("Get cloudbrains failed:", err) - // return - // } - // for _, cloudbrain := range cloudbrains { - // if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { - // cloudBrainStatusResult[cloudbrain.Status] = 1 - // } else { - // cloudBrainStatusResult[cloudbrain.Status] += 1 - // } - // } - // count = len(cloudbrains) - // totalCount = totalCount + count - // page += 1 - // } - - // var jobStatuses []string - // jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), - // string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), - // string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) - // jobStatusesCount := 0 - // for _, v := range jobStatuses { - // if _, ok := cloudBrainStatusResult[v]; !ok { - // cloudBrainStatusAnalysis[v] = 0 - // } else { - // cloudBrainStatusAnalysis[v] = cloudBrainStatusResult[v] - // } - // jobStatusesCount = jobStatusesCount + cloudBrainStatusResult[v] - // } - // cloudBrainStatusAnalysis["OTHER"] = totalCount - jobStatusesCount - cloudbrainStatusCount, err := models.GetCloudbrainStatusCount() log.Info("cloudbrainStatusCount:", cloudbrainStatusCount) if err != nil { @@ -792,7 +740,6 @@ func GetCloudbrainsStatusAnalysis(ctx *context.Context) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - // "cloudBrainStatusResult": cloudBrainStatusAnalysis, "cloudbrainStatusCount": cloudbrainStatusCount, }) } @@ -893,14 +840,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { } else { taskDetail.WorkServerNum = 1 } - // taskDetail.CardNum, taskDetail.CardType, _ = repo.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) - // if ciTasks[i].Cloudbrain.Spec != nil { - // taskDetail.CardNum = ciTasks[i].Cloudbrain.Spec.AccCardsNum - // taskDetail.CardType = ciTasks[i].Cloudbrain.Spec.AccCardType - // taskDetail.AiCenter = ciTasks[i].Cloudbrain.Spec.AiCenterName - // } - // taskDetail.AiCenter = repo.GetCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) - // taskDetail.FlavorName, _ = repo.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) @@ -926,17 +865,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { }) } -func getCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { - if task.Type == models.TypeCloudBrainOne { - return ctx.Tr("repo.cloudbrain1") - } else if task.Type == models.TypeCloudBrainTwo { - return ctx.Tr("repo.cloudbrain2") - } else if task.Type == models.TypeC2Net { - return task.AiCenter - } - return "" -} - func GetCloudbrainsCreateHoursData(ctx *context.Context) { recordCloudbrain, err := models.GetRecordBeginTime() if err != nil { @@ -1350,6 +1278,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { log.Warn("Can not get cloud brain info", err) continue } + models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { for k, v := range allCloudbrainValues(row, record, ctx) { @@ -1380,10 +1309,9 @@ func allCloudbrainHeader(ctx *context.Context) map[string]string { "H1": ctx.Tr("cloudbrain.card_duration"), "I1": ctx.Tr("repo.modelarts.train_job.start_time"), "J1": ctx.Tr("repo.modelarts.train_job.end_time"), "K1": ctx.Tr("repo.modelarts.computing_resources"), "L1": ctx.Tr("cloudbrain.card_type"), - "M1": ctx.Tr("repo.grampus.train_job.ai_center"), "N1": ctx.Tr("cloudbrain.resource_specification"), - "O1": ctx.Tr("repo.cloudbrain_creator"), "P1": ctx.Tr("repo.repo_name"), "Q1": ctx.Tr("repo.cloudbrain_task_name"), - "R1": ctx.Tr("repo.modelarts.deletetime")} - + "M1": ctx.Tr("repo.modelarts.train_job.amount_of_compute_node"), "N1": ctx.Tr("repo.grampus.train_job.ai_center"), + "O1": ctx.Tr("cloudbrain.resource_specification"), "P1": ctx.Tr("repo.cloudbrain_creator"), "Q1": ctx.Tr("repo.repo_name"), + "R1": ctx.Tr("repo.cloudbrain_task_name"), "S1": ctx.Tr("repo.modelarts.deletetime")} } func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): getCloudbrainType(rs, ctx), getCellName("C", row): rs.Status, getCellName("D", row): rs.JobType, @@ -1391,9 +1319,16 @@ func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Contex getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): repo.GetCloudbrainCardDuration(rs.Cloudbrain), getCellName("I", row): getBrainStartTime(rs), getCellName("J", row): getBrainEndTime(rs), getCellName("K", row): rs.ComputeResource, getCellName("L", row): getCloudbrainCardType(rs), - getCellName("M", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), getCellName("N", row): getCloudbrainFlavorName(rs), - getCellName("O", row): rs.Name, getCellName("P", row): getBrainRepo(rs), - getCellName("Q", row): rs.JobName, getCellName("R", row): getBrainDeleteTime(rs), + getCellName("M", row): getWorkServerNum(rs), getCellName("N", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), + getCellName("O", row): getCloudbrainFlavorName(rs), getCellName("P", row): rs.Name, + getCellName("Q", row): getBrainRepo(rs), getCellName("R", row): rs.JobName, getCellName("S", row): getBrainDeleteTime(rs), + } +} +func getWorkServerNum(rs *models.CloudbrainInfo) string { + if rs.Cloudbrain.WorkServerNumber >= 1 { + return fmt.Sprint(rs.Cloudbrain.WorkServerNumber) + } else { + return "1" } } func getBrainRepo(rs *models.CloudbrainInfo) string { @@ -1432,11 +1367,14 @@ func getCloudbrainType(rs *models.CloudbrainInfo, ctx *context.Context) string { } } func getCloudbrainCardType(rs *models.CloudbrainInfo) string { - _, cardType, _ := repo.GetCloudbrainCardNumAndType(rs.Cloudbrain) - return cardType + if rs.Cloudbrain.Spec != nil { + return rs.Cloudbrain.Spec.AccCardType + } else { + return "" + } } func getCloudbrainFlavorName(rs *models.CloudbrainInfo) string { - flavorName, _ := repo.GetCloudbrainFlavorName(rs.Cloudbrain) + flavorName := repo.GetCloudbrainFlavorName(rs.Cloudbrain) return flavorName } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 8051abdb2..c8567a2b3 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2813,16 +2813,11 @@ func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { } func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { - if task.Type == models.TypeCloudBrainOne { - return ctx.Tr("repo.cloudbrain1") - } else if task.Type == models.TypeCloudBrainTwo { - return ctx.Tr("repo.cloudbrain2") - } else if task.Type == models.TypeCDCenter { - return ctx.Tr("repo.cdCenter") - } else if task.Type == models.TypeC2Net { - return getCutStringAiCenterByAiCenter(task.AiCenter) + if task.Spec != nil { + return task.Spec.AiCenterName + } else { + return "" } - return "" } func getCutStringAiCenterByAiCenter(aiCenter string) string { if aiCenter == "" { @@ -2885,114 +2880,12 @@ func GetCloudbrainWaitTime(task models.Cloudbrain) string { } return waitTime } - -func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) { - if !models.SpecsMapInitFlag { - models.InitCloudbrainOneResourceSpecMap() - } - if !models.GpuInfosMapInitFlag { - models.InitCloudbrainOneGpuInfoMap() - } - flavorName, err := GetCloudbrainFlavorName(task) - if err != nil { - return 0, "", nil - } - return getCardNumAndTypeByFlavorname(flavorName) -} - -func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { - if FlavorName == "" { - return 0, "", nil - } else { - var beginIndex = strings.Index(FlavorName, ":") - var lastIndex = strings.LastIndex(FlavorName, ":") - var endIndex = strings.Index(FlavorName, "*") - if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) { - cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) - return 0, "", err - } - cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) - return cardNum, cardType, err - } - return 0, "", nil - } -} - -func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { - if task.Type == models.TypeCloudBrainOne { - resourceSpec, gpuInfo, err := getCloudBrainOneResourceSpec(task) - if err != nil { - log.Info("getCloudBrainOneResourceSpec err:", err) - return "", err - } else { - if resourceSpec == nil || gpuInfo == nil { - err := errors.New("resourceSpec or gpuInfo is nil") - return "", err - } else { - CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(resourceSpec.GpuNum) + "*Nvidia-" + gpuInfo.Value + - " | CPU:" + strconv.Itoa(resourceSpec.CpuNum) + "核" + strconv.Itoa(resourceSpec.MemMiB) + "MB" - return CloudbrainOneFlavorName, nil - } - } - } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net || task.Type == models.TypeCDCenter) && task.FlavorName != "" { - replaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") - return replaceFlavorName, nil - } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter) && task.FlavorName == "" && task.FlavorCode != "" { - cloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode) - return cloudbrainTwoFlavorName, nil - } else if task.Type == models.TypeCloudBrainTwo && task.JobType == string(models.JobTypeDebug) && task.FlavorName == "" && task.FlavorCode == "" { - tasks, err := models.GetModelartsReDebugTaskByJobId(task.JobID) - if err != nil { - return "", err - } - if len(tasks) >= 1 { - return getFlavorNameByFlavorCode(tasks[0].FlavorCode), nil - } - return "", nil - } - return "", nil -} - -func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, *models.GpuInfo, error) { - gpuQueueDefault := "openidebug" - if task.GpuQueue != "" { - gpuQueueDefault = task.GpuQueue - } - if task.ResourceSpecId >= 0 { - if task.JobType == string(models.JobTypeTrain) { - if models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId] != nil { - return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[gpuQueueDefault], nil - } else { - return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil - } - } else if task.JobType == string(models.JobTypeDebug) { - if models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId] != nil { - return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[gpuQueueDefault], nil - } else { - return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil - } - } else if task.JobType == string(models.JobTypeInference) { - return models.CloudbrainInferenceResourceSpecsMap[task.ResourceSpecId], models.CloudbrainInferenceGpuInfosMap[gpuQueueDefault], nil - } else if task.JobType == string(models.JobTypeBenchmark) || task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { - return models.CloudbrainBenchmarkResourceSpecsMap[task.ResourceSpecId], models.CloudbrainBenchmarkGpuInfosMap[gpuQueueDefault], nil - } +func GetCloudbrainFlavorName(task models.Cloudbrain) string { + if task.Spec != nil { + flavorName := task.Spec.ComputeResource + ":" + fmt.Sprint(task.Spec.AccCardsNum) + "*" + task.Spec.AccCardType + + ",内存:" + strconv.FormatFloat(float64(task.Spec.MemGiB), 'f', 6, 64) + "GB,共享内存:" + strconv.FormatFloat(float64(task.Spec.ShareMemGiB), 'f', 6, 64) + "GB" + return flavorName } else { - err := errors.New("ResourceSpecId is null") - return nil, nil, err - } - return nil, nil, nil -} -func getFlavorNameByFlavorCode(flavorCode string) string { - index := strings.LastIndex(flavorCode, ".") - cardNum, err := strconv.Atoi(strings.TrimSpace(flavorCode[index+1 : len(flavorCode)])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) return "" } - cloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) + - "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + - "核" + strconv.Itoa(cardNum*256) + "GB" - return cloudbrainTwoFlavorName } diff --git a/routers/user/home.go b/routers/user/home.go index fe13f7531..991c80328 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -840,9 +840,6 @@ func Cloudbrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - // ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - // ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) - } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) From c9a7ade0317eb94fe85247c9040cd8bfaca2cdf9 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sun, 9 Oct 2022 11:26:11 +0800 Subject: [PATCH 14/32] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 66 +---------------------------- 1 file changed, 2 insertions(+), 64 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 3e1cc2753..54c0ddc20 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -97,71 +97,10 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { log.Error("Can not query creatorCount.", err) return } - // todayStatusResult := make(map[string]int) - // cloudBrainNum := make(map[int]int) - // cloudBrainOneDuration := int64(0) - // cloudBrainTwoDuration := int64(0) - // intelligentNetDuration := int64(0) - // todayNewJobCount := 0 - - // if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { - // todayNewJobCount += 1 - // if _, ok := todayStatusResult[cloudbrain.Status]; !ok { - // todayStatusResult[cloudbrain.Status] = 1 - // } else { - // todayStatusResult[cloudbrain.Status] += 1 - // } - // } - - // if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { - // cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 - // } else { - // cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 - // } - - // if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - // cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration - // } - // if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - // cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration - // } - // if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - // intelligentNetDuration = intelligentNetDuration + cloudbrain.Cloudbrain.Duration - // } - - // } - // count = len(cloudbrains) - // page += 1 - // } - - // statusNameList := []string{string(models.ModelArtsTrainJobCompleted), string(models.JobFailed), string(models.ModelArtsTrainJobInit), - // string(models.JobRunning), string(models.ModelArtsStartFailed), string(models.JobStopped), string(models.JobSucceeded), - // string(models.JobWaiting), string(models.ModelArtsTrainJobKilled)} - // for _, v := range statusNameList { - // if _, ok := todayStatusResult[v]; !ok { - // todayStatusResult[v] = 0 - // } - // } - - // cloudBrainTypeList := []int{0, 1, 2} - // for _, v := range cloudBrainTypeList { - // if _, ok := cloudBrainNum[v]; !ok { - // cloudBrainNum[v] = 0 - // } - // } - - // todayRunningCount := todayStatusResult[string(models.JobRunning)] - // todayCompletedCount := todayStatusResult[string(models.ModelArtsTrainJobCompleted)] + todayStatusResult[string(models.JobFailed)] + - // todayStatusResult[string(models.ModelArtsStartFailed)] + todayStatusResult[string(models.JobStopped)] + todayStatusResult[string(models.JobSucceeded)] + todayStatusResult[string(models.ModelArtsTrainJobKilled)] - // todayWaitingCount := todayStatusResult[string(models.ModelArtsTrainJobInit)] + todayStatusResult[string(models.JobWaiting)] ctx.JSON(http.StatusOK, map[string]interface{}{ - "recordBeginTime": recordBeginTime, - "updateTime": now.Unix(), - // "cloudBrainNum": cloudBrainNum, - // "cloudBrainOneDuration": cloudBrainOneDuration, - // "cloudBrainTwoDuration": cloudBrainTwoDuration, - // "intelligentNetDuration": intelligentNetDuration, + "recordBeginTime": recordBeginTime, + "updateTime": now.Unix(), "todayCreatorCount": todayCreatorCount, "creatorCount": creatorCount, "todayRunningCount": todayRunningCount, @@ -186,7 +125,6 @@ func GetOverviewDuration(ctx *context.Context) { count := pagesize worker_server_num := 1 cardNum := 1 - // durationSum := int64(0) durationAllSum := int64(0) cardDuSum := int64(0) From c9a0a6eec69ef5f75f57be9c3aeebfb3c0d302ec Mon Sep 17 00:00:00 2001 From: liuzx Date: Sun, 9 Oct 2022 11:38:46 +0800 Subject: [PATCH 15/32] update --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 336efdd38..17f94b8c2 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2889,7 +2889,7 @@ func GetCloudbrainWaitTime(task models.Cloudbrain) string { func GetCloudbrainFlavorName(task models.Cloudbrain) string { if task.Spec != nil { flavorName := task.Spec.ComputeResource + ":" + fmt.Sprint(task.Spec.AccCardsNum) + "*" + task.Spec.AccCardType + - ",内存:" + strconv.FormatFloat(float64(task.Spec.MemGiB), 'f', 6, 64) + "GB,共享内存:" + strconv.FormatFloat(float64(task.Spec.ShareMemGiB), 'f', 6, 64) + "GB" + ",内存:" + strconv.FormatInt(int(task.Spec.MemGiB), 'f', 6, 64) + "GB,共享内存:" + strconv.FormatInt(int(task.Spec.ShareMemGiB), 'f', 6, 64) + "GB" return flavorName } else { return "" From 7c728a5b280b26d47707adff884c9f38a23e56a7 Mon Sep 17 00:00:00 2001 From: liuzx Date: Sun, 9 Oct 2022 11:43:29 +0800 Subject: [PATCH 16/32] update --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 17f94b8c2..74f6924f4 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2889,7 +2889,7 @@ func GetCloudbrainWaitTime(task models.Cloudbrain) string { func GetCloudbrainFlavorName(task models.Cloudbrain) string { if task.Spec != nil { flavorName := task.Spec.ComputeResource + ":" + fmt.Sprint(task.Spec.AccCardsNum) + "*" + task.Spec.AccCardType + - ",内存:" + strconv.FormatInt(int(task.Spec.MemGiB), 'f', 6, 64) + "GB,共享内存:" + strconv.FormatInt(int(task.Spec.ShareMemGiB), 'f', 6, 64) + "GB" + ",内存:" + strconv.FormatInt(int64(task.Spec.MemGiB), 10) + "GB,共享内存:" + strconv.FormatInt(int64(task.Spec.ShareMemGiB), 10) + "GB" return flavorName } else { return "" From 65825643894d92a91edae6ef8b9f1febf94e2209 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Sun, 9 Oct 2022 14:03:53 +0800 Subject: [PATCH 17/32] update --- web_src/vuepages/pages/resources/specification/index.vue | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/web_src/vuepages/pages/resources/specification/index.vue b/web_src/vuepages/pages/resources/specification/index.vue index 17ca4870c..4dc2657f3 100644 --- a/web_src/vuepages/pages/resources/specification/index.vue +++ b/web_src/vuepages/pages/resources/specification/index.vue @@ -22,15 +22,15 @@
- - - + + + - - - Date: Sun, 9 Oct 2022 14:45:48 +0800 Subject: [PATCH 18/32] fix-2845 --- routers/repo/modelarts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 5487231a2..f70d84208 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -985,7 +985,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) //isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath - codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" // dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" @@ -1108,8 +1108,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - // parentDir := VersionOutputPath + "/" - if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + parentDir := VersionOutputPath + "/" + if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) From b4d7fab6820cbc77892515b856a3146bafb5b150 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 11 Oct 2022 10:55:50 +0800 Subject: [PATCH 19/32] #2910 update --- models/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo.go b/models/repo.go index 6e5ee3ff6..2c4fda39b 100755 --- a/models/repo.go +++ b/models/repo.go @@ -679,7 +679,7 @@ func (repo *Repository) getAssignees(e Engine) (_ []*User, err error) { userIDs[i] = accesses[i].UserID } - if err = e.In("id", userIDs).Find(&users); err != nil { + if err = e.In("id", userIDs).OrderBy("name asc").Find(&users); err != nil { return nil, err } } From 0abf9d25659f23a36bcd7f0fcff93ecb13f5b886 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 11 Oct 2022 17:34:23 +0800 Subject: [PATCH 20/32] #2188 fix bug --- routers/repo/editor.go | 6 +++--- routers/repo/setting_protected_branch.go | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 40edc4767..b350343db 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -303,7 +303,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo } if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { ctx.Redirect(ctx.Repo.RepoLink + "/src/branch/" + util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(form.TreePath)) } @@ -475,7 +475,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { ctx.Flash.Success(ctx.Tr("repo.editor.file_delete_success", ctx.Repo.TreePath)) if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { treePath := filepath.Dir(ctx.Repo.TreePath) if treePath == "." { @@ -686,7 +686,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { ctx.Redirect(ctx.Repo.RepoLink + "/src/branch/" + util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(form.TreePath)) } diff --git a/routers/repo/setting_protected_branch.go b/routers/repo/setting_protected_branch.go index ab0fd77ee..f1ea17528 100644 --- a/routers/repo/setting_protected_branch.go +++ b/routers/repo/setting_protected_branch.go @@ -5,6 +5,7 @@ package repo import ( + "code.gitea.io/gitea/modules/util" "fmt" "strings" "time" @@ -192,7 +193,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) } if f.RequiredApprovals < 0 { ctx.Flash.Error(ctx.Tr("repo.settings.protected_branch_required_approvals_min")) - ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, branch)) + ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, util.PathEscapeSegments(branch))) } var whitelistUsers, whitelistTeams, mergeWhitelistUsers, mergeWhitelistTeams, approvalsWhitelistUsers, approvalsWhitelistTeams []int64 @@ -263,7 +264,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) return } ctx.Flash.Success(ctx.Tr("repo.settings.update_protect_branch_success", branch)) - ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, branch)) + ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, util.PathEscapeSegments(branch))) } else { if protectBranch != nil { if err := ctx.Repo.Repository.DeleteProtectedBranch(protectBranch.ID); err != nil { From 518e47e96816c8ea41ba1217c644d42a516f0f55 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 11 Oct 2022 19:58:14 +0800 Subject: [PATCH 21/32] code opt --- models/cloudbrain.go | 2 ++ modules/grampus/grampus.go | 11 +++++++++++ routers/repo/grampus.go | 11 ++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index d088a9127..6797fe332 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1490,6 +1490,8 @@ type GrampusTasks struct { ReplicaNum int `json:"replicaNum"` Datasets []GrampusDataset `json:"datasets"` Models []GrampusDataset `json:"models"` + Code GrampusDataset `json:"code"` + BootFile string `json:"bootFile"` } type GrampusDataset struct { diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index d72a7b10e..7bacb46d3 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -1,6 +1,7 @@ package grampus import ( + "code.gitea.io/gitea/modules/cloudbrain" "encoding/json" "strings" @@ -73,6 +74,7 @@ type GenerateTrainJobReq struct { PreTrainModelPath string PreTrainModelUrl string Spec *models.Specification + CodeName string } func getEndPoint() string { @@ -102,6 +104,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error centerID, centerName := getCentersParamter(ctx, req) var datasetGrampus, modelGrampus []models.GrampusDataset + var codeGrampus models.GrampusDataset if ProcessorTypeNPU == req.ProcessType { datasetGrampus = getDatasetGrampus(req.DatasetInfos) if len(req.ModelName) != 0 { @@ -114,6 +117,12 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error }, } } + codeGrampus = models.GrampusDataset{ + Name: req.CodeName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ObjectKey: req.CodeObsPath + cloudbrain.DefaultBranchName + ".zip", + } } jobResult, err := createJob(models.CreateGrampusJobRequest{ @@ -130,6 +139,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error ReplicaNum: 1, Datasets: datasetGrampus, Models: modelGrampus, + Code: codeGrampus, + BootFile: req.BootFile, }, }, }) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index abf64281a..bc61b0b55 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -713,6 +713,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain DatasetNames: datasetNames, DatasetInfos: datasetInfos, Spec: spec, + CodeName: strings.ToLower(repo.Name), } if form.ModelName != "" { //使用预训练模型训练 req.ModelName = form.ModelName @@ -976,8 +977,8 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" - command += commandDownload + //commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" + //command += commandDownload } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) @@ -986,8 +987,8 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //unzip code & dataset if processorType == grampus.ProcessorTypeNPU { - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" - command += commandUnzip + //commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" + //command += commandUnzip } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand @@ -1024,7 +1025,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo var commandCode string if processorType == grampus.ProcessorTypeNPU { - commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" + commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py /tmp/log/train.log" + paramCode + ";" } else if processorType == grampus.ProcessorTypeGPU { if pretrainModelFileName != "" { paramCode += " --ckpt_url" + "=" + workDir + "pretrainmodel/" + pretrainModelFileName From a49e4fb9ca756819f626164d5a26b62455eceee3 Mon Sep 17 00:00:00 2001 From: yanchao Date: Wed, 12 Oct 2022 09:19:33 +0800 Subject: [PATCH 22/32] =?UTF-8?q?=E5=B1=8F=E8=94=BDswagger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d6e4824ef..15f2d53d7 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -394,7 +394,7 @@ var ( DefaultGitTreesPerPage int DefaultMaxBlobSize int64 }{ - EnableSwagger: true, + EnableSwagger: false, SwaggerURL: "", MaxResponseItems: 50, DefaultPagingNum: 30, From 4c0ae28fc06a902792627fc2369366f09e41ed7f Mon Sep 17 00:00:00 2001 From: yanchao Date: Wed, 12 Oct 2022 11:01:34 +0800 Subject: [PATCH 23/32] =?UTF-8?q?xss=E6=94=BB=E5=87=BB=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/repository/create.go | 3 ++- routers/api/v1/repo/fork.go | 3 ++- routers/repo/repo.go | 3 ++- routers/repo/setting.go | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/repository/create.go b/modules/repository/create.go index 0844c43c3..049fe3abf 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "strings" + "text/template" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" @@ -34,7 +35,7 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *m Name: opts.Name, Alias: opts.Alias, LowerName: strings.ToLower(opts.Name), - Description: opts.Description, + Description: template.HTMLEscapeString(opts.Description), OriginalURL: opts.OriginalURL, OriginalServiceType: opts.GitServiceType, IsPrivate: opts.IsPrivate, diff --git a/routers/api/v1/repo/fork.go b/routers/api/v1/repo/fork.go index a753f192d..9416035e7 100644 --- a/routers/api/v1/repo/fork.go +++ b/routers/api/v1/repo/fork.go @@ -7,6 +7,7 @@ package repo import ( "fmt" + "html/template" "net/http" "code.gitea.io/gitea/models" @@ -118,7 +119,7 @@ func CreateFork(ctx *context.APIContext, form api.CreateForkOption) { forker = org } - fork, err := repo_service.ForkRepository(ctx.User, forker, repo, repo.Name, repo.Description, repo.Alias) + fork, err := repo_service.ForkRepository(ctx.User, forker, repo, repo.Name, template.HTMLEscapeString(repo.Description), repo.Alias) if err != nil { ctx.Error(http.StatusInternalServerError, "ForkRepository", err) return diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 03d2d832a..2c8c2f45b 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -12,6 +12,7 @@ import ( "path" "regexp" "strings" + "text/template" "unicode/utf8" "code.gitea.io/gitea/modules/validation" @@ -212,7 +213,7 @@ func CreatePost(ctx *context.Context, form auth.CreateRepoForm) { opts := models.GenerateRepoOptions{ Name: form.RepoName, Alias: form.Alias, - Description: form.Description, + Description: template.HTMLEscapeString(form.Description), Private: form.Private, GitContent: form.GitContent, Topics: form.Topics, diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 11efdf275..5fcfb2287 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -8,6 +8,7 @@ package repo import ( "errors" "fmt" + "html/template" "io/ioutil" "net/url" "regexp" @@ -129,7 +130,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { // In case it's just a case change. repo.Name = newRepoName repo.LowerName = strings.ToLower(newRepoName) - repo.Description = form.Description + repo.Description = template.HTMLEscapeString(form.Description) repo.Website = form.Website repo.IsTemplate = form.Template repo.Alias = newAlias From 6fb0289bba0ace02fbf580abab384c3a6c4418cf Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 12 Oct 2022 11:20:20 +0800 Subject: [PATCH 24/32] =?UTF-8?q?#2995=20=E6=96=B0=E5=BB=BA=E4=BA=91?= =?UTF-8?q?=E8=84=91=E4=BB=BB=E5=8A=A1=E9=A1=B5=E9=9D=A2=E7=9A=84=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E6=8F=90=E7=A4=BA=E4=B8=AD=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8Fxxx=5Furl=E6=94=B9=E4=B8=BA=E8=BF=90=E8=A1=8C=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 12 ++++++------ options/locale/locale_zh-CN.ini | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 19d604062..d93353063 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3210,11 +3210,11 @@ wrong_specification=You cannot use this specification, please choose another ite resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). -train_dataset_path_rule = The dataset location is stored in the environment variable data_url, the pre-trained model is storaged in the environment ckpt_url, and the output path is stored in the environment variable train_url. -infer_dataset_path_rule = The dataset location is stored in the environment variable data_url, and the output path is stored in the environment variable result_url. +train_dataset_path_rule = The dataset location is stored in the run parameter data_url, the pre-trained model is storaged in the run parameter ckpt_url, and the output path is stored in the run parameter train_url. +infer_dataset_path_rule = The dataset location is stored in the run parameter data_url, and the output path is stored in the run parameter result_url. view_sample = View sample -inference_output_path_rule = The inference output path is stored in the environment variable result_url. -model_file_path_rule=The model file location is stored in the environment variable ckpt_url +inference_output_path_rule = The inference output path is stored in the run parameter result_url. +model_file_path_rule=The model file location is stored in the run parameter ckpt_url model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] delete_task = Delete task @@ -3247,8 +3247,8 @@ Stopped_success_update_status_fail=Succeed in stopping th job, but failed to upd load_code_failed=Fail to load code, please check if the right branch is selected. error.dataset_select = dataset select error：the count exceed the limit or has same name -new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online -new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online +new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online +new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online new_infer_gpu_tooltips = The dataset is stored in %s, the model file is stored in %s, please store the inference output in %s for subsequent downloads. [points] diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 9fa7f5595..32f854449 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3228,11 +3228,11 @@ card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 -train_dataset_path_rule = 数据集位置存储在环境变量data_url中，预训练模型存放在环境变量ckpt_url中，训练输出路径存储在环境变量train_url中。 -infer_dataset_path_rule = 数据集位置存储在环境变量data_url中，推理输出路径存储在环境变量result_url中。 +train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。 +infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。 view_sample = 查看样例 -inference_output_path_rule = 推理输出路径存储在环境变量result_url中。 -model_file_path_rule = 模型文件位置存储在环境变量ckpt_url中。 +inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。 +model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。 model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] delete_task = 删除任务 @@ -3266,9 +3266,9 @@ load_code_failed=代码加载失败，请确认选择了正确的分支。 error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 -new_train_gpu_tooltips =训练脚本存储在%s中，数据集存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 -new_train_npu_tooltips =训练脚本存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 -new_infer_gpu_tooltips = 数据集存储在%s中，模型文件存储在%s中，推理输出请存储在%s中以供后续下载。 +new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 +new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 +new_infer_gpu_tooltips = 数据集存储在 %s 中，模型文件存储在 %s 中，推理输出请存储在 %s 中以供后续下载。 [points] points = 积分 From 59c37f10fe79e759c6faa862295205840679e427 Mon Sep 17 00:00:00 2001 From: yanchao Date: Wed, 12 Oct 2022 11:28:27 +0800 Subject: [PATCH 25/32] =?UTF-8?q?=E5=86=85=E9=83=A8IP=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/ai_model_manage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 957028fc6..6aec541e3 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -639,6 +639,7 @@ func ShowSingleModel(ctx *context.Context) { if value != nil { model.UserName = value.Name model.UserRelAvatarLink = value.RelAvatarLink() + model.TrainTaskInfo = "" } } ctx.JSON(http.StatusOK, models) @@ -787,6 +788,7 @@ func ShowModelPageInfo(ctx *context.Context) { for i, model := range modelResult { model.IsCanOper = isOper(ctx, model.UserId) model.IsCanDelete = isCanDelete(ctx, model.UserId) + model.TrainTaskInfo = "" userIds[i] = model.UserId } From d3fff0d9c114157fb48dbb228844991827b93a35 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 12 Oct 2022 15:15:51 +0800 Subject: [PATCH 26/32] =?UTF-8?q?=E5=86=85=E9=83=A8IP=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 2 +- routers/repo/grampus.go | 1 + routers/repo/modelarts.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 73aa9b791..16cd94729 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -760,8 +760,8 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State + task.ContainerIp = "" task.ContainerID = taskRes.TaskStatuses[0].ContainerID - task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) if task.DeletedAt.IsZero() { //normal record diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index abf64281a..18bc02567 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -837,6 +837,7 @@ func GrampusTrainJobShow(ctx *context.Context) { ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } + task.ContainerIp = "" if task.DeletedAt.IsZero() { //normal record result, err := grampus.GetJob(task.JobID) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 5487231a2..6a8ba2037 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1795,7 +1795,7 @@ func TrainJobShow(ctx *context.Context) { datasetList = append(datasetList, GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)) VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) - + VersionListTasks[i].ContainerIp = "" //add spec s, err := resource.GetCloudbrainSpec(task.Cloudbrain.ID) if err != nil { From 96ba0efa0e579b7a51014f940f5f4f7f5ebe417f Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 13 Oct 2022 09:15:15 +0800 Subject: [PATCH 27/32] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/repo/cloudbrain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 439190121..7324b6466 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -577,7 +577,6 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s r := bufio.NewReader(reader) for i := 0; i < endLine; i++ { line, error := r.ReadString('\n') - log.Info("line=" + line) if error == io.EOF { log.Info("read file completed.") break From fd2699373419ce14aaae5402a8d6fdf612726906 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 13 Oct 2022 09:16:50 +0800 Subject: [PATCH 28/32] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E4=B8=ADIP=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 6aec541e3..5b358b83b 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "regexp" "strings" "code.gitea.io/gitea/models" @@ -85,6 +86,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio accuracy["Precision"] = "" accuracyJson, _ := json.Marshal(accuracy) log.Info("accuracyJson=" + string(accuracyJson)) + aiTask.ContainerIp = "" aiTaskJson, _ := json.Marshal(aiTask) model := &models.AiModelManage{ @@ -635,16 +637,23 @@ func ShowSingleModel(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range models { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name model.UserRelAvatarLink = value.RelAvatarLink() - model.TrainTaskInfo = "" } } ctx.JSON(http.StatusOK, models) } +func removeIpInfo(model *models.AiModelManage) { + reg, _ := regexp.Compile(`[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}`) + taskInfo := model.TrainTaskInfo + taskInfo = reg.ReplaceAllString(taskInfo, "") + model.TrainTaskInfo = taskInfo +} + func queryUserName(intSlice []int64) map[int64]*models.User { keys := make(map[int64]string) uniqueElements := []int64{} @@ -678,6 +687,7 @@ func ShowOneVersionOtherModel(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range aimodels { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -788,13 +798,13 @@ func ShowModelPageInfo(ctx *context.Context) { for i, model := range modelResult { model.IsCanOper = isOper(ctx, model.UserId) model.IsCanDelete = isCanDelete(ctx, model.UserId) - model.TrainTaskInfo = "" userIds[i] = model.UserId } userNameMap := queryUserName(userIds) for _, model := range modelResult { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -868,6 +878,7 @@ func QueryModelListForPredict(ctx *context.Context) { nameMap := make(map[string][]*models.AiModelManage) for _, model := range modelResult { + removeIpInfo(model) if _, value := nameMap[model.Name]; !value { models := make([]*models.AiModelManage, 0) models = append(models, model) From cf8172d95ac69c76fe8f0f6d6a96beab2c430210 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 13 Oct 2022 11:49:09 +0800 Subject: [PATCH 29/32] fix issue --- templates/repo/cloudbrain/trainjob/show.tmpl | 2 +- templates/repo/grampus/trainjob/show.tmpl | 2 +- templates/repo/modelarts/trainjob/show.tmpl | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 3c71140bf..21e26571b 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -253,7 +253,7 @@
{{if and ($.canDownload) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 45b6501fd..67548696f 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -265,7 +265,7 @@
{{if and ($.canDownload) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}},EngineName:{{.EngineName}},ComputeResource:{{.ComputeResource}},Type:{{.Type}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 0c9bb3ba4..1cdb595dd 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -249,7 +249,7 @@ {{$.CsrfTokenHtml}} {{if and (.CanModify) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}},EngineID:{{.EngineID}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} @@ -783,7 +783,6 @@ } let dirKey="isOnlyDir--:&"; function loadSelectedModelFile(trainJob){ - console.log("trainJob=" + trainJob); $('#choice_file').dropdown('clear') $("#model-file").empty() if(trainJob ==null || trainJob ==""){ From 6ebaae60aa5076f36e4a539820d5c075c79dc12a Mon Sep 17 00:00:00 2001 From: chenshihai Date: Thu, 13 Oct 2022 14:57:22 +0800 Subject: [PATCH 30/32] =?UTF-8?q?#2991=20=E4=B8=AA=E4=BA=BA=E4=B8=AD?= =?UTF-8?q?=E5=BF=83-=E4=BA=91=E8=84=91=E4=BB=BB=E5=8A=A1=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E9=9B=86=E7=BE=A4=E3=80=81=E6=99=BA=E7=AE=97=E4=B8=AD?= =?UTF-8?q?=E5=BF=83=E5=AD=97=E6=AE=B5=E6=9C=89=E6=97=B6=E5=80=99=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + templates/admin/cloudbrain/list.tmpl | 75 +++++++++++++++++++++------- templates/user/dashboard/cloudbrains.tmpl | 81 +++++++++++++++++++++++-------- 4 files changed, 121 insertions(+), 37 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 19d604062..ac4903fc0 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3234,6 +3234,7 @@ point_hr = Point/hr DEBUG = DEBUG SNN4IMAGENET = BENCHMARK BRAINSCORE = BENCHMARK +MODELSAFETY = BENCHMARK TRAIN = TRAIN INFERENCE = INFERENCE BENCHMARK = BENCHMARK diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 9fa7f5595..60cb43582 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3252,6 +3252,7 @@ point_hr = 积分/时 DEBUG = 调试任务 SNN4IMAGENET = 评测任务 BRAINSCORE = 评测任务 +MODELSAFETY = 评测任务 TRAIN = 训练任务 INFERENCE = 推理任务 BENCHMARK = 评测任务 diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index 89649a307..4c500b5e6 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -89,7 +89,7 @@
{{$JobID := '0'}} - {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}} + {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}} {{$JobID = .Cloudbrain.ID}} {{else}} {{$JobID = .JobID}} @@ -110,6 +110,13 @@ {{.DisplayJobName}} + {{else if eq .JobType "MODELSAFETY"}} + + {{.DisplayJobName}} + {{else if eq .JobType "INFERENCE"}}
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} + {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -185,6 +190,17 @@ var spanEl = document.querySelector('.card_type_{{.DisplayJobName}}_{{$JobID}}'); spanEl.setAttribute('title', cardType); spanEl.innerText = cardType; + + var cluster = spec.Cluster || '--'; + var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--'; + spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', cluster); + spanEl.innerText = clusterName; + + var aiCenter = spec.AiCenterName || '--'; + spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', aiCenter); + spanEl.innerText = aiCenter; })(); @@ -239,8 +255,19 @@
{{end}} - +
+ {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.stop"}} + +
+ {{else}} {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
{{$.CsrfTokenHtml}} @@ -259,6 +286,7 @@ {{$.i18n.Tr "repo.stop"}} {{end}} + {{end}}
{{if eq .JobType "TRAIN"}} @@ -269,17 +297,30 @@
{{end}} - - {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.delete"}} - - + {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{else}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{end}}
diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl index defed27bc..d91c783fc 100755 --- a/templates/user/dashboard/cloudbrains.tmpl +++ b/templates/user/dashboard/cloudbrains.tmpl @@ -71,7 +71,7 @@
{{$JobID := '0'}} - {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}} + {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}} {{$JobID = .Cloudbrain.ID}} {{else}} {{$JobID = .JobID}} @@ -92,6 +92,13 @@ {{.DisplayJobName}} + {{else if eq .JobType "MODELSAFETY"}} + + {{.DisplayJobName}} + {{else if eq .JobType "INFERENCE"}}
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} + {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -169,6 +174,17 @@ var spanEl = document.querySelector('.card_type_{{.DisplayJobName}}_{{$JobID}}'); spanEl.setAttribute('title', cardType); spanEl.innerText = cardType; + + var cluster = spec.Cluster || '--'; + var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--'; + spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', cluster); + spanEl.innerText = clusterName; + + var aiCenter = spec.AiCenterName || '--'; + spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', aiCenter); + spanEl.innerText = aiCenter; })(); @@ -202,23 +218,35 @@ {{end}}
- {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}} -
- {{$.CsrfTokenHtml}} + {{if eq .JobType "MODELSAFETY"}} + + {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.stop"}} + +
+ {{else}} + {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.stop"}} + +
+ {{else}} + class='ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "STOPPED" "SUCCEEDED" "CREATE_FAILED"}}disabled {{else}} blue {{end}}button' + data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 1}}modelarts/inference-job{{else}}cloudbrain/train-job{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}modelarts/train-job{{else if eq .Cloudbrain.Type 0}}cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}grampus/train-job{{end}}{{end}}' + data-jobid="{{$JobID}}" data-version="{{.VersionName}}"> {{$.i18n.Tr "repo.stop"}} - - {{else}} - - {{$.i18n.Tr "repo.stop"}} - + {{end}} {{end}}
{{if eq .JobType "BENCHMARK"}} @@ -228,7 +256,6 @@ {{$.i18n.Tr "repo.score"}}
- {{end}} {{if eq .JobType "TRAIN"}} @@ -239,6 +266,18 @@
{{end}} + {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{else}}
@@ -250,6 +289,8 @@ {{$.i18n.Tr "repo.delete"}}
+ {{end}} +
From c571d0d627e374465344ee9b80fddf0e51ad08eb Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 17 Oct 2022 09:24:15 +0800 Subject: [PATCH 31/32] bug fix --- routers/admin/resources.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/admin/resources.go b/routers/admin/resources.go index 8a8c55f86..1d3870a14 100644 --- a/routers/admin/resources.go +++ b/routers/admin/resources.go @@ -221,6 +221,7 @@ func GetResourceSceneList(ctx *context.Context) { func AddResourceScene(ctx *context.Context, req models.ResourceSceneReq) { req.CreatorId = ctx.User.ID + req.ExclusiveOrg = strings.ReplaceAll(req.ExclusiveOrg, " ", "") err := resource.AddResourceScene(req) if err != nil { log.Error("AddResourceScene error. %v", err) @@ -238,6 +239,7 @@ func UpdateResourceScene(ctx *context.Context, req models.ResourceSceneReq) { var err error switch action { case "edit": + req.ExclusiveOrg = strings.ReplaceAll(req.ExclusiveOrg, " ", "") err = resource.UpdateResourceScene(req) case "delete": err = resource.DeleteResourceScene(id) From cfda54463ea4fa65cdb45c544a0093f8e32e9602 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 17 Oct 2022 17:06:50 +0800 Subject: [PATCH 32/32] del unneed code --- routers/repo/grampus.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 63176184e..b78bdebd3 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -978,8 +978,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - //commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" - //command += commandDownload + //no need to download code & dataset by internet } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) @@ -988,8 +987,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //unzip code & dataset if processorType == grampus.ProcessorTypeNPU { - //commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" - //command += commandUnzip + //no need to process } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand