From 36402724026517361a106eaabf31d45c587e0c59 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 13 Oct 2022 14:42:52 +0800 Subject: [PATCH 01/50] update --- models/cloudbrain_static.go | 55 ++++++++++ models/models.go | 1 + modules/cron/tasks_basic.go | 19 +++- routers/repo/cloudbrain_statistic.go | 188 +++++++++++++++++++++++++++++++++++ routers/routes/routes.go | 8 +- 5 files changed, 265 insertions(+), 6 deletions(-) create mode 100644 routers/repo/cloudbrain_statistic.go diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 48df111a0..bd2bbaef8 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,6 +38,25 @@ type TaskDetail struct { Spec *Specification `json:"Spec"` } +type CloudbrainDurationStatistic struct { + ID int64 `xorm:"pk autoincr"` + Cluster string `xorm:"notnull"` + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + QueueCode string + CardsTotalNum int + + DateTime string + HourTime int + CardsTotalDuration int + + DeletedTime timeutil.TimeStamp `xorm:"deleted"` + CreatedTime timeutil.TimeStamp `xorm:"created"` + UpdatedTime timeutil.TimeStamp `xorm:"updated"` +} + func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -199,3 +218,39 @@ func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string } return dateHourMap, nil } + +func GetCloudbrainRunning() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobRunning)}, + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func GetCloudbrainCompleteByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}), + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { + return xStatistic.Insert(cloudbrainDurationStatistic) +} diff --git a/models/models.go b/models/models.go index 4c2079cd8..7665c3b0f 100755 --- a/models/models.go +++ b/models/models.go @@ -179,6 +179,7 @@ func init() { new(UserMetrics), new(UserAnalysisPara), new(Invitation), + new(CloudbrainDurationStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 04cd7fe41..1fe10cb2c 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,12 +5,13 @@ package cron import ( - "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/services/cloudbrain/resource" - "code.gitea.io/gitea/modules/modelarts" "context" "time" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/migrations" repository_service "code.gitea.io/gitea/modules/repository" @@ -254,6 +255,17 @@ func registerSyncModelArtsTempJobs() { }) } +func registerHandleCloudbrainDurationStatistic() { + RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 60m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.CloudbrainDurationStatistic() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -271,6 +283,7 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + registerHandleCloudbrainDurationStatistic() registerSyncCloudbrainStatus() registerHandleOrgStatistic() diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go new file mode 100644 index 000000000..1aa943932 --- /dev/null +++ b/routers/repo/cloudbrain_statistic.go @@ -0,0 +1,188 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +func CloudbrainDurationStatistic() { + log.Info("Generate Cloudbrain Duration statistic begin") + // yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + // CloudbrainDurationStatisticHour(yesterday) + log.Info("Generate Cloudbrain Duration statistic end") +} + +func CloudbrainDurationStatisticHour(ctx *context.Context) { + //获取规定时间段的云脑任务列表 + // page := ctx.QueryInt("page") + // pageSize := ctx.QueryInt("pagesize") + // if page <= 0 { + // page = 1 + // } + // if pageSize <= 0 { + // pageSize = 10 + // } + + // cloudBrainDurationRes := make(map[string]map[string]int) + // cloudBrainOneCardRes := make(map[string]int) + // cloudBrainTwoCardRes := make(map[string]int) + // c2NetCardRes := make(map[string]int) + // cDNetCenterCardRes := make(map[string]int) + // var WorkServerNumber int + // var AccCardsNum int + // endTime := time.Now().Unix() + // beginTime := time.Now().AddDate(0, 0, -1).Unix() + + // hour := time.Now().Hour() + // tStr := time.Now().Format("2006-01-02 15:04:05") + + currentTime := time.Now() + m, _ := time.ParseDuration("-1h") + beginTime := currentTime.Add(m).Unix() + endTime := currentTime.Unix() + // fmt.Println(beginTime) + + ciTasks1, err := models.GetCloudbrainRunning() + if err != nil { + // ctx.ServerError("Get job failed:", err) + log.Info("GetCloudbrainRunning err: %v", err) + return + } + ciTasks2, err := models.GetCloudbrainCompleteByTime(beginTime, endTime) + ciTasks := append(ciTasks1, ciTasks2...) + log.Info("beginTime: %s", beginTime) + log.Info("endTime: %s", endTime) + if err != nil { + // ctx.ServerError("Get job failed:", err) + log.Info("GetCloudbrainCompleteByTime err: %v", err) + return + } + models.LoadSpecs4CloudbrainInfo(ciTasks) + log.Info("ciTasks here: %s", ciTasks) + log.Info("count here: %s", len(ciTasks)) + cloudBrainCardRes := getCloudBrainCardRes(ciTasks, beginTime, endTime) + + //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 + + // cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + // DateTime: date, + // HourTime: userNumber, + // Cluster: repositorySize, + // AiCenterName: allDatasetSize, + // AiCenterCode: organizationNumber, + // ComputeResource: repositoryNumer, + // AccCardType: forkRepositoryNumber, + // CardsTotalNum: mirrorRepositoryNumber, + // CardsTotalDuration: privateRepositoryNumer, + // QueueCode: publicRepositoryNumer, + // CreatedTime: privateRepositoryNumer, + // UpdatedTime: publicRepositoryNumer, + // } + + // if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + // log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + // } + // log.Info("cloudBrainDurationRes2: %s", cloudBrainDurationRes) + // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) + // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) + // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) + log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) + // log.Info("c2NetCardRes: %s", c2NetCardRes) + log.Info("finish summary cloudbrainDurationStat") +} + +func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { + var WorkServerNumber int + var AccCardsNum int + // cloudBrainCardRes := make(map[string]int) + cloudBrainAicenterNameList := make(map[string]string) + cloudBrainCardTypeList := make(map[string]string) + cloudBrainCenterNameAndCardType := make(map[string]map[string]int) + // var cloudbrainDurationInfo models.CloudbrainDurationInfo + for _, cloudbrain := range ciTasks { + + if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { + cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName + } + + if cloudbrain.Cloudbrain.StartTime == 0 { + cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix + } + if cloudbrain.Cloudbrain.EndTime == 0 { + cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix + } + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber + } else { + WorkServerNumber = 1 + } + if cloudbrain.Cloudbrain.Spec == nil { + AccCardsNum = 1 + } else { + AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + for k, _ := range cloudBrainAicenterNameList { + if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { + if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType + } + for i, _ := range cloudBrainCardTypeList { + if cloudBrainCenterNameAndCardType == nil { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = 0 + } + if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { + if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } + } + } + // cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = cloudBrainCardRes[cloudBrainCardTypeList[i]] + + // if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } else { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } + // } else { + // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } else { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } + // } + } + } + } + + return cloudBrainCenterNameAndCardType +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 66a357c79..e52686ee4 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,15 +6,16 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - "code.gitea.io/gitea/services/reward" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/modules/slideimage" "code.gitea.io/gitea/routers/image" @@ -374,6 +375,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/^:type(issues|pulls)$", reqSignIn, user.Issues) m.Get("/milestones", reqSignIn, reqMilestonesDashboardPageEnabled, user.Milestones) m.Get("/cloudbrains", reqSignIn, user.Cloudbrains) + m.Get("/duration", repo.CloudbrainDurationStatisticHour) // ***** START: User ***** m.Group("/user", func() { From ebd4c78e290e1919b89bbea3214978713b9d3e3f Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 13 Oct 2022 15:28:25 +0800 Subject: [PATCH 02/50] update --- routers/repo/cloudbrain_statistic.go | 39 +++--------------------------------- 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 1aa943932..550e290d0 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -101,9 +101,7 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end cloudBrainAicenterNameList := make(map[string]string) cloudBrainCardTypeList := make(map[string]string) cloudBrainCenterNameAndCardType := make(map[string]map[string]int) - // var cloudbrainDurationInfo models.CloudbrainDurationInfo for _, cloudbrain := range ciTasks { - if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName } @@ -125,14 +123,14 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } for k, _ := range cloudBrainAicenterNameList { + if cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] == nil { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] = make(map[string]int) + } if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType } for i, _ := range cloudBrainCardTypeList { - if cloudBrainCenterNameAndCardType == nil { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = 0 - } if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { @@ -149,37 +147,6 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end } } } - // cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = cloudBrainCardRes[cloudBrainCardTypeList[i]] - - // if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { - // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } else { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } - // } else { - // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } else { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } - // } } } } From d3bc16efa423a4e048cbd0aace4e59c0bbb9a496 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 13 Oct 2022 16:01:38 +0800 Subject: [PATCH 03/50] update --- models/cloudbrain_static.go | 19 ++++++++++++ routers/repo/cloudbrain_statistic.go | 56 +++++++++++++++++++++++++----------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index bd2bbaef8..b760fdc31 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -251,6 +251,25 @@ func GetCloudbrainCompleteByTime(beginTime int64, endTime int64) ([]*CloudbrainI return cloudbrains, nil } +func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*CloudbrainSpec, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, + ) + cond = cond.And( + builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}, + ) + sess.OrderBy("cloudbrain_spec.created_unix ASC limit 1") + cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10) + if err := sess.Table(&CloudbrainSpec{}).Where(cond). + Find(&cloudbrainSpecs); err != nil { + log.Info("find error.") + } + return cloudbrainSpecs, nil +} + func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { return xStatistic.Insert(cloudbrainDurationStatistic) } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 550e290d0..0bc4938ff 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -63,7 +63,31 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(ciTasks) log.Info("ciTasks here: %s", ciTasks) log.Info("count here: %s", len(ciTasks)) - cloudBrainCardRes := getCloudBrainCardRes(ciTasks, beginTime, endTime) + cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime) + for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo { + for cardType, cardDuration := range CardTypeInfo { + cloudbrain, err := models.GetSpecByAiCenterCodeAndType(centerCode, cardType) + if err != nil { + log.Info("GetSpecByAiCenterCodeAndType err: %v", err) + return + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: date, + HourTime: userNumber, + Cluster: cloudbrain[0].Cluster, + AiCenterName: cloudbrain[0].AiCenterName, + AiCenterCode: centerCode, + ComputeResource: cloudbrain[0].ComputeResource, + AccCardType: cardType, + CardsTotalNum: mirrorRepositoryNumber, + CardsTotalDuration: cardDuration, + QueueCode: cloudbrain[0].QueueCode, + CreatedTime: privateRepositoryNumer, + UpdatedTime: publicRepositoryNumer, + } + } + + } //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 @@ -89,21 +113,21 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) - log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) + // log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) // log.Info("c2NetCardRes: %s", c2NetCardRes) log.Info("finish summary cloudbrainDurationStat") } -func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { +func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { var WorkServerNumber int var AccCardsNum int // cloudBrainCardRes := make(map[string]int) - cloudBrainAicenterNameList := make(map[string]string) + cloudBrainAiCenterCodeList := make(map[string]string) cloudBrainCardTypeList := make(map[string]string) - cloudBrainCenterNameAndCardType := make(map[string]map[string]int) + cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) for _, cloudbrain := range ciTasks { - if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { - cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName + if _, ok := cloudBrainAiCenterCodeList[cloudbrain.Cloudbrain.Spec.AiCenterCode]; !ok { + cloudBrainAiCenterCodeList[cloudbrain.Cloudbrain.Spec.AiCenterCode] = cloudbrain.Cloudbrain.Spec.AiCenterCode } if cloudbrain.Cloudbrain.StartTime == 0 { @@ -122,11 +146,11 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end } else { AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } - for k, _ := range cloudBrainAicenterNameList { - if cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] == nil { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] = make(map[string]int) + for k, _ := range cloudBrainAiCenterCodeList { + if cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]] == nil { + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]] = make(map[string]int) } - if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { + if cloudbrain.Cloudbrain.Spec.AiCenterCode == cloudBrainAiCenterCodeList[k] { if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType } @@ -134,15 +158,15 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) } else { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) } } else { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) } else { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) } } } @@ -151,5 +175,5 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end } } - return cloudBrainCenterNameAndCardType + return cloudBrainCenterCodeAndCardType } From a8341fcb425841b4babac0087c4794dfea024d1b Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 14 Oct 2022 17:51:05 +0800 Subject: [PATCH 04/50] update --- models/cloudbrain_spec.go | 4 +- models/cloudbrain_static.go | 69 ++++++++++++++-- routers/api/v1/api.go | 4 + routers/api/v1/repo/cloudbrain_dashboard.go | 81 +++++++++++++++++- routers/repo/cloudbrain_statistic.go | 123 ++++++++++++++-------------- 5 files changed, 208 insertions(+), 73 deletions(-) diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go index c32e4b0fd..49a4d603e 100644 --- a/models/cloudbrain_spec.go +++ b/models/cloudbrain_spec.go @@ -9,7 +9,7 @@ type CloudbrainSpec struct { SpecId int64 `xorm:"index"` SourceSpecId string AccCardsNum int - AccCardType string + AccCardType string `xorm:"index"` CpuCores int MemGiB float32 GPUMemGiB float32 @@ -19,7 +19,7 @@ type CloudbrainSpec struct { QueueId int64 QueueCode string Cluster string - AiCenterCode string + AiCenterCode string `xorm:"index"` AiCenterName string IsExclusive bool ExclusiveOrg string diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index b760fdc31..b8f1c8e62 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "strconv" "time" @@ -39,18 +40,20 @@ type TaskDetail struct { } type CloudbrainDurationStatistic struct { - ID int64 `xorm:"pk autoincr"` - Cluster string `xorm:"notnull"` + ID int64 `xorm:"pk autoincr"` + Cluster string AiCenterCode string AiCenterName string ComputeResource string AccCardType string - QueueCode string - CardsTotalNum int + TotalUse bool + TotalCanUse bool DateTime string + DayTime string HourTime int CardsTotalDuration int + CardsTotalNum int DeletedTime timeutil.TimeStamp `xorm:"deleted"` CreatedTime timeutil.TimeStamp `xorm:"created"` @@ -256,12 +259,8 @@ func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*C defer sess.Close() var cond = builder.NewCond() cond = cond.And( - builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, + builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}), ) - cond = cond.And( - builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}, - ) - sess.OrderBy("cloudbrain_spec.created_unix ASC limit 1") cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10) if err := sess.Table(&CloudbrainSpec{}).Where(cond). Find(&cloudbrainSpecs); err != nil { @@ -273,3 +272,55 @@ func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*C func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { return xStatistic.Insert(cloudbrainDurationStatistic) } + +func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, tatalUse bool, totalCanUse bool) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_use = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, tatalUse, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func GetCanUseCardInfo() ([]*ResourceQueue, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), + ) + ResourceQueues := make([]*ResourceQueue, 0, 10) + if err := sess.Table(&ResourceQueue{}).Where(cond). + Find(&ResourceQueues); err != nil { + log.Info("find error.") + } + return ResourceQueues, nil +} +func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse bool, totalCanUse bool) ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain_duration_statistic.created_time": beginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_time": endTime.Unix()}), + ) + cond = cond.And( + builder.And(builder.Eq{"cloudbrain_duration_statistic.total_use": totalUse}, builder.Eq{"cloudbrain_duration_statistic.total_can_use": totalCanUse}), + ) + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10) + if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond). + Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e1d725ed..06be12e92 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -599,6 +599,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData) m.Get("/waitting_top_data", repo.GetWaittingTop) m.Get("/running_top_data", repo.GetRunningTop) + + m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) + m.Get("/resource_usage", repo.GetCloudbrainResourceUsage) + m.Get("/resource_usage_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 54c0ddc20..7d348d578 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -545,7 +545,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0) beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) if err != nil { - log.Error("Parameter is wrong", err) + log.Error("getCloudbrainTimePeroid error:", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } @@ -1403,3 +1403,82 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t return beginTime, endTime, nil } + +func GetCloudbrainResourceOverview(ctx *context.Context) { + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "resourceQueues": resourceQueues, + }) + +} + +func GetCloudbrainResourceUsage(ctx *context.Context) { + recordBeginTime := time.Now().AddDate(0, 0, -6) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("getCloudbrainTimePeroid error:", err) + return + } + cardUsageRes := make(map[string]int) + cardCanUsageRes := make(map[string]int) + cardUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, true, false) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + cardCanUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, false, true) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + + for _, cloudbrainStat := range cardUseInfo { + if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } + } + + for _, cloudbrainStat := range cardCanUseInfo { + if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cardUseInfo": cardUseInfo, + "cardCanUseInfo": cardCanUseInfo, + "cardUsageRes": cardUsageRes, + "cardCanUsageRes": cardCanUsageRes, + }) + +} +func GetCloudbrainResourceUsageDetail(ctx *context.Context) { + recordBeginTime := time.Now().AddDate(0, 0, -6) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("getCloudbrainTimePeroid error:", err) + return + } + totalUse := true + totalCanUse := false + cardDurationStatisticsInfo, err := models.GetCardDurationStatistics(beginTime, endTime, totalUse, totalCanUse) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cardDurationStatisticsInfo": cardDurationStatisticsInfo, + "beginTime": beginTime, + "endTime": endTime, + }) + +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 0bc4938ff..c7546c1c6 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -6,6 +6,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" ) func CloudbrainDurationStatistic() { @@ -16,28 +17,9 @@ func CloudbrainDurationStatistic() { } func CloudbrainDurationStatisticHour(ctx *context.Context) { - //获取规定时间段的云脑任务列表 - // page := ctx.QueryInt("page") - // pageSize := ctx.QueryInt("pagesize") - // if page <= 0 { - // page = 1 - // } - // if pageSize <= 0 { - // pageSize = 10 - // } - - // cloudBrainDurationRes := make(map[string]map[string]int) - // cloudBrainOneCardRes := make(map[string]int) - // cloudBrainTwoCardRes := make(map[string]int) - // c2NetCardRes := make(map[string]int) - // cDNetCenterCardRes := make(map[string]int) - // var WorkServerNumber int - // var AccCardsNum int - // endTime := time.Now().Unix() - // beginTime := time.Now().AddDate(0, 0, -1).Unix() - - // hour := time.Now().Hour() - // tStr := time.Now().Format("2006-01-02 15:04:05") + hourTime := time.Now().Hour() + dateTime := time.Now().Format("2006-01-02 15:04:05") + dayTime := time.Now().Format("2006-01-02") currentTime := time.Now() m, _ := time.ParseDuration("-1h") @@ -71,50 +53,69 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { log.Info("GetSpecByAiCenterCodeAndType err: %v", err) return } - cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - DateTime: date, - HourTime: userNumber, - Cluster: cloudbrain[0].Cluster, - AiCenterName: cloudbrain[0].AiCenterName, - AiCenterCode: centerCode, - ComputeResource: cloudbrain[0].ComputeResource, - AccCardType: cardType, - CardsTotalNum: mirrorRepositoryNumber, - CardsTotalDuration: cardDuration, - QueueCode: cloudbrain[0].QueueCode, - CreatedTime: privateRepositoryNumer, - UpdatedTime: publicRepositoryNumer, + log.Info("cloudbrain: %s", cloudbrain) + if cloudbrain != nil { + totalUse := true + totalCanUse := false + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalUse, totalCanUse); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: cloudbrain[0].Cluster, + AiCenterName: cloudbrain[0].AiCenterName, + AiCenterCode: centerCode, + ComputeResource: cloudbrain[0].ComputeResource, + AccCardType: cardType, + CardsTotalDuration: cardDuration, + CreatedTime: timeutil.TimeStampNow(), + TotalUse: true, + TotalCanUse: false, + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } } } } - //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 - - // cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - // DateTime: date, - // HourTime: userNumber, - // Cluster: repositorySize, - // AiCenterName: allDatasetSize, - // AiCenterCode: organizationNumber, - // ComputeResource: repositoryNumer, - // AccCardType: forkRepositoryNumber, - // CardsTotalNum: mirrorRepositoryNumber, - // CardsTotalDuration: privateRepositoryNumer, - // QueueCode: publicRepositoryNumer, - // CreatedTime: privateRepositoryNumer, - // UpdatedTime: publicRepositoryNumer, - // } - - // if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { - // log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) - // } - // log.Info("cloudBrainDurationRes2: %s", cloudBrainDurationRes) - // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) - // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) - // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) - // log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) - // log.Info("c2NetCardRes: %s", c2NetCardRes) + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + log.Info("resourceQueues here: %s", resourceQueues) + for _, resourceQueue := range resourceQueues { + totalUse := false + totalCanUse := true + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalUse, totalCanUse); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cardsTotalDuration := resourceQueue.CardsTotalNum * 1 * 60 * 60 + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: resourceQueue.Cluster, + AiCenterName: resourceQueue.AiCenterName, + AiCenterCode: resourceQueue.AiCenterCode, + ComputeResource: resourceQueue.ComputeResource, + AccCardType: resourceQueue.AccCardType, + CardsTotalDuration: cardsTotalDuration, + CardsTotalNum: resourceQueue.CardsTotalNum, + CreatedTime: timeutil.TimeStampNow(), + TotalUse: false, + TotalCanUse: true, + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + } log.Info("finish summary cloudbrainDurationStat") } From 52d69f1287693f5b010b1a83150a0018d2286bcd Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 17 Oct 2022 18:02:14 +0800 Subject: [PATCH 05/50] update --- models/cloudbrain_static.go | 53 +++-- modules/cron/tasks_basic.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 301 ++++++++++++++++++++++++---- routers/repo/cloudbrain_statistic.go | 25 +-- 4 files changed, 313 insertions(+), 68 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index b8f1c8e62..ae9358dc1 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -46,7 +46,6 @@ type CloudbrainDurationStatistic struct { AiCenterName string ComputeResource string AccCardType string - TotalUse bool TotalCanUse bool DateTime string @@ -55,9 +54,20 @@ type CloudbrainDurationStatistic struct { CardsTotalDuration int CardsTotalNum int - DeletedTime timeutil.TimeStamp `xorm:"deleted"` - CreatedTime timeutil.TimeStamp `xorm:"created"` - UpdatedTime timeutil.TimeStamp `xorm:"updated"` + DeletedUnix timeutil.TimeStamp `xorm:"deleted"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` +} +type DurationStatisticOptions struct { + BeginTime time.Time + EndTime time.Time + AiCenterCode string +} +type DateCloudbrainStatistic struct { + Date string `json:"date"` + AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` + AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` + AiCenterUsageRate map[string]int `json:"aiCenterUsageRate"` } func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { @@ -273,14 +283,14 @@ func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDu return xStatistic.Insert(cloudbrainDurationStatistic) } -func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, tatalUse bool, totalCanUse bool) error { +func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, totalCanUse bool) error { sess := xStatistic.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { return fmt.Errorf("Begin: %v", err) } - if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_use = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, tatalUse, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { + if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { return fmt.Errorf("Delete: %v", err) } @@ -307,16 +317,21 @@ func GetCanUseCardInfo() ([]*ResourceQueue, error) { } return ResourceQueues, nil } -func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse bool, totalCanUse bool) ([]*CloudbrainDurationStatistic, error) { + +func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { sess := xStatistic.NewSession() defer sess.Close() var cond = builder.NewCond() - cond = cond.And( - builder.And(builder.Gte{"cloudbrain_duration_statistic.created_time": beginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_time": endTime.Unix()}), - ) - cond = cond.And( - builder.And(builder.Eq{"cloudbrain_duration_statistic.total_use": totalUse}, builder.Eq{"cloudbrain_duration_statistic.total_can_use": totalCanUse}), - ) + if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 { + cond = cond.And( + builder.And(builder.Gte{"cloudbrain_duration_statistic.created_unix": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_unix": opts.EndTime.Unix()}), + ) + } + if opts.AiCenterCode != "" { + cond = cond.And( + builder.Eq{"cloudbrain_duration_statistic.ai_center_code": opts.AiCenterCode}, + ) + } CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10) if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond). Find(&CloudbrainDurationStatistics); err != nil { @@ -324,3 +339,15 @@ func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse } return CloudbrainDurationStatistics, nil } + +func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1") + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0) + if err := sess.Table(&CloudbrainDurationStatistic{}).Unscoped(). + Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 1fe10cb2c..b68d747d8 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -261,7 +261,7 @@ func registerHandleCloudbrainDurationStatistic() { RunAtStart: false, Schedule: "@every 60m", }, func(ctx context.Context, _ *models.User, _ Config) error { - repo.CloudbrainDurationStatistic() + repo.CloudbrainDurationStatisticHour() return nil }) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 7d348d578..caac24aad 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -532,6 +532,21 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int } +func getPageDateCloudbrainDuration(dateCloudbrainDuration []models.DateCloudbrainStatistic, page int, pagesize int) []models.DateCloudbrainStatistic { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(dateCloudbrainDuration)-1 { + return nil + } + if end > len(dateCloudbrainDuration)-1 { + return dateCloudbrainDuration[begin:] + } else { + return dateCloudbrainDuration[begin:end] + } + +} + func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { queryType := ctx.QueryTrim("type") beginTimeStr := ctx.QueryTrim("beginTime") @@ -1426,59 +1441,277 @@ func GetCloudbrainResourceUsage(ctx *context.Context) { } cardUsageRes := make(map[string]int) cardCanUsageRes := make(map[string]int) - cardUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, true, false) - if err != nil { - log.Error("GetCardDurationStatistics error:", err) - return - } - cardCanUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, false, true) + cardUtilizationRate := make(map[string]int) + // cardDurationStatistics, err := models.GetCardDurationStatistics(beginTime, endTime) + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) if err != nil { log.Error("GetCardDurationStatistics error:", err) return } - for _, cloudbrainStat := range cardUseInfo { - if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + for _, cloudbrainStat := range cardDurationStatistics { + if cloudbrainStat.TotalCanUse { + if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } } else { - cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } } } - - for _, cloudbrainStat := range cardCanUseInfo { - if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration - } else { - cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + for k, v := range cardCanUsageRes { + for j, i := range cardUsageRes { + if k == j { + cardUtilizationRate[k] = i / v + } } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "cardUseInfo": cardUseInfo, - "cardCanUseInfo": cardCanUseInfo, - "cardUsageRes": cardUsageRes, - "cardCanUsageRes": cardCanUsageRes, + "cardDurationStatistics": cardDurationStatistics, + "cardUsageRes": cardUsageRes, + "cardCanUsageRes": cardCanUsageRes, + "cardUtilizationRate": cardUtilizationRate, }) } + func GetCloudbrainResourceUsageDetail(ctx *context.Context) { - recordBeginTime := time.Now().AddDate(0, 0, -6) - beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) - if err != nil { - log.Error("getCloudbrainTimePeroid error:", err) - return + queryType := ctx.QueryTrim("type") + now := time.Now() + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var endTimeTemp time.Time + dayCloudbrainDuration := make([]models.DateCloudbrainStatistic, 0) + var err error + var count int + if queryType != "" { + if queryType == "all" { + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get GetDurationRecordBeginTime", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "today" { + beginTime = now.AddDate(0, 0, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + endTimeTemp = time.Date(endTimeTemp.Year(), endTimeTemp.Month(), endTimeTemp.Day(), 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + if endTime.After(time.Now()) { + endTime = time.Now() + } + endTimeTemp = beginTime.AddDate(0, 0, 1) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } + } - totalUse := true - totalCanUse := false - cardDurationStatisticsInfo, err := models.GetCardDurationStatistics(beginTime, endTime, totalUse, totalCanUse) - if err != nil { - log.Error("GetCardDurationStatistics error:", err) - return + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 5 } + pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + ctx.JSON(http.StatusOK, map[string]interface{}{ - "cardDurationStatisticsInfo": cardDurationStatisticsInfo, - "beginTime": beginTime, - "endTime": endTime, + "totalCount": count, + "pageDateCloudbrainDuration": pageDateCloudbrainDuration, }) } + +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]int) { + + aiCenterTotalDuration := make(map[string]int) + aiCenterUsageDuration := make(map[string]int) + aiCenterUsageRate := make(map[string]int) + for _, cloudbrainStatistic := range cloudbrainStatistics { + if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { + if cloudbrainStatistic.TotalCanUse { + if _, ok := aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } else { + if _, ok := aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } + } + } + for k, v := range aiCenterTotalDuration { + for i, j := range aiCenterUsageDuration { + if k == i { + aiCenterUsageRate[k] = j / v + } + } + } + + return aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate +} + +func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models.DateCloudbrainStatistic, int, error) { + now := time.Now() + endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + if endTimeTemp.Equal(endTime) { + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + } + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return nil, 0, err + } + dayCloudbrainInfo := make([]models.DateCloudbrainStatistic, 0) + count := 0 + for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { + aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) + dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateCloudbrainStatistic{ + Date: endTimeTemp.Format("2006/01/02"), + AiCenterUsageDuration: aiCenterUsageDuration, + AiCenterTotalDuration: aiCenterTotalDuration, + AiCenterUsageRate: aiCenterUsageRate, + }) + endTime = endTimeTemp + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + count += 1 + } + return dayCloudbrainInfo, count, nil +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index c7546c1c6..da1849096 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -4,19 +4,11 @@ import ( "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" ) -func CloudbrainDurationStatistic() { - log.Info("Generate Cloudbrain Duration statistic begin") - // yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") - // CloudbrainDurationStatisticHour(yesterday) - log.Info("Generate Cloudbrain Duration statistic end") -} - -func CloudbrainDurationStatisticHour(ctx *context.Context) { +func CloudbrainDurationStatisticHour() { hourTime := time.Now().Hour() dateTime := time.Now().Format("2006-01-02 15:04:05") dayTime := time.Now().Format("2006-01-02") @@ -25,11 +17,9 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { m, _ := time.ParseDuration("-1h") beginTime := currentTime.Add(m).Unix() endTime := currentTime.Unix() - // fmt.Println(beginTime) ciTasks1, err := models.GetCloudbrainRunning() if err != nil { - // ctx.ServerError("Get job failed:", err) log.Info("GetCloudbrainRunning err: %v", err) return } @@ -38,7 +28,6 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { log.Info("beginTime: %s", beginTime) log.Info("endTime: %s", endTime) if err != nil { - // ctx.ServerError("Get job failed:", err) log.Info("GetCloudbrainCompleteByTime err: %v", err) return } @@ -55,9 +44,8 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { } log.Info("cloudbrain: %s", cloudbrain) if cloudbrain != nil { - totalUse := true totalCanUse := false - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalUse, totalCanUse); err != nil { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalCanUse); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) return } @@ -71,8 +59,7 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { ComputeResource: cloudbrain[0].ComputeResource, AccCardType: cardType, CardsTotalDuration: cardDuration, - CreatedTime: timeutil.TimeStampNow(), - TotalUse: true, + CreatedUnix: timeutil.TimeStampNow(), TotalCanUse: false, } if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { @@ -90,9 +77,8 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { } log.Info("resourceQueues here: %s", resourceQueues) for _, resourceQueue := range resourceQueues { - totalUse := false totalCanUse := true - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalUse, totalCanUse); err != nil { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalCanUse); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) return } @@ -108,8 +94,7 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { AccCardType: resourceQueue.AccCardType, CardsTotalDuration: cardsTotalDuration, CardsTotalNum: resourceQueue.CardsTotalNum, - CreatedTime: timeutil.TimeStampNow(), - TotalUse: false, + CreatedUnix: timeutil.TimeStampNow(), TotalCanUse: true, } if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { From 9d1ceabf9cfab9a6fd581803bc8a7f8e88e18ccf Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 18 Oct 2022 18:04:22 +0800 Subject: [PATCH 06/50] update --- models/cloudbrain_static.go | 23 +++-- modules/cron/tasks_basic.go | 4 +- routers/api/v1/repo/cloudbrain_dashboard.go | 148 ++++++++++++++++++++++++++-- 3 files changed, 161 insertions(+), 14 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index ae9358dc1..3c5da2fd3 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -63,11 +63,23 @@ type DurationStatisticOptions struct { EndTime time.Time AiCenterCode string } + +type DurationRateStatistic struct { + AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` + AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` + TotalUsageRate float64 `json:"totalUsageRate"` +} type DateCloudbrainStatistic struct { - Date string `json:"date"` - AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` - AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` - AiCenterUsageRate map[string]int `json:"aiCenterUsageRate"` + Date string `json:"date"` + AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` + AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` + AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` +} + +type HourTimeStatistic struct { + HourTimeUsageDuration map[int]int `json:"hourTimeUsageDuration"` + HourTimeTotalDuration map[int]int `json:"hourTimeTotalDuration"` + HourTimeUsageRate map[int]float64 `json:"hourTimeUsageRate"` } func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { @@ -345,8 +357,7 @@ func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) { defer sess.Close() sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1") CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0) - if err := sess.Table(&CloudbrainDurationStatistic{}).Unscoped(). - Find(&CloudbrainDurationStatistics); err != nil { + if err := sess.Table(&CloudbrainDurationStatistic{}).Find(&CloudbrainDurationStatistics); err != nil { log.Info("find error.") } return CloudbrainDurationStatistics, nil diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index b68d747d8..38ac37852 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -259,7 +259,7 @@ func registerHandleCloudbrainDurationStatistic() { RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "@every 60m", + Schedule: "55 59 * * * ?", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.CloudbrainDurationStatisticHour() return nil @@ -283,7 +283,6 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() - registerHandleCloudbrainDurationStatistic() registerSyncCloudbrainStatus() registerHandleOrgStatistic() @@ -292,4 +291,5 @@ func initBasicTasks() { //registerRewardPeriodTask() registerCloudbrainPointDeductTask() + registerHandleCloudbrainDurationStatistic() } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index caac24aad..468692663 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1487,13 +1487,15 @@ func GetCloudbrainResourceUsage(ctx *context.Context) { func GetCloudbrainResourceUsageDetail(ctx *context.Context) { queryType := ctx.QueryTrim("type") now := time.Now() - beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") + aiCenterCode := ctx.QueryTrim("aiCenterCode") + var beginTime time.Time var endTime time.Time var endTimeTemp time.Time dayCloudbrainDuration := make([]models.DateCloudbrainStatistic, 0) + hourCloudbrainDuration := models.HourTimeStatistic{} var err error var count int if queryType != "" { @@ -1505,14 +1507,22 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { return } brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + log.Info("recordCloudbrainDuration:", recordCloudbrainDuration) + log.Info("brainRecordBeginTime:", brainRecordBeginTime) beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) if err != nil { log.Error("Can not query dayCloudbrainDuration.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) return } + hourCloudbrainDuration = getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query hourCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } } else if queryType == "today" { beginTime = now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) @@ -1644,19 +1654,21 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { pagesize = 5 } pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + durationRateStatistic := getDurationStatistic(beginTime, endTime) ctx.JSON(http.StatusOK, map[string]interface{}{ "totalCount": count, "pageDateCloudbrainDuration": pageDateCloudbrainDuration, + "durationRateStatistic": durationRateStatistic, + "hourCloudbrainDuration": hourCloudbrainDuration, }) } -func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]int) { - +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]float64) { aiCenterTotalDuration := make(map[string]int) aiCenterUsageDuration := make(map[string]int) - aiCenterUsageRate := make(map[string]int) + aiCenterUsageRate := make(map[string]float64) for _, cloudbrainStatistic := range cloudbrainStatistics { if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { if cloudbrainStatistic.TotalCanUse { @@ -1674,10 +1686,21 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain } } } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return nil, nil, nil + } + for _, v := range ResourceAiCenterRes { + if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { + aiCenterUsageDuration[v.AiCenterCode] = 0 + } + } + for k, v := range aiCenterTotalDuration { for i, j := range aiCenterUsageDuration { if k == i { - aiCenterUsageRate[k] = j / v + aiCenterUsageRate[k] = float64(j) / float64(v) } } } @@ -1685,6 +1708,61 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain return aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate } +func getDurationStatistic(beginTime time.Time, endTime time.Time) models.DurationRateStatistic { + aiCenterTotalDurationStat := make(map[string]int) + aiCenterUsageDurationStat := make(map[string]int) + durationRateStatistic := models.DurationRateStatistic{} + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return durationRateStatistic + } + for _, cloudbrainStatistic := range cardDurationStatistics { + if cloudbrainStatistic.TotalCanUse { + if _, ok := aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } else { + if _, ok := aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } + } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return durationRateStatistic + } + for _, v := range ResourceAiCenterRes { + if _, ok := aiCenterUsageDurationStat[v.AiCenterCode]; !ok { + aiCenterUsageDurationStat[v.AiCenterCode] = 0 + } + } + totalCanUse := float64(0) + totalUse := float64(0) + for k, v := range aiCenterTotalDurationStat { + for i, j := range aiCenterUsageDurationStat { + if k == i { + totalUse += float64(j) + totalCanUse += float64(v) + } + } + } + totalUsageRate := totalUse / totalCanUse + + durationRateStatistic.AiCenterTotalDurationStat = aiCenterTotalDurationStat + durationRateStatistic.AiCenterUsageDurationStat = aiCenterUsageDurationStat + durationRateStatistic.TotalUsageRate = totalUsageRate + return durationRateStatistic +} + func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models.DateCloudbrainStatistic, int, error) { now := time.Now() endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) @@ -1699,6 +1777,7 @@ func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models. log.Error("GetCardDurationStatistics error:", err) return nil, 0, err } + dayCloudbrainInfo := make([]models.DateCloudbrainStatistic, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { @@ -1711,7 +1790,64 @@ func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models. }) endTime = endTimeTemp endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + if endTimeTemp.Before(beginTime) && beginTime.Before(endTime) { + endTimeTemp = beginTime + } count += 1 } return dayCloudbrainInfo, count, nil } + +func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) models.HourTimeStatistic { + hourTimeTotalDuration := make(map[int]int) + hourTimeUsageDuration := make(map[int]int) + hourTimeUsageRate := make(map[int]float64) + hourTimeStatistic := models.HourTimeStatistic{} + + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return hourTimeStatistic + } + for _, cloudbrainStatistic := range cardDurationStatistics { + if cloudbrainStatistic.AiCenterCode == aiCenterCode { + if cloudbrainStatistic.TotalCanUse { + if _, ok := hourTimeTotalDuration[cloudbrainStatistic.HourTime]; !ok { + hourTimeTotalDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + } else { + hourTimeTotalDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration + } + } else { + if _, ok := hourTimeUsageDuration[cloudbrainStatistic.HourTime]; !ok { + hourTimeUsageDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + } else { + hourTimeUsageDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration + } + } + hourTimeList := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} + for _, v := range hourTimeList { + if _, ok := hourTimeUsageDuration[v]; !ok { + hourTimeUsageDuration[v] = 0 + } + if _, ok := hourTimeTotalDuration[v]; !ok { + hourTimeTotalDuration[v] = 0 + } + } + + for k, v := range hourTimeTotalDuration { + for i, j := range hourTimeUsageDuration { + if k == i { + hourTimeUsageRate[k] = float64(j) / float64(v) + } + } + } + } + } + hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration + hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration + hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate + return hourTimeStatistic +} From c0d643a2b41fa2fd8d858a40a27837cfd7faf80f Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 18 Oct 2022 18:05:31 +0800 Subject: [PATCH 07/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 468692663..781bc6280 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1511,7 +1511,7 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { log.Info("brainRecordBeginTime:", brainRecordBeginTime) beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) if err != nil { log.Error("Can not query dayCloudbrainDuration.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) From f1ad30cd11d2b2e2380e748bff73e1625f1bef14 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 19 Oct 2022 16:13:36 +0800 Subject: [PATCH 08/50] update --- models/cloudbrain_static.go | 33 ++- routers/api/v1/api.go | 5 +- routers/api/v1/repo/cloudbrain_dashboard.go | 368 ++++++++++++++-------------- 3 files changed, 214 insertions(+), 192 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 3c5da2fd3..ea93015b3 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,6 +38,16 @@ type TaskDetail struct { WorkServerNum int64 `json:"WorkServerNum"` Spec *Specification `json:"Spec"` } +type CardTypeAndNum struct { + CardType string `json:"CardType"` + Num int `json:"Num"` + ComputeResource string `json:"computeResource"` +} +type ResourceOverview struct { + Cluster string `json:"cluster"` + AiCenterCode string `json:"aiCenterCode"` + CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` +} type CloudbrainDurationStatistic struct { ID int64 `xorm:"pk autoincr"` @@ -69,17 +79,24 @@ type DurationRateStatistic struct { AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` TotalUsageRate float64 `json:"totalUsageRate"` } -type DateCloudbrainStatistic struct { - Date string `json:"date"` - AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` - AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` - AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` + +// type DateCloudbrainStatistic struct { +// Date string `json:"date"` +// AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` +// AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` +// AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` +// } +type DateUsageStatistic struct { + Date string `json:"date"` + UsageDuration int `json:"usageDuration"` + TotalDuration int `json:"totalDuration"` + UsageRate float64 `json:"usageRate"` } type HourTimeStatistic struct { - HourTimeUsageDuration map[int]int `json:"hourTimeUsageDuration"` - HourTimeTotalDuration map[int]int `json:"hourTimeTotalDuration"` - HourTimeUsageRate map[int]float64 `json:"hourTimeUsageRate"` + HourTimeUsageDuration map[string]int `json:"hourTimeUsageDuration"` + HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"` + HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"` } func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 06be12e92..2964e87ce 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -601,8 +601,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/running_top_data", repo.GetRunningTop) m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) - m.Get("/resource_usage", repo.GetCloudbrainResourceUsage) - m.Get("/resource_usage_detail", repo.GetCloudbrainResourceUsageDetail) + m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) + m.Get("/resource_usage_rate", repo.GetCloudbrainResourceUsage) + m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 781bc6280..2e5e77078 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" "time" @@ -532,17 +533,17 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int } -func getPageDateCloudbrainDuration(dateCloudbrainDuration []models.DateCloudbrainStatistic, page int, pagesize int) []models.DateCloudbrainStatistic { +func getPageDateCloudbrainDuration(dateUsageStatistic []models.DateUsageStatistic, page int, pagesize int) []models.DateUsageStatistic { begin := (page - 1) * pagesize end := (page) * pagesize - if begin > len(dateCloudbrainDuration)-1 { + if begin > len(dateUsageStatistic)-1 { return nil } - if end > len(dateCloudbrainDuration)-1 { - return dateCloudbrainDuration[begin:] + if end > len(dateUsageStatistic)-1 { + return dateUsageStatistic[begin:] } else { - return dateCloudbrainDuration[begin:end] + return dateUsageStatistic[begin:end] } } @@ -1425,9 +1426,53 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Info("GetCanUseCardInfo err: %v", err) return } + // ResourceAiCenterRes, err := models.GetResourceAiCenters() + // if err != nil { + // log.Error("Can not get ResourceAiCenterRes.", err) + // return + // } + resourceOverviews := []models.ResourceOverview{} + resourceOpenIOne := models.ResourceOverview{} + resourceOpenITwo := models.ResourceOverview{} + // resourceChengdu := models.ResourceOverview{} + + for _, resourceQueue := range resourceQueues { + if resourceQueue.Cluster == models.OpenICluster { + if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { + resourceOpenIOne.Cluster = models.OpenICluster + resourceOpenIOne.AiCenterCode = models.AICenterOfCloudBrainOne + cardTypeNum := models.CardTypeAndNum{} + cardTypeNum.CardType = resourceQueue.AccCardType + cardTypeNum.Num = resourceQueue.CardsTotalNum + cardTypeNum.ComputeResource = resourceQueue.ComputeResource + resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) + } + if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { + resourceOpenITwo.Cluster = models.OpenICluster + resourceOpenITwo.AiCenterCode = models.AICenterOfCloudBrainTwo + cardTypeNum := models.CardTypeAndNum{} + cardTypeNum.CardType = resourceQueue.AccCardType + cardTypeNum.Num = resourceQueue.CardsTotalNum + cardTypeNum.ComputeResource = resourceQueue.ComputeResource + resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) + } + // if resourceQueue.AiCenterCode == models.AICenterOfChengdu { + // resourceChengdu.Cluster = models.OpenICluster + // resourceChengdu.AiCenterCode = models.AICenterOfChengdu + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceQueue.AccCardType + // cardTypeNum.Num = resourceQueue.CardsTotalNum + // cardTypeNum.ComputeResource = resourceQueue.ComputeResource + // resourceChengdu.CardTypeAndNum = append(resourceChengdu.CardTypeAndNum, cardTypeNum) + // } + } + } + resourceOverviews = append(resourceOverviews, resourceOpenIOne) + resourceOverviews = append(resourceOverviews, resourceOpenITwo) + // resourceOverviews = append(resourceOverviews, resourceChengdu) ctx.JSON(http.StatusOK, map[string]interface{}{ - "resourceQueues": resourceQueues, + "resourceOverviews": resourceOverviews, }) } @@ -1485,119 +1530,96 @@ func GetCloudbrainResourceUsage(ctx *context.Context) { } func GetCloudbrainResourceUsageDetail(ctx *context.Context) { + aiCenterCode := ctx.QueryTrim("aiCenterCode") + log.Info("aiCenterCode: %v", aiCenterCode) + if aiCenterCode == "" { + aiCenterCode = "OpenIOne" + } + beginTime, endTime := getBeginAndEndTime(ctx) + dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + return + } + hourCloudbrainDuration, err := getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query hourCloudbrainDuration.", err) + return + } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 36500 + } + pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "totalCount": count, + "pageDateCloudbrainDuration": pageDateCloudbrainDuration, + "hourCloudbrainDuration": hourCloudbrainDuration, + }) +} + +func GetDurationRateStatistic(ctx *context.Context) { + beginTime, endTime := getBeginAndEndTime(ctx) + durationRateStatistic := getDurationStatistic(beginTime, endTime) + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "durationRateStatistic": durationRateStatistic, + }) + +} + +func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { queryType := ctx.QueryTrim("type") now := time.Now() beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") - aiCenterCode := ctx.QueryTrim("aiCenterCode") var beginTime time.Time var endTime time.Time - var endTimeTemp time.Time - dayCloudbrainDuration := make([]models.DateCloudbrainStatistic, 0) - hourCloudbrainDuration := models.HourTimeStatistic{} var err error - var count int if queryType != "" { if queryType == "all" { recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() if err != nil { log.Error("Can not get GetDurationRecordBeginTime", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return + return beginTime, endTime } brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() - log.Info("recordCloudbrainDuration:", recordCloudbrainDuration) - log.Info("brainRecordBeginTime:", brainRecordBeginTime) beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - hourCloudbrainDuration = getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) - if err != nil { - log.Error("Can not query hourCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "today" { beginTime = now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "yesterday" { beginTime = now.AddDate(0, 0, -1) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - } else if queryType == "last_7day" { beginTime = now.AddDate(0, 0, -6) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now - endTimeTemp = time.Date(endTimeTemp.Year(), endTimeTemp.Month(), endTimeTemp.Day(), 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "last_30day" { beginTime = now.AddDate(0, 0, -29) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "current_month" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "current_year" { endTime = now beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - } else if queryType == "last_month" { - lastMonthTime := now.AddDate(0, -1, 0) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - } } else { @@ -1607,105 +1629,81 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { if err != nil { log.Error("Can not get recordCloudbrain", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return + return beginTime, endTime } brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else { beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) - return + return beginTime, endTime } endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) - return + return beginTime, endTime } if endTime.After(time.Now()) { endTime = time.Now() } - endTimeTemp = beginTime.AddDate(0, 0, 1) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } } - - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pagesize := ctx.QueryInt("pagesize") - if pagesize <= 0 { - pagesize = 5 - } - pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) - durationRateStatistic := getDurationStatistic(beginTime, endTime) - - ctx.JSON(http.StatusOK, map[string]interface{}{ - "totalCount": count, - "pageDateCloudbrainDuration": pageDateCloudbrainDuration, - "durationRateStatistic": durationRateStatistic, - "hourCloudbrainDuration": hourCloudbrainDuration, - }) - + return beginTime, endTime } -func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]float64) { - aiCenterTotalDuration := make(map[string]int) - aiCenterUsageDuration := make(map[string]int) - aiCenterUsageRate := make(map[string]float64) +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (int, int, float64) { + totalDuration := int(0) + usageDuration := int(0) + usageRate := float64(0) + for _, cloudbrainStatistic := range cloudbrainStatistics { if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { if cloudbrainStatistic.TotalCanUse { - if _, ok := aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration - } else { - aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration - } + totalDuration += cloudbrainStatistic.CardsTotalDuration } else { - if _, ok := aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration - } else { - aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration - } - } - } - } - ResourceAiCenterRes, err := models.GetResourceAiCenters() - if err != nil { - log.Error("Can not get ResourceAiCenterRes.", err) - return nil, nil, nil - } - for _, v := range ResourceAiCenterRes { - if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { - aiCenterUsageDuration[v.AiCenterCode] = 0 + usageDuration += cloudbrainStatistic.CardsTotalDuration + } + // if cloudbrainStatistic.TotalCanUse { + // if _, ok := aiCenterTotalDuration[Date]; !ok { + // aiCenterTotalDuration[Date] = cloudbrainStatistic.CardsTotalDuration + // } else { + // aiCenterTotalDuration[Date] += cloudbrainStatistic.CardsTotalDuration + // } + // } else { + // if _, ok := aiCenterUsageDuration[Date]; !ok { + // aiCenterUsageDuration[Date] = cloudbrainStatistic.CardsTotalDuration + // } else { + // aiCenterUsageDuration[Date] += cloudbrainStatistic.CardsTotalDuration + // } + // } } } - - for k, v := range aiCenterTotalDuration { - for i, j := range aiCenterUsageDuration { - if k == i { - aiCenterUsageRate[k] = float64(j) / float64(v) - } - } - } - - return aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate + // ResourceAiCenterRes, err := models.GetResourceAiCenters() + // if err != nil { + // log.Error("Can not get ResourceAiCenterRes.", err) + // return nil, nil, nil + // } + // for _, v := range ResourceAiCenterRes { + // if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { + // aiCenterUsageDuration[v.AiCenterCode] = 0 + // } + // } + + // for k, v := range aiCenterTotalDuration { + // for i, j := range aiCenterUsageDuration { + // if k == i { + // aiCenterUsageRate[k] = float64(j) / float64(v) + // } + // } + // } + // usageRate = float64(usageDuration) / float64(totalDuration) + + return totalDuration, usageDuration, usageRate } func getDurationStatistic(beginTime time.Time, endTime time.Time) models.DurationRateStatistic { @@ -1763,30 +1761,31 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) models.Duratio return durationRateStatistic } -func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models.DateCloudbrainStatistic, int, error) { +func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { now := time.Now() endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) if endTimeTemp.Equal(endTime) { endTimeTemp = endTimeTemp.AddDate(0, 0, -1) } cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ - BeginTime: beginTime, - EndTime: endTime, + BeginTime: beginTime, + EndTime: endTime, + AiCenterCode: aiCenterCode, }) if err != nil { log.Error("GetCardDurationStatistics error:", err) return nil, 0, err } - dayCloudbrainInfo := make([]models.DateCloudbrainStatistic, 0) + dayCloudbrainInfo := make([]models.DateUsageStatistic, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { - aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) - dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateCloudbrainStatistic{ - Date: endTimeTemp.Format("2006/01/02"), - AiCenterUsageDuration: aiCenterUsageDuration, - AiCenterTotalDuration: aiCenterTotalDuration, - AiCenterUsageRate: aiCenterUsageRate, + TotalDuration, UsageDuration, UsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) + dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateUsageStatistic{ + Date: endTimeTemp.Format("2006/01/02"), + UsageDuration: UsageDuration, + TotalDuration: TotalDuration, + UsageRate: UsageRate, }) endTime = endTimeTemp endTimeTemp = endTimeTemp.AddDate(0, 0, -1) @@ -1798,10 +1797,10 @@ func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models. return dayCloudbrainInfo, count, nil } -func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) models.HourTimeStatistic { - hourTimeTotalDuration := make(map[int]int) - hourTimeUsageDuration := make(map[int]int) - hourTimeUsageRate := make(map[int]float64) +func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) (models.HourTimeStatistic, error) { + hourTimeTotalDuration := make(map[string]int) + hourTimeUsageDuration := make(map[string]int) + hourTimeUsageRate := make(map[string]float64) hourTimeStatistic := models.HourTimeStatistic{} cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ @@ -1810,44 +1809,49 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC }) if err != nil { log.Error("GetCardDurationStatistics error:", err) - return hourTimeStatistic + return hourTimeStatistic, err } for _, cloudbrainStatistic := range cardDurationStatistics { if cloudbrainStatistic.AiCenterCode == aiCenterCode { if cloudbrainStatistic.TotalCanUse { - if _, ok := hourTimeTotalDuration[cloudbrainStatistic.HourTime]; !ok { - hourTimeTotalDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + if _, ok := hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok { + hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration } else { - hourTimeTotalDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration + hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration } } else { - if _, ok := hourTimeUsageDuration[cloudbrainStatistic.HourTime]; !ok { - hourTimeUsageDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + if _, ok := hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok { + hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration } else { - hourTimeUsageDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration - } - } - hourTimeList := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} - for _, v := range hourTimeList { - if _, ok := hourTimeUsageDuration[v]; !ok { - hourTimeUsageDuration[v] = 0 - } - if _, ok := hourTimeTotalDuration[v]; !ok { - hourTimeTotalDuration[v] = 0 - } - } - - for k, v := range hourTimeTotalDuration { - for i, j := range hourTimeUsageDuration { - if k == i { - hourTimeUsageRate[k] = float64(j) / float64(v) - } + hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration } } } } + hourTimeList := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"} + for _, v := range hourTimeList { + if _, ok := hourTimeUsageDuration[v]; !ok { + hourTimeUsageDuration[v] = 0 + } + if _, ok := hourTimeTotalDuration[v]; !ok { + hourTimeTotalDuration[v] = 0 + } + // if _, ok := hourTimeUsageRate[v]; !ok { + // hourTimeUsageRate[v] = 0 + // } + + } + + // for k, v := range hourTimeTotalDuration { + // for i, j := range hourTimeUsageDuration { + // if k == i { + // hourTimeUsageRate[k] = float64(j) / float64(v) + // } + // } + // } + hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate - return hourTimeStatistic + return hourTimeStatistic, nil } From ebc00c2bfaa7806a6e0b761606db369543e66fc5 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 19 Oct 2022 17:12:41 +0800 Subject: [PATCH 09/50] update --- models/cloudbrain_static.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 31 ++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index ea93015b3..a693e0555 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -45,6 +45,7 @@ type CardTypeAndNum struct { } type ResourceOverview struct { Cluster string `json:"cluster"` + AiCenterName string `json:"aiCenterName"` AiCenterCode string `json:"aiCenterCode"` CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 2e5e77078..517905c9b 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1426,11 +1426,11 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Info("GetCanUseCardInfo err: %v", err) return } - // ResourceAiCenterRes, err := models.GetResourceAiCenters() - // if err != nil { - // log.Error("Can not get ResourceAiCenterRes.", err) - // return - // } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return + } resourceOverviews := []models.ResourceOverview{} resourceOpenIOne := models.ResourceOverview{} resourceOpenITwo := models.ResourceOverview{} @@ -1439,8 +1439,9 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { - resourceOpenIOne.Cluster = models.OpenICluster - resourceOpenIOne.AiCenterCode = models.AICenterOfCloudBrainOne + resourceOpenIOne.Cluster = resourceQueue.Cluster + resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode + resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName cardTypeNum := models.CardTypeAndNum{} cardTypeNum.CardType = resourceQueue.AccCardType cardTypeNum.Num = resourceQueue.CardsTotalNum @@ -1448,8 +1449,9 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) } if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { - resourceOpenITwo.Cluster = models.OpenICluster - resourceOpenITwo.AiCenterCode = models.AICenterOfCloudBrainTwo + resourceOpenITwo.Cluster = resourceQueue.Cluster + resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode + resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName cardTypeNum := models.CardTypeAndNum{} cardTypeNum.CardType = resourceQueue.AccCardType cardTypeNum.Num = resourceQueue.CardsTotalNum @@ -1470,6 +1472,17 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceOverviews = append(resourceOverviews, resourceOpenIOne) resourceOverviews = append(resourceOverviews, resourceOpenITwo) // resourceOverviews = append(resourceOverviews, resourceChengdu) + for _, resourceAiCenterRes := range ResourceAiCenterRes { + if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { + if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { + test := models.ResourceOverview{} + test.Cluster = models.C2NetCluster + test.AiCenterCode = resourceAiCenterRes.AiCenterCode + test.AiCenterName = resourceAiCenterRes.AiCenterName + resourceOverviews = append(resourceOverviews, test) + } + } + } ctx.JSON(http.StatusOK, map[string]interface{}{ "resourceOverviews": resourceOverviews, From 0a4875c4cb8e9727add429c5393d897a4b0c5d85 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 19 Oct 2022 17:58:35 +0800 Subject: [PATCH 10/50] update --- models/cloudbrain_static.go | 6 +- routers/api/v1/repo/cloudbrain_dashboard.go | 92 ++++++++++++++++++++--------- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index a693e0555..84b659dd4 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -76,9 +76,9 @@ type DurationStatisticOptions struct { } type DurationRateStatistic struct { - AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` - AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` - TotalUsageRate float64 `json:"totalUsageRate"` + AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` + AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` + UsageRate map[string]float32 `json:"UsageRate"` } // type DateCloudbrainStatistic struct { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 517905c9b..82bdb7be4 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1577,10 +1577,12 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { func GetDurationRateStatistic(ctx *context.Context) { beginTime, endTime := getBeginAndEndTime(ctx) - durationRateStatistic := getDurationStatistic(beginTime, endTime) + OpenIDurationRate, C2NetDurationRate, totalUsageRate := getDurationStatistic(beginTime, endTime) ctx.JSON(http.StatusOK, map[string]interface{}{ - "durationRateStatistic": durationRateStatistic, + "openIDurationRate": OpenIDurationRate, + "c2NetDurationRate": C2NetDurationRate, + "totalUsageRate": totalUsageRate, }) } @@ -1719,59 +1721,91 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain return totalDuration, usageDuration, usageRate } -func getDurationStatistic(beginTime time.Time, endTime time.Time) models.DurationRateStatistic { - aiCenterTotalDurationStat := make(map[string]int) - aiCenterUsageDurationStat := make(map[string]int) - durationRateStatistic := models.DurationRateStatistic{} +func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float32) { + OpenITotalDuration := make(map[string]int) + OpenIUsageDuration := make(map[string]int) + OpenIUsageRate := make(map[string]float32) + C2NetTotalDuration := make(map[string]int) + C2NetUsageDuration := make(map[string]int) + OpenIDurationRate := models.DurationRateStatistic{} + C2NetDurationRate := models.DurationRateStatistic{} cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ BeginTime: beginTime, EndTime: endTime, }) if err != nil { log.Error("GetCardDurationStatistics error:", err) - return durationRateStatistic + return OpenIDurationRate, C2NetDurationRate, 0 } for _, cloudbrainStatistic := range cardDurationStatistics { - if cloudbrainStatistic.TotalCanUse { - if _, ok := aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + if cloudbrainStatistic.Cluster == models.OpenICluster { + if cloudbrainStatistic.TotalCanUse { + if _, ok := OpenITotalDuration[cloudbrainStatistic.AiCenterName]; !ok { + OpenITotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + OpenITotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } else { - aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + if _, ok := OpenIUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { + OpenIUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + OpenIUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } - } else { - if _, ok := aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } + if cloudbrainStatistic.Cluster == models.C2NetCluster { + if cloudbrainStatistic.TotalCanUse { + if _, ok := C2NetTotalDuration[cloudbrainStatistic.AiCenterName]; !ok { + C2NetTotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + C2NetTotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } else { - aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + if _, ok := C2NetUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { + C2NetUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + C2NetUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } } } ResourceAiCenterRes, err := models.GetResourceAiCenters() if err != nil { log.Error("Can not get ResourceAiCenterRes.", err) - return durationRateStatistic + return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - if _, ok := aiCenterUsageDurationStat[v.AiCenterCode]; !ok { - aiCenterUsageDurationStat[v.AiCenterCode] = 0 + if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { + // if v.AiCenterCode != models.AICenterOfCloudBrainTwo { + if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { + C2NetUsageDuration[v.AiCenterName] = 0 + } + // } + } else { + if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { + OpenIUsageDuration[v.AiCenterName] = 0 + } } } - totalCanUse := float64(0) - totalUse := float64(0) - for k, v := range aiCenterTotalDurationStat { - for i, j := range aiCenterUsageDurationStat { + // totalCanUse := float64(0) + // totalUse := float64(0) + totalUsageRate := float32(0) + for k, v := range OpenITotalDuration { + for i, j := range OpenIUsageDuration { if k == i { - totalUse += float64(j) - totalCanUse += float64(v) + OpenIUsageRate[k] = float32(j) / float32(v) } } } - totalUsageRate := totalUse / totalCanUse + // totalUsageRate := totalUse / totalCanUse - durationRateStatistic.AiCenterTotalDurationStat = aiCenterTotalDurationStat - durationRateStatistic.AiCenterUsageDurationStat = aiCenterUsageDurationStat - durationRateStatistic.TotalUsageRate = totalUsageRate - return durationRateStatistic + OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration + OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration + OpenIDurationRate.UsageRate = OpenIUsageRate + C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration + C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration + // C2NetDurationRate.TotalUsageRate = totalUsageRate + return OpenIDurationRate, C2NetDurationRate, totalUsageRate } func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { From e25291a069936602bece9f051f5c7f9cf5ce583e Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 09:56:26 +0800 Subject: [PATCH 11/50] update --- models/cloudbrain_static.go | 33 +++++-- routers/api/v1/repo/cloudbrain_dashboard.go | 132 +++++++++++++++++----------- 2 files changed, 110 insertions(+), 55 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 84b659dd4..3f2dcf94a 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -80,6 +80,17 @@ type DurationRateStatistic struct { AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` UsageRate map[string]float32 `json:"UsageRate"` } +type ResourceDetail struct { + QueueCode string + Cluster string `xorm:"notnull"` + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + CardsTotalNum int + IsAutomaticSync bool + // CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` +} // type DateCloudbrainStatistic struct { // Date string `json:"date"` @@ -336,16 +347,26 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - var cond = builder.NewCond() - cond = cond.And( - builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), - ) + // var cond = builder.NewCond() + // cond = cond.And( + // builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), + // ) + sess.OrderBy("resource_queue.id ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) - if err := sess.Table(&ResourceQueue{}).Where(cond). - Find(&ResourceQueues); err != nil { + if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") } return ResourceQueues, nil + // Cols("queue_code", "cluster", "ai_center_name", "ai_center_code", "compute_resource", "acc_card_type", "cards_total_num") + // sess := x.NewSession() + // defer sess.Close() + // sess.OrderBy("resource_queue.id ASC limit 1") + // cloudbrains := make([]*CloudbrainInfo, 0) + // if err := sess.Table(&Cloudbrain{}).Unscoped(). + // Find(&cloudbrains); err != nil { + // log.Info("find error.") + // } + // return cloudbrains, nil } func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 82bdb7be4..fe3e015ac 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1426,68 +1426,102 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Info("GetCanUseCardInfo err: %v", err) return } - ResourceAiCenterRes, err := models.GetResourceAiCenters() - if err != nil { - log.Error("Can not get ResourceAiCenterRes.", err) - return - } - resourceOverviews := []models.ResourceOverview{} - resourceOpenIOne := models.ResourceOverview{} - resourceOpenITwo := models.ResourceOverview{} - // resourceChengdu := models.ResourceOverview{} - + log.Info("resourceQueues:", resourceQueues) + OpenIResourceDetail := []models.ResourceDetail{} + C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { - if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { - resourceOpenIOne.Cluster = resourceQueue.Cluster - resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode - resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName - cardTypeNum := models.CardTypeAndNum{} - cardTypeNum.CardType = resourceQueue.AccCardType - cardTypeNum.Num = resourceQueue.CardsTotalNum - cardTypeNum.ComputeResource = resourceQueue.ComputeResource - resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) - } - if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { - resourceOpenITwo.Cluster = resourceQueue.Cluster - resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode - resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName - cardTypeNum := models.CardTypeAndNum{} - cardTypeNum.CardType = resourceQueue.AccCardType - cardTypeNum.Num = resourceQueue.CardsTotalNum - cardTypeNum.ComputeResource = resourceQueue.ComputeResource - resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) - } - // if resourceQueue.AiCenterCode == models.AICenterOfChengdu { - // resourceChengdu.Cluster = models.OpenICluster - // resourceChengdu.AiCenterCode = models.AICenterOfChengdu + // var resourceDetail models.ResourceDetail + // if _, ok := resourceDetail[resourceQueue.AiCenterCode]; !ok { + // resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + // } else { // cardTypeNum := models.CardTypeAndNum{} // cardTypeNum.CardType = resourceQueue.AccCardType // cardTypeNum.Num = resourceQueue.CardsTotalNum // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // resourceChengdu.CardTypeAndNum = append(resourceChengdu.CardTypeAndNum, cardTypeNum) // } + + var resourceDetail models.ResourceDetail + resourceDetail.QueueCode = resourceQueue.QueueCode + resourceDetail.Cluster = resourceQueue.Cluster + resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.ComputeResource = resourceQueue.ComputeResource + resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum + resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync + OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) + // } else { + } - } - resourceOverviews = append(resourceOverviews, resourceOpenIOne) - resourceOverviews = append(resourceOverviews, resourceOpenITwo) - // resourceOverviews = append(resourceOverviews, resourceChengdu) - for _, resourceAiCenterRes := range ResourceAiCenterRes { - if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { - if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { - test := models.ResourceOverview{} - test.Cluster = models.C2NetCluster - test.AiCenterCode = resourceAiCenterRes.AiCenterCode - test.AiCenterName = resourceAiCenterRes.AiCenterName - resourceOverviews = append(resourceOverviews, test) - } + if resourceQueue.Cluster == models.C2NetCluster { + var resourceDetail models.ResourceDetail + resourceDetail.QueueCode = resourceQueue.QueueCode + resourceDetail.Cluster = resourceQueue.Cluster + resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.ComputeResource = resourceQueue.ComputeResource + resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum + resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync + C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) } + + // ResourceAiCenterRes, err := models.GetResourceAiCenters() + // if err != nil { + // log.Error("Can not get ResourceAiCenterRes.", err) + // return + // } + // resourceOverviews := []models.ResourceOverview{} + // resourceOpenIOne := models.ResourceOverview{} + // resourceOpenITwo := models.ResourceOverview{} + // // resourceChengdu := models.ResourceOverview{} + + // for _, resourceQueue := range resourceQueues { + // if resourceQueue.Cluster == models.OpenICluster { + // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { + // resourceOpenIOne.Cluster = resourceQueue.Cluster + // resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode + // resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceQueue.AccCardType + // cardTypeNum.Num = resourceQueue.CardsTotalNum + // cardTypeNum.ComputeResource = resourceQueue.ComputeResource + // resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) + // } + // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { + // resourceOpenITwo.Cluster = resourceQueue.Cluster + // resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode + // resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceQueue.AccCardType + // cardTypeNum.Num = resourceQueue.CardsTotalNum + // cardTypeNum.ComputeResource = resourceQueue.ComputeResource + // resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) + // } + // } + // } + // resourceOverviews = append(resourceOverviews, resourceOpenIOne) + // resourceOverviews = append(resourceOverviews, resourceOpenITwo) + // // resourceOverviews = append(resourceOverviews, resourceChengdu) + // for _, resourceAiCenterRes := range ResourceAiCenterRes { + // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { + // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { + // test := models.ResourceOverview{} + // test.Cluster = models.C2NetCluster + // test.AiCenterCode = resourceAiCenterRes.AiCenterCode + // test.AiCenterName = resourceAiCenterRes.AiCenterName + // resourceOverviews = append(resourceOverviews, test) + // } + // } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "resourceOverviews": resourceOverviews, + // "resourceOverviews": resourceOverviews, + "OpenIResourceDetail": OpenIResourceDetail, + "C2NetResourceDetail": C2NetResourceDetail, + // "resourceQueues": resourceQueues, }) - } func GetCloudbrainResourceUsage(ctx *context.Context) { From c1a63edd2cfd3516ca8a4e9d89472fac076a4a3f Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 20 Oct 2022 10:37:13 +0800 Subject: [PATCH 12/50] add --- modules/setting/setting.go | 8 ++++++++ routers/repo/ai_model_convert.go | 18 +++++++++++++++++- templates/repo/modelmanage/convertIndex.tmpl | 6 ++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c6afae05a..89a0fd2d6 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -694,8 +694,12 @@ var ( GPU_PYTORCH_IMAGE string GpuQueue string GPU_TENSORFLOW_IMAGE string + GPU_PADDLE_IMAGE string + GPU_MXNET_IMAGE string NPU_MINDSPORE_16_IMAGE string PytorchOnnxBootFile string + PaddleOnnxBootFile string + MXnetOnnxBootFile string PytorchTrTBootFile string MindsporeBootFile string TensorFlowNpuBootFile string @@ -1576,6 +1580,10 @@ func getModelConvertConfig() { ModelConvert.NPU_PoolID = sec.Key("NPU_PoolID").MustString("pool7908321a") ModelConvert.NPU_MINDSPORE_IMAGE_ID = sec.Key("NPU_MINDSPORE_IMAGE_ID").MustInt(121) ModelConvert.NPU_TENSORFLOW_IMAGE_ID = sec.Key("NPU_TENSORFLOW_IMAGE_ID").MustInt(35) + ModelConvert.GPU_PADDLE_IMAGE = sec.Key("GPU_PADDLE_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:paddle2.3.0_gpu_cuda11.2_cudnn8") + ModelConvert.GPU_MXNET_IMAGE = sec.Key("GPU_MXNET_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:mxnet191cu_cuda102_py37") + ModelConvert.PaddleOnnxBootFile = sec.Key("PaddleOnnxBootFile").MustString("convert_paddle.py") + ModelConvert.MXnetOnnxBootFile = sec.Key("MXnetOnnxBootFile").MustString("convert_mxnet.py") } func getModelAppConfig() { diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 9a5874956..bd6a01072 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -29,7 +29,9 @@ const ( tplModelConvertInfo = "repo/modelmanage/convertshowinfo" PYTORCH_ENGINE = 0 TENSORFLOW_ENGINE = 1 - MINDSPORE_ENGIN = 2 + MINDSPORE_ENGINE = 2 + PADDLE_ENGINE = 4 + MXNET_ENGINE = 6 ModelMountPath = "/model" CodeMountPath = "/code" DataSetMountPath = "/dataset" @@ -395,6 +397,20 @@ func createGpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context deleteLocalDir(relatetiveModelPath) dataActualPath = setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/dataset" } + } else if modelConvert.SrcEngine == PADDLE_ENGINE { + IMAGE_URL = setting.ModelConvert.GPU_PADDLE_IMAGE + if modelConvert.DestFormat == CONVERT_FORMAT_ONNX { + command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PaddleOnnxBootFile) + } else { + return errors.New("Not support the format.") + } + } else if modelConvert.SrcEngine == MXNET_ENGINE { + IMAGE_URL = setting.ModelConvert.GPU_MXNET_IMAGE + if modelConvert.DestFormat == CONVERT_FORMAT_ONNX { + command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.MXnetOnnxBootFile) + } else { + return errors.New("Not support the format.") + } } log.Info("dataActualPath=" + dataActualPath) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 92eefca2e..26e79b04c 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -103,7 +103,7 @@
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow{{else if eq .SrcEngine 2}}MindSpore {{end}} + {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow {{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle {{else if eq .SrcEngine 6}}MXNet {{end}}
{{if eq .DestFormat 0}}ONNX {{else if eq .DestFormat 1}}TensorRT {{end}} @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdparams",".params",".json"]; for(var i =0; iPyTorch"; html +=""; html +=""; + html +=""; + html +=""; $('#SrcEngine').html(html); srcEngineChanged(); } From 5f471bf4d8e531e8006b9e69ff9c1988d45d8a6d Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 12:02:38 +0800 Subject: [PATCH 13/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 49 ++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index fe3e015ac..7acb82ade 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1431,6 +1431,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { + // for _, openIResourceDetail := range OpenIResourceDetail { + // aiCenterCode := reflect.ValueOf(&openIResourceDetail.AiCenterCode).Elem() + // } + // var resourceDetail models.ResourceDetail // if _, ok := resourceDetail[resourceQueue.AiCenterCode]; !ok { // resourceDetail.AiCenterCode = resourceQueue.AiCenterCode @@ -1466,6 +1470,17 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) } + // for _, resourceDetail := range OpenIResourceDetail { + // for _, resourceDetails := range OpenIResourceDetail { + // if resourceDetail.AiCenterCode == resourceDetails.AiCenterCode { + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceDetails.AccCardType + // cardTypeNum.Num = resourceDetails.CardsTotalNum + // cardTypeNum.ComputeResource = resourceDetails.ComputeResource + // resourceDetail.CardTypeAndNum = append(resourceDetail.CardTypeAndNum, cardTypeNum) + // } + // } + // } // ResourceAiCenterRes, err := models.GetResourceAiCenters() // if err != nil { @@ -1515,12 +1530,44 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { // } // } } + // aiCenterNum := make(map[string]map[string]int) + // // cardTypeNum := make(map[string]int) + // for _, openIResourceDetail := range OpenIResourceDetail { + // if _, ok := aiCenterNum[openIResourceDetail.AiCenterCode]; !ok { + // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum + // } else { + // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum + // } + // } + AiCenterCodeList := make(map[string]string) + CardTypeList := make(map[string]string) + AiCenterCardTypeNum := make(map[string]map[string]int) + for _, openIResourceDetail := range OpenIResourceDetail { + if _, ok := AiCenterCodeList[openIResourceDetail.AiCenterCode]; !ok { + AiCenterCodeList[openIResourceDetail.AiCenterCode] = openIResourceDetail.AiCenterCode + } + for k, _ := range AiCenterCodeList { + if AiCenterCardTypeNum[AiCenterCodeList[k]] == nil { + AiCenterCardTypeNum[AiCenterCodeList[k]] = make(map[string]int) + } + if openIResourceDetail.AiCenterCode == AiCenterCodeList[k] { + if _, ok := CardTypeList[openIResourceDetail.AccCardType]; !ok { + CardTypeList[openIResourceDetail.AccCardType] = openIResourceDetail.AccCardType + } + for i, _ := range CardTypeList { + if openIResourceDetail.AccCardType == CardTypeList[i] { + AiCenterCardTypeNum[AiCenterCodeList[k]][CardTypeList[i]] += openIResourceDetail.CardsTotalNum + } + } + } + } + } ctx.JSON(http.StatusOK, map[string]interface{}{ // "resourceOverviews": resourceOverviews, "OpenIResourceDetail": OpenIResourceDetail, "C2NetResourceDetail": C2NetResourceDetail, - // "resourceQueues": resourceQueues, + "AiCenterCardTypeNum": AiCenterCardTypeNum, }) } From 66a28d19117775e3636d367847b813146922ad43 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 16:25:14 +0800 Subject: [PATCH 14/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 137 ++++++++-------------------- 1 file changed, 40 insertions(+), 97 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 7acb82ade..ca414f805 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1431,27 +1431,13 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { - // for _, openIResourceDetail := range OpenIResourceDetail { - // aiCenterCode := reflect.ValueOf(&openIResourceDetail.AiCenterCode).Elem() - // } - - // var resourceDetail models.ResourceDetail - // if _, ok := resourceDetail[resourceQueue.AiCenterCode]; !ok { - // resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - // } else { - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceQueue.AccCardType - // cardTypeNum.Num = resourceQueue.CardsTotalNum - // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // } - var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode resourceDetail.AiCenterName = resourceQueue.AiCenterName resourceDetail.ComputeResource = resourceQueue.ComputeResource - resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) @@ -1465,98 +1451,54 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.AiCenterCode = resourceQueue.AiCenterCode resourceDetail.AiCenterName = resourceQueue.AiCenterName resourceDetail.ComputeResource = resourceQueue.ComputeResource - resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) } - // for _, resourceDetail := range OpenIResourceDetail { - // for _, resourceDetails := range OpenIResourceDetail { - // if resourceDetail.AiCenterCode == resourceDetails.AiCenterCode { - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceDetails.AccCardType - // cardTypeNum.Num = resourceDetails.CardsTotalNum - // cardTypeNum.ComputeResource = resourceDetails.ComputeResource - // resourceDetail.CardTypeAndNum = append(resourceDetail.CardTypeAndNum, cardTypeNum) - // } - // } - // } - - // ResourceAiCenterRes, err := models.GetResourceAiCenters() - // if err != nil { - // log.Error("Can not get ResourceAiCenterRes.", err) - // return - // } - // resourceOverviews := []models.ResourceOverview{} - // resourceOpenIOne := models.ResourceOverview{} - // resourceOpenITwo := models.ResourceOverview{} - // // resourceChengdu := models.ResourceOverview{} - - // for _, resourceQueue := range resourceQueues { - // if resourceQueue.Cluster == models.OpenICluster { - // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { - // resourceOpenIOne.Cluster = resourceQueue.Cluster - // resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode - // resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceQueue.AccCardType - // cardTypeNum.Num = resourceQueue.CardsTotalNum - // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) - // } - // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { - // resourceOpenITwo.Cluster = resourceQueue.Cluster - // resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode - // resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceQueue.AccCardType - // cardTypeNum.Num = resourceQueue.CardsTotalNum - // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) - // } - // } - // } - // resourceOverviews = append(resourceOverviews, resourceOpenIOne) - // resourceOverviews = append(resourceOverviews, resourceOpenITwo) - // // resourceOverviews = append(resourceOverviews, resourceChengdu) - // for _, resourceAiCenterRes := range ResourceAiCenterRes { - // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { - // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { - // test := models.ResourceOverview{} - // test.Cluster = models.C2NetCluster - // test.AiCenterCode = resourceAiCenterRes.AiCenterCode - // test.AiCenterName = resourceAiCenterRes.AiCenterName - // resourceOverviews = append(resourceOverviews, test) - // } - // } - } - // aiCenterNum := make(map[string]map[string]int) - // // cardTypeNum := make(map[string]int) - // for _, openIResourceDetail := range OpenIResourceDetail { - // if _, ok := aiCenterNum[openIResourceDetail.AiCenterCode]; !ok { - // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum - // } else { - // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum - // } - // } - AiCenterCodeList := make(map[string]string) + } + AiCenterNameList := make(map[string]string) CardTypeList := make(map[string]string) - AiCenterCardTypeNum := make(map[string]map[string]int) + openIResourceNum := make(map[string]map[string]int) for _, openIResourceDetail := range OpenIResourceDetail { - if _, ok := AiCenterCodeList[openIResourceDetail.AiCenterCode]; !ok { - AiCenterCodeList[openIResourceDetail.AiCenterCode] = openIResourceDetail.AiCenterCode + if _, ok := AiCenterNameList[openIResourceDetail.AiCenterName]; !ok { + AiCenterNameList[openIResourceDetail.AiCenterName] = openIResourceDetail.AiCenterName } - for k, _ := range AiCenterCodeList { - if AiCenterCardTypeNum[AiCenterCodeList[k]] == nil { - AiCenterCardTypeNum[AiCenterCodeList[k]] = make(map[string]int) + for k, _ := range AiCenterNameList { + if openIResourceNum[AiCenterNameList[k]] == nil { + openIResourceNum[AiCenterNameList[k]] = make(map[string]int) } - if openIResourceDetail.AiCenterCode == AiCenterCodeList[k] { + if openIResourceDetail.AiCenterName == AiCenterNameList[k] { if _, ok := CardTypeList[openIResourceDetail.AccCardType]; !ok { CardTypeList[openIResourceDetail.AccCardType] = openIResourceDetail.AccCardType } for i, _ := range CardTypeList { if openIResourceDetail.AccCardType == CardTypeList[i] { - AiCenterCardTypeNum[AiCenterCodeList[k]][CardTypeList[i]] += openIResourceDetail.CardsTotalNum + openIResourceNum[AiCenterNameList[k]][CardTypeList[i]] += openIResourceDetail.CardsTotalNum + } + } + } + } + } + + c2NetAiCenterNameList := make(map[string]string) + c2NetCardTypeList := make(map[string]string) + c2NetResourceNum := make(map[string]map[string]int) + for _, c2NetResourceDetail := range C2NetResourceDetail { + if _, ok := c2NetAiCenterNameList[c2NetResourceDetail.AiCenterName]; !ok { + c2NetAiCenterNameList[c2NetResourceDetail.AiCenterName] = c2NetResourceDetail.AiCenterName + } + for k, _ := range c2NetAiCenterNameList { + if c2NetResourceNum[c2NetAiCenterNameList[k]] == nil { + c2NetResourceNum[c2NetAiCenterNameList[k]] = make(map[string]int) + } + if c2NetResourceDetail.AiCenterName == c2NetAiCenterNameList[k] { + if _, ok := c2NetCardTypeList[c2NetResourceDetail.AccCardType]; !ok { + c2NetCardTypeList[c2NetResourceDetail.AccCardType] = c2NetResourceDetail.AccCardType + } + for i, _ := range c2NetCardTypeList { + if c2NetResourceDetail.AccCardType == c2NetCardTypeList[i] { + c2NetResourceNum[c2NetAiCenterNameList[k]][c2NetCardTypeList[i]] += c2NetResourceDetail.CardsTotalNum } } } @@ -1565,9 +1507,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { ctx.JSON(http.StatusOK, map[string]interface{}{ // "resourceOverviews": resourceOverviews, - "OpenIResourceDetail": OpenIResourceDetail, - "C2NetResourceDetail": C2NetResourceDetail, - "AiCenterCardTypeNum": AiCenterCardTypeNum, + // "OpenIResourceDetail": OpenIResourceDetail, + // "C2NetResourceDetail": C2NetResourceDetail, + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, }) } From d8f40329c650e53539bc1d1d925c0fe869af5aa3 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 16:30:35 +0800 Subject: [PATCH 15/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index ca414f805..de6ae9eca 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1421,6 +1421,12 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t } func GetCloudbrainResourceOverview(ctx *context.Context) { + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get GetDurationRecordBeginTime", err) + return + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() resourceQueues, err := models.GetCanUseCardInfo() if err != nil { log.Info("GetCanUseCardInfo err: %v", err) @@ -1509,8 +1515,9 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { // "resourceOverviews": resourceOverviews, // "OpenIResourceDetail": OpenIResourceDetail, // "C2NetResourceDetail": C2NetResourceDetail, - "openI": openIResourceNum, - "c2Net": c2NetResourceNum, + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, + "brainRecordBeginTime": brainRecordBeginTime, }) } From 4eeead973bbd379d6c8a751ca5402501a3e1b7fe Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 16:50:19 +0800 Subject: [PATCH 16/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index de6ae9eca..3ce46c60a 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1426,7 +1426,8 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Error("Can not get GetDurationRecordBeginTime", err) return } - brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + recordBeginTime := recordCloudbrainDuration[0].CreatedUnix + recordUpdateTime := time.Now().Unix() resourceQueues, err := models.GetCanUseCardInfo() if err != nil { log.Info("GetCanUseCardInfo err: %v", err) @@ -1512,12 +1513,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - // "resourceOverviews": resourceOverviews, - // "OpenIResourceDetail": OpenIResourceDetail, - // "C2NetResourceDetail": C2NetResourceDetail, - "openI": openIResourceNum, - "c2Net": c2NetResourceNum, - "brainRecordBeginTime": brainRecordBeginTime, + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, + "recordUpdateTime": recordUpdateTime, + "recordBeginTime": recordBeginTime, }) } From 4206670658a168bfdc2a32e49f5494af9a3de1ca Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 10:04:12 +0800 Subject: [PATCH 17/50] update --- routers/api/v1/api.go | 1 - routers/api/v1/repo/cloudbrain_dashboard.go | 131 ++++++++-------------------- 2 files changed, 35 insertions(+), 97 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 2964e87ce..b719db71e 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -602,7 +602,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) - m.Get("/resource_usage_rate", repo.GetCloudbrainResourceUsage) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 3ce46c60a..6824858fb 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1520,63 +1520,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { }) } -func GetCloudbrainResourceUsage(ctx *context.Context) { - recordBeginTime := time.Now().AddDate(0, 0, -6) - beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) - if err != nil { - log.Error("getCloudbrainTimePeroid error:", err) - return - } - cardUsageRes := make(map[string]int) - cardCanUsageRes := make(map[string]int) - cardUtilizationRate := make(map[string]int) - // cardDurationStatistics, err := models.GetCardDurationStatistics(beginTime, endTime) - cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ - BeginTime: beginTime, - EndTime: endTime, - }) - if err != nil { - log.Error("GetCardDurationStatistics error:", err) - return - } - - for _, cloudbrainStat := range cardDurationStatistics { - if cloudbrainStat.TotalCanUse { - if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration - } else { - cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration - } - } else { - if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration - } else { - cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration - } - } - } - for k, v := range cardCanUsageRes { - for j, i := range cardUsageRes { - if k == j { - cardUtilizationRate[k] = i / v - } - } - } - - ctx.JSON(http.StatusOK, map[string]interface{}{ - "cardDurationStatistics": cardDurationStatistics, - "cardUsageRes": cardUsageRes, - "cardCanUsageRes": cardCanUsageRes, - "cardUtilizationRate": cardUtilizationRate, - }) - -} - func GetCloudbrainResourceUsageDetail(ctx *context.Context) { aiCenterCode := ctx.QueryTrim("aiCenterCode") - log.Info("aiCenterCode: %v", aiCenterCode) if aiCenterCode == "" { - aiCenterCode = "OpenIOne" + aiCenterCode = models.AICenterOfCloudBrainOne } beginTime, endTime := getBeginAndEndTime(ctx) dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) @@ -1713,40 +1660,16 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain } else { usageDuration += cloudbrainStatistic.CardsTotalDuration } - // if cloudbrainStatistic.TotalCanUse { - // if _, ok := aiCenterTotalDuration[Date]; !ok { - // aiCenterTotalDuration[Date] = cloudbrainStatistic.CardsTotalDuration - // } else { - // aiCenterTotalDuration[Date] += cloudbrainStatistic.CardsTotalDuration - // } - // } else { - // if _, ok := aiCenterUsageDuration[Date]; !ok { - // aiCenterUsageDuration[Date] = cloudbrainStatistic.CardsTotalDuration - // } else { - // aiCenterUsageDuration[Date] += cloudbrainStatistic.CardsTotalDuration - // } - // } } } - // ResourceAiCenterRes, err := models.GetResourceAiCenters() - // if err != nil { - // log.Error("Can not get ResourceAiCenterRes.", err) - // return nil, nil, nil - // } - // for _, v := range ResourceAiCenterRes { - // if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { - // aiCenterUsageDuration[v.AiCenterCode] = 0 - // } - // } - - // for k, v := range aiCenterTotalDuration { - // for i, j := range aiCenterUsageDuration { - // if k == i { - // aiCenterUsageRate[k] = float64(j) / float64(v) - // } - // } - // } - // usageRate = float64(usageDuration) / float64(totalDuration) + if totalDuration == 0 || usageDuration == 0 { + usageRate = 0 + } else { + usageRate = float64(usageDuration) / float64(totalDuration) + } + // if + // usageRate, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", float32(usageDuration)/float32(totalDuration)), 64) + // totalUsageRate = totalUse / totalCanUse return totalDuration, usageDuration, usageRate } @@ -1817,8 +1740,8 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati } } } - // totalCanUse := float64(0) - // totalUse := float64(0) + totalCanUse := float32(0) + totalUse := float32(0) totalUsageRate := float32(0) for k, v := range OpenITotalDuration { for i, j := range OpenIUsageDuration { @@ -1827,7 +1750,19 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati } } } - // totalUsageRate := totalUse / totalCanUse + for _, v := range OpenITotalDuration { + totalCanUse += float32(v) + } + for _, v := range OpenIUsageRate { + totalUse += float32(v) + } + if totalCanUse == 0 || totalUse == 0 { + totalUsageRate = 0 + } else { + totalUsageRate = totalUse / totalCanUse + } + // totalUsageRate = totalUse / totalCanUse + // strconv.FormatFloat(*100, 'f', 4, 64) + "%" OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration @@ -1919,13 +1854,17 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC } - // for k, v := range hourTimeTotalDuration { - // for i, j := range hourTimeUsageDuration { - // if k == i { - // hourTimeUsageRate[k] = float64(j) / float64(v) - // } - // } - // } + for k, v := range hourTimeTotalDuration { + for i, j := range hourTimeUsageDuration { + if k == i { + if v == 0 || j == 0 { + hourTimeUsageRate[k] = 0 + } else { + hourTimeUsageRate[k] = float64(j) / float64(v) + } + } + } + } hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration From 6e65b7144d1e8dd36d1589244e84fd170dd2ff10 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 10:53:53 +0800 Subject: [PATCH 18/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 6824858fb..691fe5002 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1728,16 +1728,18 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { - // if v.AiCenterCode != models.AICenterOfCloudBrainTwo { - if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { - C2NetUsageDuration[v.AiCenterName] = 0 - } - // } - } else { + // if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { + if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { OpenIUsageDuration[v.AiCenterName] = 0 } + if _, ok := OpenITotalDuration[v.AiCenterName]; !ok { + OpenITotalDuration[v.AiCenterName] = 0 + } + } else { + if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { + C2NetUsageDuration[v.AiCenterName] = 0 + } } } totalCanUse := float32(0) @@ -1773,6 +1775,13 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, totalUsageRate } +func cutString(str string, lens int) string { + if len(str) < lens { + return str + } + return str[:lens] +} + func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { now := time.Now() endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) From 3695fd996c94557a72e78a1f22055e82afbe0955 Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 21 Oct 2022 16:40:38 +0800 Subject: [PATCH 19/50] added --- models/ai_model_manage.go | 2 +- routers/repo/ai_model_convert.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a88da8fe5..d9adda2dc 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -88,7 +88,7 @@ type AiModelQueryOptions struct { } func (a *AiModelConvert) IsGpuTrainTask() bool { - if a.SrcEngine == 0 || a.SrcEngine == 1 { + if a.SrcEngine == 0 || a.SrcEngine == 1 || a.SrcEngine == 4 || a.SrcEngine == 6 { return true } return false diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index bd6a01072..13b02f9d0 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/set ting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" uuid "github.com/satori/go.uuid" From e1a454d5704835a3a78802eb2b8ffac74eaee892 Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 21 Oct 2022 16:43:59 +0800 Subject: [PATCH 20/50] add --- routers/repo/ai_model_convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 13b02f9d0..bd6a01072 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/set ting" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" uuid "github.com/satori/go.uuid" From dc36c0d3a899e12203bf00d6fa3ce831f017b100 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 16:53:05 +0800 Subject: [PATCH 21/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 691fe5002..1f4ea1ca3 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -122,7 +122,7 @@ func GetOverviewDuration(ctx *context.Context) { now := time.Now() endTime := now page := 1 - pagesize := 10000 + pagesize := 1000 count := pagesize worker_server_num := 1 cardNum := 1 From 97b60991194d44421f5de1a90eebe64e7f73c010 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 18:02:51 +0800 Subject: [PATCH 22/50] fix-bug --- models/cloudbrain.go | 4 +- routers/api/v1/repo/cloudbrain_dashboard.go | 87 +++++++++++++---------------- 2 files changed, 40 insertions(+), 51 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6135dac40..3f58284fd 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2296,9 +2296,9 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.Limit(opts.PageSize, start) } - sess.OrderBy("cloudbrain.created_unix DESC") + // sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 935006476..81c795087 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -120,9 +120,6 @@ func GetOverviewDuration(ctx *context.Context) { recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix now := time.Now() endTime := now - page := 1 - pagesize := 1000 - count := pagesize worker_server_num := 1 cardNum := 1 durationAllSum := int64(0) @@ -138,54 +135,46 @@ func GetOverviewDuration(ctx *context.Context) { c2NetDuration := int64(0) cDCenterDuration := int64(0) - for count == pagesize && count != 0 { - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: pagesize, - }, - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - models.LoadSpecs4CloudbrainInfo(cloudbrains) - - for _, cloudbrain := range cloudbrains { - if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { - worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber - } else { - worker_server_num = 1 - } - if cloudbrain.Cloudbrain.Spec == nil { - cardNum = 1 - } else { - cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum - } - duration := cloudbrain.Duration - durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration += duration - cloudBrainOneCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration += duration - cloudBrainTwoCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - c2NetDuration += duration - c2NetCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { - cDCenterDuration += duration - cDNetCardDuSum += durationSum - } + cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + models.LoadSpecs4CloudbrainInfo(cloudbrains) - durationAllSum += duration - cardDuSum += durationSum - count = len(cloudbrains) - page += 1 + for _, cloudbrain := range cloudbrains { + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber + } else { + worker_server_num = 1 } + if cloudbrain.Cloudbrain.Spec == nil { + cardNum = 1 + } else { + cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + duration := cloudbrain.Duration + durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration += duration + cloudBrainOneCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration + cloudBrainTwoCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration + c2NetCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDNetCardDuSum += durationSum + } + + durationAllSum += duration + cardDuSum += durationSum } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, From 4964fe5364d432cf57119e1113f7d9e2c545d355 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 24 Oct 2022 10:27:56 +0800 Subject: [PATCH 23/50] merge --- routers/routes/routes.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index be603785f..899918339 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -20,10 +20,6 @@ import ( "code.gitea.io/gitea/routers/modelapp" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/modules/slideimage" "code.gitea.io/gitea/routers/image" From fc5692e781ce2abea4af8386c968a1ae75038e8e Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 24 Oct 2022 11:48:12 +0800 Subject: [PATCH 24/50] update --- models/cloudbrain_static.go | 34 +---------------------------- routers/api/v1/repo/cloudbrain_dashboard.go | 24 +++++++------------- routers/repo/cloudbrain_statistic.go | 2 -- 3 files changed, 9 insertions(+), 51 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 3f2dcf94a..3bc6ad296 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,17 +38,6 @@ type TaskDetail struct { WorkServerNum int64 `json:"WorkServerNum"` Spec *Specification `json:"Spec"` } -type CardTypeAndNum struct { - CardType string `json:"CardType"` - Num int `json:"Num"` - ComputeResource string `json:"computeResource"` -} -type ResourceOverview struct { - Cluster string `json:"cluster"` - AiCenterName string `json:"aiCenterName"` - AiCenterCode string `json:"aiCenterCode"` - CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` -} type CloudbrainDurationStatistic struct { ID int64 `xorm:"pk autoincr"` @@ -78,7 +67,7 @@ type DurationStatisticOptions struct { type DurationRateStatistic struct { AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` - UsageRate map[string]float32 `json:"UsageRate"` + UsageRate map[string]float64 `json:"UsageRate"` } type ResourceDetail struct { QueueCode string @@ -89,15 +78,8 @@ type ResourceDetail struct { AccCardType string CardsTotalNum int IsAutomaticSync bool - // CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` } -// type DateCloudbrainStatistic struct { -// Date string `json:"date"` -// AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` -// AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` -// AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` -// } type DateUsageStatistic struct { Date string `json:"date"` UsageDuration int `json:"usageDuration"` @@ -347,26 +329,12 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - // var cond = builder.NewCond() - // cond = cond.And( - // builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), - // ) sess.OrderBy("resource_queue.id ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") } return ResourceQueues, nil - // Cols("queue_code", "cluster", "ai_center_name", "ai_center_code", "compute_resource", "acc_card_type", "cards_total_num") - // sess := x.NewSession() - // defer sess.Close() - // sess.OrderBy("resource_queue.id ASC limit 1") - // cloudbrains := make([]*CloudbrainInfo, 0) - // if err := sess.Table(&Cloudbrain{}).Unscoped(). - // Find(&cloudbrains); err != nil { - // log.Info("find error.") - // } - // return cloudbrains, nil } func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0ae516750..0bfd2a825 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1663,10 +1663,10 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain return totalDuration, usageDuration, usageRate } -func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float32) { +func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float64) { OpenITotalDuration := make(map[string]int) OpenIUsageDuration := make(map[string]int) - OpenIUsageRate := make(map[string]float32) + OpenIUsageRate := make(map[string]float64) C2NetTotalDuration := make(map[string]int) C2NetUsageDuration := make(map[string]int) OpenIDurationRate := models.DurationRateStatistic{} @@ -1717,7 +1717,6 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - // if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { OpenIUsageDuration[v.AiCenterName] = 0 @@ -1731,36 +1730,33 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati } } } - totalCanUse := float32(0) - totalUse := float32(0) - totalUsageRate := float32(0) + totalCanUse := float64(0) + totalUse := float64(0) + totalUsageRate := float64(0) for k, v := range OpenITotalDuration { for i, j := range OpenIUsageDuration { if k == i { - OpenIUsageRate[k] = float32(j) / float32(v) + OpenIUsageRate[k] = float64(j) / float64(v) } } } for _, v := range OpenITotalDuration { - totalCanUse += float32(v) + totalCanUse += float64(v) } for _, v := range OpenIUsageRate { - totalUse += float32(v) + totalUse += float64(v) } if totalCanUse == 0 || totalUse == 0 { totalUsageRate = 0 } else { totalUsageRate = totalUse / totalCanUse } - // totalUsageRate = totalUse / totalCanUse - // strconv.FormatFloat(*100, 'f', 4, 64) + "%" OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration OpenIDurationRate.UsageRate = OpenIUsageRate C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration - // C2NetDurationRate.TotalUsageRate = totalUsageRate return OpenIDurationRate, C2NetDurationRate, totalUsageRate } @@ -1846,10 +1842,6 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC if _, ok := hourTimeTotalDuration[v]; !ok { hourTimeTotalDuration[v] = 0 } - // if _, ok := hourTimeUsageRate[v]; !ok { - // hourTimeUsageRate[v] = 0 - // } - } for k, v := range hourTimeTotalDuration { diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index da1849096..0cc3fc317 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -42,7 +42,6 @@ func CloudbrainDurationStatisticHour() { log.Info("GetSpecByAiCenterCodeAndType err: %v", err) return } - log.Info("cloudbrain: %s", cloudbrain) if cloudbrain != nil { totalCanUse := false if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalCanUse); err != nil { @@ -107,7 +106,6 @@ func CloudbrainDurationStatisticHour() { func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { var WorkServerNumber int var AccCardsNum int - // cloudBrainCardRes := make(map[string]int) cloudBrainAiCenterCodeList := make(map[string]string) cloudBrainCardTypeList := make(map[string]string) cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) From 409b4be0dc5f653babad9d3dc36e9da54a8e7e0f Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 24 Oct 2022 16:25:01 +0800 Subject: [PATCH 25/50] update --- modules/cron/tasks_basic.go | 2 +- routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 13 ++++++------- routers/repo/cloudbrain_statistic.go | 4 ---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 958595960..f9661b892 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -270,7 +270,7 @@ func registerHandleCloudbrainDurationStatistic() { RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "55 59 * * * ?", + Schedule: "59 59 * * * ?", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.CloudbrainDurationStatisticHour() return nil diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b719db71e..c464f252c 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -603,6 +603,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) + m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0bfd2a825..6da731f5e 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1431,21 +1431,19 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) - // } else { - } if resourceQueue.Cluster == models.C2NetCluster { var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum @@ -1553,6 +1551,10 @@ func GetDurationRateStatistic(ctx *context.Context) { } +func CloudbrainDurationStatisticForTest(ctx *context.Context) { + repo.CloudbrainDurationStatisticHour() +} + func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { queryType := ctx.QueryTrim("type") now := time.Now() @@ -1656,9 +1658,6 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain } else { usageRate = float64(usageDuration) / float64(totalDuration) } - // if - // usageRate, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", float32(usageDuration)/float32(totalDuration)), 64) - // totalUsageRate = totalUse / totalCanUse return totalDuration, usageDuration, usageRate } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 0cc3fc317..703e0635f 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -25,15 +25,11 @@ func CloudbrainDurationStatisticHour() { } ciTasks2, err := models.GetCloudbrainCompleteByTime(beginTime, endTime) ciTasks := append(ciTasks1, ciTasks2...) - log.Info("beginTime: %s", beginTime) - log.Info("endTime: %s", endTime) if err != nil { log.Info("GetCloudbrainCompleteByTime err: %v", err) return } models.LoadSpecs4CloudbrainInfo(ciTasks) - log.Info("ciTasks here: %s", ciTasks) - log.Info("count here: %s", len(ciTasks)) cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime) for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo { for cardType, cardDuration := range CardTypeInfo { From 674f4701c998cbd6f81e31d28c9f4a424f9fd31a Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 10:57:32 +0800 Subject: [PATCH 26/50] add convert --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 26e79b04c..1802b3fdd 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdparams",".params",".json"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdiparams",".params",".json"]; for(var i =0; i Date: Tue, 25 Oct 2022 15:22:04 +0800 Subject: [PATCH 27/50] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 3 +++ routers/routes/routes.go | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 6da731f5e..9d410f1c2 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1553,6 +1553,9 @@ func GetDurationRateStatistic(ctx *context.Context) { func CloudbrainDurationStatisticForTest(ctx *context.Context) { repo.CloudbrainDurationStatisticHour() + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": 0, + }) } func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 899918339..4fe7c6622 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -389,7 +389,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/^:type(issues|pulls)$", reqSignIn, user.Issues) m.Get("/milestones", reqSignIn, reqMilestonesDashboardPageEnabled, user.Milestones) m.Get("/cloudbrains", reqSignIn, user.Cloudbrains) - m.Get("/duration", repo.CloudbrainDurationStatisticHour) // ***** START: User ***** m.Group("/user", func() { From 31261936608e110d64347bd55b6377277ebd447e Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 16:11:34 +0800 Subject: [PATCH 28/50] modified --- options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 773a338c1..ecd2bb038 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3216,7 +3216,7 @@ view_sample = View sample inference_output_path_rule = The inference output path is stored in the run parameter result_url. model_file_path_rule=The model file location is stored in the run parameter ckpt_url model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] -model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] +model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json] delete_task = Delete task task_delete_confirm = Are you sure you want to delete this task? Once this task is deleted, it cannot be recovered. operate_confirm = confirm diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8ba4d252d..74e46c6ea 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3234,7 +3234,7 @@ view_sample = 查看样例 inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。 model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。 model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] -model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] +model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json] delete_task = 删除任务 task_delete_confirm = 你确认删除该任务么？此任务一旦删除不可恢复。 operate_confirm = 确定操作 From 3dc3273f53afd2b6c03a7adebd60477ea73a3035 Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 16:16:54 +0800 Subject: [PATCH 29/50] modified --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 1802b3fdd..05b5306c8 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdiparams",".params",".json"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel",".pdiparams",".params",".json"]; for(var i =0; i Date: Thu, 27 Oct 2022 11:48:55 +0800 Subject: [PATCH 30/50] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 53 +++++- web_src/js/features/cloudbrainShow.js | 263 +++++++++++++++++++++------- 2 files changed, 254 insertions(+), 62 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index e95a1233d..462319168 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -548,14 +548,17 @@
-
+
{{$.i18n.Tr "repo.modelarts.download_log"}} - +
+ + 全屏查看 +
@@ -577,7 +580,11 @@
+ + + + 

                             

 
@@ -609,6 +616,41 @@
                 

             
         
+
+        

+            

+                

+                    
日志文件

+                    

+                        

+                            
+                            退出全屏
+                        

+                    

+                

+                

+                    
+                        
+                    
+                    
+                        
+                    
+                    
+                    

+                        

+                            

+                        

+                        

+                    

+                    
+                

+            

+        

         {{end}} {{template "base/paginate" .}}
     
     
@@ -715,12 +757,17 @@
 
         
     
+    
 
 {{template "base/footer" .}}
 
 
 
 
\ No newline at end of file
diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl
index 84d16d00c..37dc9c83a 100755
--- a/templates/repo/cloudbrain/benchmark/show.tmpl
+++ b/templates/repo/cloudbrain/benchmark/show.tmpl
@@ -524,14 +524,17 @@
                         
                     
                     

-                        
                          

@@ -549,12 +552,16 @@
                              

                              

-                                 

-                                     

-                                 

-                                 
-                                 
-                                 

+                                

+                                    

+                                

+                                
+                                
+                                
+                                
+                                	                                
+                                
+                                

                              

  
                          

@@ -565,6 +572,7 @@
                 
             
         
+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index 546f16848..79312ed1e 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -535,14 +535,17 @@
                     
 
                     

-                        

+                        

                              
                                  
                                  {{$.i18n.Tr "repo.modelarts.download_log"}}
                              
- 
+                             

+                                
+                                全屏查看
+                            

                          

                          

@@ -559,13 +562,17 @@
                                  
                              

                              

-                                 

+                                style="height: 300px !important; overflow: auto;">
+                                

                                      

-                                 

-                                 
-                                 
-                                 

+                                

+                                
+                                
+                                
+                                
+                                	                                
+                                
+                                

                              

  
                          

@@ -589,7 +596,7 @@
                 

             
         
-        
+        {{template "custom/max_log" .}}
         {{end}}
     
 
diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl
index a54989df4..315a84f45 100644
--- a/templates/repo/cloudbrain/trainjob/show.tmpl
+++ b/templates/repo/cloudbrain/trainjob/show.tmpl
@@ -524,14 +524,17 @@
                     
 
                     

-                       

+                       

                             
                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-
+                            

+                                
+                                全屏查看
+                            

                         

                         

                             
@@ -552,7 +555,11 @@
                                     

                                 

                                 
-                                
+                                
+                                
+                                
+                                	                                
+                                
                                 

                             

 
@@ -581,6 +588,7 @@
                 

             
         
+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl
index 67c415488..a22cb1ab7 100755
--- a/templates/repo/grampus/trainjob/show.tmpl
+++ b/templates/repo/grampus/trainjob/show.tmpl
@@ -520,14 +520,17 @@
                         
                     
                     

-                        

+                        

                             
                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-
+                            

+                                
+                                全屏查看
+                            

                         

                         

                             
@@ -548,7 +551,11 @@
                                     

                                 

                                 
-                                
+                                
+                                
+                                
+                                                           
+                                
                                 

                             

 
@@ -576,6 +583,7 @@
                 

             
         
+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl
index 7d671ae4c..c8ff73b2d 100644
--- a/templates/repo/modelarts/inferencejob/show.tmpl
+++ b/templates/repo/modelarts/inferencejob/show.tmpl
@@ -460,14 +460,17 @@ td, th {
                         
 
                         

-                            

+                            

                                 
                                     
                                     {{$.i18n.Tr "repo.modelarts.download_log"}}
                                 
-
+                                

+                                    
+                                    全屏查看
+                                

                             

                             

                                     
@@ -486,8 +489,12 @@ td, th {
                                     

                                         

                                     

-                                    
-                                    
+                                    
+                                    
+                                    
+                                    
+                                    	                                
+                                    
                                     

                                 

 
@@ -510,12 +517,8 @@ td, th {
                 

             

         
+        {{template "custom/max_log" .}}
         {{end}}
-
-
-
-
-
     
     
 
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl
index 462319168..e86848cc9 100755
--- a/templates/repo/modelarts/trainjob/show.tmpl
+++ b/templates/repo/modelarts/trainjob/show.tmpl
@@ -616,41 +616,7 @@
                 
             
         
-
-        

-            

-                

-                    
日志文件

-                    

-                        

-                            
-                            退出全屏
-                        

-                    

-                

-                

-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                    

-                        

-                            

-                        

-                        

-                    

-                    
-                

-            

-        

+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
@@ -764,10 +730,6 @@