From a8341fcb425841b4babac0087c4794dfea024d1b Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 14 Oct 2022 17:51:05 +0800 Subject: [PATCH] update --- models/cloudbrain_spec.go | 4 +- models/cloudbrain_static.go | 69 ++++++++++++++-- routers/api/v1/api.go | 4 + routers/api/v1/repo/cloudbrain_dashboard.go | 81 +++++++++++++++++- routers/repo/cloudbrain_statistic.go | 123 ++++++++++++++-------------- 5 files changed, 208 insertions(+), 73 deletions(-) diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go index c32e4b0fd..49a4d603e 100644 --- a/models/cloudbrain_spec.go +++ b/models/cloudbrain_spec.go @@ -9,7 +9,7 @@ type CloudbrainSpec struct { SpecId int64 `xorm:"index"` SourceSpecId string AccCardsNum int - AccCardType string + AccCardType string `xorm:"index"` CpuCores int MemGiB float32 GPUMemGiB float32 @@ -19,7 +19,7 @@ type CloudbrainSpec struct { QueueId int64 QueueCode string Cluster string - AiCenterCode string + AiCenterCode string `xorm:"index"` AiCenterName string IsExclusive bool ExclusiveOrg string diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index b760fdc31..b8f1c8e62 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "strconv" "time" @@ -39,18 +40,20 @@ type TaskDetail struct { } type CloudbrainDurationStatistic struct { - ID int64 `xorm:"pk autoincr"` - Cluster string `xorm:"notnull"` + ID int64 `xorm:"pk autoincr"` + Cluster string AiCenterCode string AiCenterName string ComputeResource string AccCardType string - QueueCode string - CardsTotalNum int + TotalUse bool + TotalCanUse bool DateTime string + DayTime string HourTime int CardsTotalDuration int + CardsTotalNum int DeletedTime timeutil.TimeStamp `xorm:"deleted"` CreatedTime timeutil.TimeStamp `xorm:"created"` @@ -256,12 +259,8 @@ func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*C defer sess.Close() var cond = builder.NewCond() cond = cond.And( - builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, + builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}), ) - cond = cond.And( - builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}, - ) - sess.OrderBy("cloudbrain_spec.created_unix ASC limit 1") cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10) if err := sess.Table(&CloudbrainSpec{}).Where(cond). Find(&cloudbrainSpecs); err != nil { @@ -273,3 +272,55 @@ func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*C func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { return xStatistic.Insert(cloudbrainDurationStatistic) } + +func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, tatalUse bool, totalCanUse bool) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_use = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, tatalUse, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func GetCanUseCardInfo() ([]*ResourceQueue, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), + ) + ResourceQueues := make([]*ResourceQueue, 0, 10) + if err := sess.Table(&ResourceQueue{}).Where(cond). + Find(&ResourceQueues); err != nil { + log.Info("find error.") + } + return ResourceQueues, nil +} +func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse bool, totalCanUse bool) ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain_duration_statistic.created_time": beginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_time": endTime.Unix()}), + ) + cond = cond.And( + builder.And(builder.Eq{"cloudbrain_duration_statistic.total_use": totalUse}, builder.Eq{"cloudbrain_duration_statistic.total_can_use": totalCanUse}), + ) + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10) + if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond). + Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e1d725ed..06be12e92 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -599,6 +599,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData) m.Get("/waitting_top_data", repo.GetWaittingTop) m.Get("/running_top_data", repo.GetRunningTop) + + m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) + m.Get("/resource_usage", repo.GetCloudbrainResourceUsage) + m.Get("/resource_usage_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 54c0ddc20..7d348d578 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -545,7 +545,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0) beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) if err != nil { - log.Error("Parameter is wrong", err) + log.Error("getCloudbrainTimePeroid error:", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } @@ -1403,3 +1403,82 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t return beginTime, endTime, nil } + +func GetCloudbrainResourceOverview(ctx *context.Context) { + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "resourceQueues": resourceQueues, + }) + +} + +func GetCloudbrainResourceUsage(ctx *context.Context) { + recordBeginTime := time.Now().AddDate(0, 0, -6) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("getCloudbrainTimePeroid error:", err) + return + } + cardUsageRes := make(map[string]int) + cardCanUsageRes := make(map[string]int) + cardUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, true, false) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + cardCanUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, false, true) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + + for _, cloudbrainStat := range cardUseInfo { + if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } + } + + for _, cloudbrainStat := range cardCanUseInfo { + if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cardUseInfo": cardUseInfo, + "cardCanUseInfo": cardCanUseInfo, + "cardUsageRes": cardUsageRes, + "cardCanUsageRes": cardCanUsageRes, + }) + +} +func GetCloudbrainResourceUsageDetail(ctx *context.Context) { + recordBeginTime := time.Now().AddDate(0, 0, -6) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("getCloudbrainTimePeroid error:", err) + return + } + totalUse := true + totalCanUse := false + cardDurationStatisticsInfo, err := models.GetCardDurationStatistics(beginTime, endTime, totalUse, totalCanUse) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cardDurationStatisticsInfo": cardDurationStatisticsInfo, + "beginTime": beginTime, + "endTime": endTime, + }) + +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 0bc4938ff..c7546c1c6 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -6,6 +6,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" ) func CloudbrainDurationStatistic() { @@ -16,28 +17,9 @@ func CloudbrainDurationStatistic() { } func CloudbrainDurationStatisticHour(ctx *context.Context) { - //获取规定时间段的云脑任务列表 - // page := ctx.QueryInt("page") - // pageSize := ctx.QueryInt("pagesize") - // if page <= 0 { - // page = 1 - // } - // if pageSize <= 0 { - // pageSize = 10 - // } - - // cloudBrainDurationRes := make(map[string]map[string]int) - // cloudBrainOneCardRes := make(map[string]int) - // cloudBrainTwoCardRes := make(map[string]int) - // c2NetCardRes := make(map[string]int) - // cDNetCenterCardRes := make(map[string]int) - // var WorkServerNumber int - // var AccCardsNum int - // endTime := time.Now().Unix() - // beginTime := time.Now().AddDate(0, 0, -1).Unix() - - // hour := time.Now().Hour() - // tStr := time.Now().Format("2006-01-02 15:04:05") + hourTime := time.Now().Hour() + dateTime := time.Now().Format("2006-01-02 15:04:05") + dayTime := time.Now().Format("2006-01-02") currentTime := time.Now() m, _ := time.ParseDuration("-1h") @@ -71,50 +53,69 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { log.Info("GetSpecByAiCenterCodeAndType err: %v", err) return } - cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - DateTime: date, - HourTime: userNumber, - Cluster: cloudbrain[0].Cluster, - AiCenterName: cloudbrain[0].AiCenterName, - AiCenterCode: centerCode, - ComputeResource: cloudbrain[0].ComputeResource, - AccCardType: cardType, - CardsTotalNum: mirrorRepositoryNumber, - CardsTotalDuration: cardDuration, - QueueCode: cloudbrain[0].QueueCode, - CreatedTime: privateRepositoryNumer, - UpdatedTime: publicRepositoryNumer, + log.Info("cloudbrain: %s", cloudbrain) + if cloudbrain != nil { + totalUse := true + totalCanUse := false + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalUse, totalCanUse); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: cloudbrain[0].Cluster, + AiCenterName: cloudbrain[0].AiCenterName, + AiCenterCode: centerCode, + ComputeResource: cloudbrain[0].ComputeResource, + AccCardType: cardType, + CardsTotalDuration: cardDuration, + CreatedTime: timeutil.TimeStampNow(), + TotalUse: true, + TotalCanUse: false, + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } } } } - //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 - - // cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - // DateTime: date, - // HourTime: userNumber, - // Cluster: repositorySize, - // AiCenterName: allDatasetSize, - // AiCenterCode: organizationNumber, - // ComputeResource: repositoryNumer, - // AccCardType: forkRepositoryNumber, - // CardsTotalNum: mirrorRepositoryNumber, - // CardsTotalDuration: privateRepositoryNumer, - // QueueCode: publicRepositoryNumer, - // CreatedTime: privateRepositoryNumer, - // UpdatedTime: publicRepositoryNumer, - // } - - // if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { - // log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) - // } - // log.Info("cloudBrainDurationRes2: %s", cloudBrainDurationRes) - // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) - // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) - // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) - // log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) - // log.Info("c2NetCardRes: %s", c2NetCardRes) + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + log.Info("resourceQueues here: %s", resourceQueues) + for _, resourceQueue := range resourceQueues { + totalUse := false + totalCanUse := true + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalUse, totalCanUse); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cardsTotalDuration := resourceQueue.CardsTotalNum * 1 * 60 * 60 + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: resourceQueue.Cluster, + AiCenterName: resourceQueue.AiCenterName, + AiCenterCode: resourceQueue.AiCenterCode, + ComputeResource: resourceQueue.ComputeResource, + AccCardType: resourceQueue.AccCardType, + CardsTotalDuration: cardsTotalDuration, + CardsTotalNum: resourceQueue.CardsTotalNum, + CreatedTime: timeutil.TimeStampNow(), + TotalUse: false, + TotalCanUse: true, + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + } log.Info("finish summary cloudbrainDurationStat") }