diff --git a/models/cloudbrain.go b/models/cloudbrain.go index d3e93a437..989d0905b 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2281,9 +2281,9 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.Limit(opts.PageSize, start) } - sess.OrderBy("cloudbrain.created_unix DESC") + // sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go index c32e4b0fd..49a4d603e 100644 --- a/models/cloudbrain_spec.go +++ b/models/cloudbrain_spec.go @@ -9,7 +9,7 @@ type CloudbrainSpec struct { SpecId int64 `xorm:"index"` SourceSpecId string AccCardsNum int - AccCardType string + AccCardType string `xorm:"index"` CpuCores int MemGiB float32 GPUMemGiB float32 @@ -19,7 +19,7 @@ type CloudbrainSpec struct { QueueId int64 QueueCode string Cluster string - AiCenterCode string + AiCenterCode string `xorm:"index"` AiCenterName string IsExclusive bool ExclusiveOrg string diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 48df111a0..19e55fb6d 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "strconv" "time" @@ -38,6 +39,60 @@ type TaskDetail struct { Spec *Specification `json:"Spec"` } +type CloudbrainDurationStatistic struct { + ID int64 `xorm:"pk autoincr"` + Cluster string + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + + DateTime string + DayTime string + HourTime int + CardsUseDuration int + CardsTotalDuration int + CardsTotalNum int + + DeletedUnix timeutil.TimeStamp `xorm:"deleted"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` +} +type DurationStatisticOptions struct { + BeginTime time.Time + EndTime time.Time + AiCenterCode string +} + +type DurationRateStatistic struct { + AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` + AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` + UsageRate map[string]float64 `json:"UsageRate"` +} +type ResourceDetail struct { + QueueCode string + Cluster string `xorm:"notnull"` + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + CardsTotalNum int + IsAutomaticSync bool +} + +type DateUsageStatistic struct { + Date string `json:"date"` + UsageDuration int `json:"usageDuration"` + TotalDuration int `json:"totalDuration"` + UsageRate float64 `json:"usageRate"` +} + +type HourTimeStatistic struct { + HourTimeUsageDuration map[string]int `json:"hourTimeUsageDuration"` + HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"` + HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"` +} + func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -199,3 +254,121 @@ func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string } return dateHourMap, nil } + +func GetCloudbrainRunning() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobRunning)}, + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}), + ) + cond = cond.Or( + builder.Eq{"cloudbrain.status": string(JobRunning)}, + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*CloudbrainSpec, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}), + ) + cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10) + if err := sess.Table(&CloudbrainSpec{}).Where(cond). + Find(&cloudbrainSpecs); err != nil { + log.Info("find error.") + } + return cloudbrainSpecs, nil +} + +func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { + return xStatistic.Insert(cloudbrainDurationStatistic) +} + +func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ?", date, hour, aiCenterCode, accCardType).Delete(&CloudbrainDurationStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func GetCanUseCardInfo() ([]*ResourceQueue, error) { + sess := x.NewSession() + defer sess.Close() + sess.OrderBy("resource_queue.id ASC") + ResourceQueues := make([]*ResourceQueue, 0, 10) + if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { + log.Info("find error.") + } + return ResourceQueues, nil +} + +func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + var cond = builder.NewCond() + if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 { + cond = cond.And( + builder.And(builder.Gte{"cloudbrain_duration_statistic.created_unix": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_unix": opts.EndTime.Unix()}), + ) + } + if opts.AiCenterCode != "" { + cond = cond.And( + builder.Eq{"cloudbrain_duration_statistic.ai_center_code": opts.AiCenterCode}, + ) + } + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10) + if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond). + Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} + +func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1") + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0) + if err := sess.Table(&CloudbrainDurationStatistic{}).Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} diff --git a/models/models.go b/models/models.go index ff64bfad2..b2b7a6472 100755 --- a/models/models.go +++ b/models/models.go @@ -183,6 +183,7 @@ func init() { new(UserMetrics), new(UserAnalysisPara), new(Invitation), + new(CloudbrainDurationStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 8dbc8d1ed..aac4f87c3 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -266,6 +266,17 @@ func registerSyncModelArtsTempJobs() { }) } +func registerHandleCloudbrainDurationStatistic() { + RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "1 0 * * * ?", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.CloudbrainDurationStatisticHour() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -293,4 +304,5 @@ func initBasicTasks() { registerCloudbrainPointDeductTask() registerHandleModelSafetyTask() + registerHandleCloudbrainDurationStatistic() } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e1d725ed..c464f252c 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -599,6 +599,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData) m.Get("/waitting_top_data", repo.GetWaittingTop) m.Get("/running_top_data", repo.GetRunningTop) + + m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) + m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) + m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) + m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 935006476..fdc8d64bb 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" "time" @@ -120,9 +121,6 @@ func GetOverviewDuration(ctx *context.Context) { recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix now := time.Now() endTime := now - page := 1 - pagesize := 1000 - count := pagesize worker_server_num := 1 cardNum := 1 durationAllSum := int64(0) @@ -138,54 +136,46 @@ func GetOverviewDuration(ctx *context.Context) { c2NetDuration := int64(0) cDCenterDuration := int64(0) - for count == pagesize && count != 0 { - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: pagesize, - }, - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - models.LoadSpecs4CloudbrainInfo(cloudbrains) - - for _, cloudbrain := range cloudbrains { - if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { - worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber - } else { - worker_server_num = 1 - } - if cloudbrain.Cloudbrain.Spec == nil { - cardNum = 1 - } else { - cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum - } - duration := cloudbrain.Duration - durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration += duration - cloudBrainOneCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration += duration - cloudBrainTwoCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - c2NetDuration += duration - c2NetCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { - cDCenterDuration += duration - cDNetCardDuSum += durationSum - } + cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + models.LoadSpecs4CloudbrainInfo(cloudbrains) - durationAllSum += duration - cardDuSum += durationSum - count = len(cloudbrains) - page += 1 + for _, cloudbrain := range cloudbrains { + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber + } else { + worker_server_num = 1 } + if cloudbrain.Cloudbrain.Spec == nil { + cardNum = 1 + } else { + cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + duration := cloudbrain.Duration + durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration += duration + cloudBrainOneCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration + cloudBrainTwoCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration + c2NetCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDNetCardDuSum += durationSum + } + + durationAllSum += duration + cardDuSum += durationSum } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, @@ -532,6 +522,21 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int } +func getPageDateCloudbrainDuration(dateUsageStatistic []models.DateUsageStatistic, page int, pagesize int) []models.DateUsageStatistic { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(dateUsageStatistic)-1 { + return nil + } + if end > len(dateUsageStatistic)-1 { + return dateUsageStatistic[begin:] + } else { + return dateUsageStatistic[begin:end] + } + +} + func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { queryType := ctx.QueryTrim("type") beginTimeStr := ctx.QueryTrim("beginTime") @@ -545,7 +550,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0) beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) if err != nil { - log.Error("Parameter is wrong", err) + log.Error("getCloudbrainTimePeroid error:", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } @@ -1403,3 +1408,424 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t return beginTime, endTime, nil } + +func GetCloudbrainResourceOverview(ctx *context.Context) { + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get GetDurationRecordBeginTime", err) + return + } + recordBeginTime := recordCloudbrainDuration[0].CreatedUnix + recordUpdateTime := time.Now().Unix() + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + OpenIResourceDetail := []models.ResourceDetail{} + C2NetResourceDetail := []models.ResourceDetail{} + for _, resourceQueue := range resourceQueues { + if resourceQueue.Cluster == models.OpenICluster { + var resourceDetail models.ResourceDetail + resourceDetail.QueueCode = resourceQueue.QueueCode + resourceDetail.Cluster = resourceQueue.Cluster + resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode + resourceDetail.ComputeResource = resourceQueue.ComputeResource + resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" + resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum + resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync + OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) + } + if resourceQueue.Cluster == models.C2NetCluster { + var resourceDetail models.ResourceDetail + resourceDetail.QueueCode = resourceQueue.QueueCode + resourceDetail.Cluster = resourceQueue.Cluster + resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode + resourceDetail.ComputeResource = resourceQueue.ComputeResource + resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" + resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum + resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync + C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) + } + } + openIResourceNum := make(map[string]map[string]int) + + for _, openIResourceDetail := range OpenIResourceDetail { + if _, ok := openIResourceNum[openIResourceDetail.AiCenterName]; !ok { + openIResourceNum[openIResourceDetail.AiCenterName] = make(map[string]int) + } + if _, ok := openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType]; !ok { + openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum + } else { + openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum + } + } + + c2NetResourceNum := make(map[string]map[string]int) + for _, c2NetResourceDetail := range C2NetResourceDetail { + if _, ok := c2NetResourceNum[c2NetResourceDetail.AiCenterName]; !ok { + c2NetResourceNum[c2NetResourceDetail.AiCenterName] = make(map[string]int) + } + if _, ok := c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType]; !ok { + c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType] = c2NetResourceDetail.CardsTotalNum + } else { + c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType] += c2NetResourceDetail.CardsTotalNum + } + + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, + "recordUpdateTime": recordUpdateTime, + "recordBeginTime": recordBeginTime, + }) +} + +func GetCloudbrainResourceUsageDetail(ctx *context.Context) { + aiCenterCode := ctx.QueryTrim("aiCenterCode") + if aiCenterCode == "" { + aiCenterCode = models.AICenterOfCloudBrainOne + } + beginTime, endTime := getBeginAndEndTime(ctx) + dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + return + } + hourCloudbrainDuration, err := getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query hourCloudbrainDuration.", err) + return + } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 36500 + } + pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "totalCount": count, + "pageDateCloudbrainDuration": pageDateCloudbrainDuration, + "hourCloudbrainDuration": hourCloudbrainDuration, + }) +} + +func GetDurationRateStatistic(ctx *context.Context) { + beginTime, endTime := getBeginAndEndTime(ctx) + OpenIDurationRate, C2NetDurationRate, totalUsageRate := getDurationStatistic(beginTime, endTime) + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "openIDurationRate": OpenIDurationRate, + "c2NetDurationRate": C2NetDurationRate, + "totalUsageRate": totalUsageRate, + }) + +} + +func CloudbrainDurationStatisticForTest(ctx *context.Context) { + repo.CloudbrainDurationStatisticHour() + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": 0, + }) +} + +func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { + queryType := ctx.QueryTrim("type") + now := time.Now() + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + + var beginTime time.Time + var endTime time.Time + var err error + if queryType != "" { + if queryType == "all" { + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get GetDurationRecordBeginTime", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return beginTime, endTime + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + } else if queryType == "today" { + beginTime = now.AddDate(0, 0, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + } else if queryType == "last_month" { + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return beginTime, endTime + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return beginTime, endTime + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return beginTime, endTime + } + if endTime.After(time.Now()) { + endTime = time.Now() + } + } + + } + return beginTime, endTime +} + +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (int, int, float64) { + totalDuration := int(0) + usageDuration := int(0) + usageRate := float64(0) + + for _, cloudbrainStatistic := range cloudbrainStatistics { + if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { + totalDuration += cloudbrainStatistic.CardsTotalDuration + usageDuration += cloudbrainStatistic.CardsUseDuration + } + } + if totalDuration == 0 || usageDuration == 0 { + usageRate = 0 + } else { + usageRate = float64(usageDuration) / float64(totalDuration) + } + + return totalDuration, usageDuration, usageRate +} + +func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float64) { + OpenITotalDuration := make(map[string]int) + OpenIUsageDuration := make(map[string]int) + OpenIUsageRate := make(map[string]float64) + + C2NetTotalDuration := make(map[string]int) + C2NetUsageDuration := make(map[string]int) + OpenIDurationRate := models.DurationRateStatistic{} + C2NetDurationRate := models.DurationRateStatistic{} + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return OpenIDurationRate, C2NetDurationRate, 0 + } + for _, cloudbrainStatistic := range cardDurationStatistics { + if cloudbrainStatistic.Cluster == models.OpenICluster { + if _, ok := OpenITotalDuration[cloudbrainStatistic.AiCenterName]; !ok { + OpenITotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + OpenITotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } + if _, ok := OpenIUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { + OpenIUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration + } else { + OpenIUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration + } + } + if cloudbrainStatistic.Cluster == models.C2NetCluster { + if _, ok := C2NetTotalDuration[cloudbrainStatistic.AiCenterName]; !ok { + C2NetTotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + C2NetTotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } + if _, ok := C2NetUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { + C2NetUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration + } else { + C2NetUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration + } + } + } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return OpenIDurationRate, C2NetDurationRate, 0 + } + for _, v := range ResourceAiCenterRes { + if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { + if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { + OpenIUsageDuration[v.AiCenterName] = 0 + } + if _, ok := OpenITotalDuration[v.AiCenterName]; !ok { + OpenITotalDuration[v.AiCenterName] = 0 + } + } else { + if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { + C2NetUsageDuration[v.AiCenterName] = 0 + } + } + } + totalCanUse := float64(0) + totalUse := float64(0) + totalUsageRate := float64(0) + for k, v := range OpenITotalDuration { + for i, j := range OpenIUsageDuration { + if k == i { + OpenIUsageRate[k] = float64(j) / float64(v) + } + } + } + for _, v := range OpenITotalDuration { + totalCanUse += float64(v) + } + for _, v := range OpenIUsageRate { + totalUse += float64(v) + } + if totalCanUse == 0 || totalUse == 0 { + totalUsageRate = 0 + } else { + totalUsageRate = totalUse / totalCanUse + } + + OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration + OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration + OpenIDurationRate.UsageRate = OpenIUsageRate + C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration + C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration + return OpenIDurationRate, C2NetDurationRate, totalUsageRate +} + +func cutString(str string, lens int) string { + if len(str) < lens { + return str + } + return str[:lens] +} + +func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { + now := time.Now() + endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + if endTimeTemp.Equal(endTime) { + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + } + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + AiCenterCode: aiCenterCode, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return nil, 0, err + } + + dayCloudbrainInfo := make([]models.DateUsageStatistic, 0) + count := 0 + for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { + TotalDuration, UsageDuration, UsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) + dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateUsageStatistic{ + Date: endTimeTemp.Format("2006/01/02"), + UsageDuration: UsageDuration, + TotalDuration: TotalDuration, + UsageRate: UsageRate, + }) + endTime = endTimeTemp + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + if endTimeTemp.Before(beginTime) && beginTime.Before(endTime) { + endTimeTemp = beginTime + } + count += 1 + } + return dayCloudbrainInfo, count, nil +} + +func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) (models.HourTimeStatistic, error) { + hourTimeTotalDuration := make(map[string]int) + hourTimeUsageDuration := make(map[string]int) + hourTimeUsageRate := make(map[string]float64) + hourTimeStatistic := models.HourTimeStatistic{} + + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return hourTimeStatistic, err + } + for _, cloudbrainStatistic := range cardDurationStatistics { + if cloudbrainStatistic.AiCenterCode == aiCenterCode { + if _, ok := hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok { + hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration + } else { + hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration + } + if _, ok := hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok { + hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsUseDuration + } else { + hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsUseDuration + } + } + } + hourTimeList := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"} + for _, v := range hourTimeList { + if _, ok := hourTimeUsageDuration[v]; !ok { + hourTimeUsageDuration[v] = 0 + } + if _, ok := hourTimeTotalDuration[v]; !ok { + hourTimeTotalDuration[v] = 0 + } + } + + for k, v := range hourTimeTotalDuration { + for i, j := range hourTimeUsageDuration { + if k == i { + if v == 0 || j == 0 { + hourTimeUsageRate[k] = 0 + } else { + hourTimeUsageRate[k] = float64(j) / float64(v) + } + } + } + } + + hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration + hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration + hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate + return hourTimeStatistic, nil +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go new file mode 100644 index 000000000..3814c2daf --- /dev/null +++ b/routers/repo/cloudbrain_statistic.go @@ -0,0 +1,169 @@ +package repo + +import ( + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" +) + +func CloudbrainDurationStatisticHour() { + + dateTime := time.Now().Format("2006-01-02 15:04:05") + dayTime := time.Now().Format("2006-01-02") + now := time.Now() + + currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()) + + m, _ := time.ParseDuration("-1h") + beginTime := currentTime.Add(m).Unix() + endTime := currentTime.Unix() + hourTime := currentTime.Add(m).Hour() + + ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime) + if err != nil { + log.Info("GetCloudbrainByTime err: %v", err) + return + } + specMap := make(map[string]*models.Specification) + models.LoadSpecs4CloudbrainInfo(ciTasks) + for _, cloudbrain := range ciTasks { + if _, ok := specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if cloudbrain.Cloudbrain.Spec != nil { + specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec + } + } + } + + cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime) + + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + cardsTotalDurationMap := make(map[string]int) + for _, resourceQueue := range resourceQueues { + cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterName+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60 + } + + for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo { + for cardType, cardDuration := range CardTypeInfo { + spec := specMap[centerCode+"/"+cardType] + if spec != nil { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + if _, ok := cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource]; !ok { + cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource] = 0 + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: spec.Cluster, + AiCenterName: spec.AiCenterName, + AiCenterCode: centerCode, + AccCardType: cardType, + ComputeResource: spec.ComputeResource, + CardsUseDuration: cardDuration, + CardsTotalDuration: cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource], + CreatedUnix: timeutil.TimeStampNow(), + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + delete(cardsTotalDurationMap, spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource) + } + } + } + + for key, cardsTotalDuration := range cardsTotalDurationMap { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[2], strings.Split(key, "/")[3]); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: strings.Split(key, "/")[0], + AiCenterName: strings.Split(key, "/")[1], + AiCenterCode: strings.Split(key, "/")[2], + AccCardType: strings.Split(key, "/")[3], + ComputeResource: strings.Split(key, "/")[4], + CardsUseDuration: 0, + CardsTotalDuration: cardsTotalDuration, + CreatedUnix: timeutil.TimeStampNow(), + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + } + + log.Info("finish summary cloudbrainDurationStat") +} + +func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { + var WorkServerNumber int + var AccCardsNum int + cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) + for _, cloudbrain := range ciTasks { + + if cloudbrain.Cloudbrain.StartTime == 0 { + cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix + } + if cloudbrain.Cloudbrain.EndTime == 0 { + cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix + } + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber + } else { + WorkServerNumber = 1 + } + if cloudbrain.Cloudbrain.Spec == nil { + AccCardsNum = 1 + } else { + AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode]; !ok { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode] = make(map[string]int) + } + + if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } + } else { + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } + + } + } + + return cloudBrainCenterCodeAndCardType +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 4fd2246bd..fd8b274e6 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,17 +6,18 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/badge" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - badge_service "code.gitea.io/gitea/services/badge" - "code.gitea.io/gitea/services/reward" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/badge" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" + badge_service "code.gitea.io/gitea/services/badge" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/routers/modelapp" "code.gitea.io/gitea/modules/slideimage"