Browse Source

Merge pull request 'fix-2906' (#3083) from fix-2906 into V20221102

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/3083
Reviewed-by: zouap <zouap@pcl.ac.cn>
pull/3162/head
zouap 2 years ago
parent
commit
57e4c1c1a5
9 changed files with 846 additions and 59 deletions
  1. +2
    -2
      models/cloudbrain.go
  2. +2
    -2
      models/cloudbrain_spec.go
  3. +173
    -0
      models/cloudbrain_static.go
  4. +1
    -0
      models/models.go
  5. +12
    -0
      modules/cron/tasks_basic.go
  6. +5
    -0
      routers/api/v1/api.go
  7. +476
    -50
      routers/api/v1/repo/cloudbrain_dashboard.go
  8. +169
    -0
      routers/repo/cloudbrain_statistic.go
  9. +6
    -5
      routers/routes/routes.go

+ 2
- 2
models/cloudbrain.go View File

@@ -2281,9 +2281,9 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er
}
sess.Limit(opts.PageSize, start)
}
sess.OrderBy("cloudbrain.created_unix DESC")
// sess.OrderBy("cloudbrain.created_unix DESC")
cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum)
if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond).
if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond).
Find(&cloudbrains); err != nil {
return nil, 0, fmt.Errorf("Find: %v", err)
}


+ 2
- 2
models/cloudbrain_spec.go View File

@@ -9,7 +9,7 @@ type CloudbrainSpec struct {
SpecId int64 `xorm:"index"`
SourceSpecId string
AccCardsNum int
AccCardType string
AccCardType string `xorm:"index"`
CpuCores int
MemGiB float32
GPUMemGiB float32
@@ -19,7 +19,7 @@ type CloudbrainSpec struct {
QueueId int64
QueueCode string
Cluster string
AiCenterCode string
AiCenterCode string `xorm:"index"`
AiCenterName string
IsExclusive bool
ExclusiveOrg string


+ 173
- 0
models/cloudbrain_static.go View File

@@ -1,6 +1,7 @@
package models

import (
"fmt"
"strconv"
"time"

@@ -38,6 +39,60 @@ type TaskDetail struct {
Spec *Specification `json:"Spec"`
}

type CloudbrainDurationStatistic struct {
ID int64 `xorm:"pk autoincr"`
Cluster string
AiCenterCode string
AiCenterName string
ComputeResource string
AccCardType string

DateTime string
DayTime string
HourTime int
CardsUseDuration int
CardsTotalDuration int
CardsTotalNum int

DeletedUnix timeutil.TimeStamp `xorm:"deleted"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
type DurationStatisticOptions struct {
BeginTime time.Time
EndTime time.Time
AiCenterCode string
}

type DurationRateStatistic struct {
AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"`
AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"`
UsageRate map[string]float64 `json:"UsageRate"`
}
type ResourceDetail struct {
QueueCode string
Cluster string `xorm:"notnull"`
AiCenterCode string
AiCenterName string
ComputeResource string
AccCardType string
CardsTotalNum int
IsAutomaticSync bool
}

type DateUsageStatistic struct {
Date string `json:"date"`
UsageDuration int `json:"usageDuration"`
TotalDuration int `json:"totalDuration"`
UsageRate float64 `json:"usageRate"`
}

type HourTimeStatistic struct {
HourTimeUsageDuration map[string]int `json:"hourTimeUsageDuration"`
HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"`
HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"`
}

func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(distinct user_id) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
@@ -199,3 +254,121 @@ func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string
}
return dateHourMap, nil
}

func GetCloudbrainRunning() ([]*CloudbrainInfo, error) {
sess := x.NewSession()
defer sess.Close()
var cond = builder.NewCond()
cond = cond.And(
builder.Eq{"cloudbrain.status": string(JobRunning)},
)
sess.OrderBy("cloudbrain.created_unix ASC")
cloudbrains := make([]*CloudbrainInfo, 0, 10)
if err := sess.Table(&Cloudbrain{}).Where(cond).
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
return cloudbrains, nil
}

func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) {
sess := x.NewSession()
defer sess.Close()
var cond = builder.NewCond()
cond = cond.And(
builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}),
)
cond = cond.Or(
builder.Eq{"cloudbrain.status": string(JobRunning)},
)
sess.OrderBy("cloudbrain.created_unix ASC")
cloudbrains := make([]*CloudbrainInfo, 0, 10)
if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond).
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
return cloudbrains, nil
}

func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*CloudbrainSpec, error) {
sess := x.NewSession()
defer sess.Close()
var cond = builder.NewCond()
cond = cond.And(
builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}),
)
cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10)
if err := sess.Table(&CloudbrainSpec{}).Where(cond).
Find(&cloudbrainSpecs); err != nil {
log.Info("find error.")
}
return cloudbrainSpecs, nil
}

func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) {
return xStatistic.Insert(cloudbrainDurationStatistic)
}

func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string) error {
sess := xStatistic.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return fmt.Errorf("Begin: %v", err)
}

if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ?", date, hour, aiCenterCode, accCardType).Delete(&CloudbrainDurationStatistic{}); err != nil {
return fmt.Errorf("Delete: %v", err)
}

if err := sess.Commit(); err != nil {
sess.Close()
return fmt.Errorf("Commit: %v", err)
}

sess.Close()
return nil
}

func GetCanUseCardInfo() ([]*ResourceQueue, error) {
sess := x.NewSession()
defer sess.Close()
sess.OrderBy("resource_queue.id ASC")
ResourceQueues := make([]*ResourceQueue, 0, 10)
if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil {
log.Info("find error.")
}
return ResourceQueues, nil
}

func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) {
sess := xStatistic.NewSession()
defer sess.Close()
var cond = builder.NewCond()
if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 {
cond = cond.And(
builder.And(builder.Gte{"cloudbrain_duration_statistic.created_unix": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_unix": opts.EndTime.Unix()}),
)
}
if opts.AiCenterCode != "" {
cond = cond.And(
builder.Eq{"cloudbrain_duration_statistic.ai_center_code": opts.AiCenterCode},
)
}
CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10)
if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond).
Find(&CloudbrainDurationStatistics); err != nil {
log.Info("find error.")
}
return CloudbrainDurationStatistics, nil
}

func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) {
sess := xStatistic.NewSession()
defer sess.Close()
sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1")
CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0)
if err := sess.Table(&CloudbrainDurationStatistic{}).Find(&CloudbrainDurationStatistics); err != nil {
log.Info("find error.")
}
return CloudbrainDurationStatistics, nil
}

+ 1
- 0
models/models.go View File

@@ -183,6 +183,7 @@ func init() {
new(UserMetrics),
new(UserAnalysisPara),
new(Invitation),
new(CloudbrainDurationStatistic),
)

gonicNames := []string{"SSL", "UID"}


+ 12
- 0
modules/cron/tasks_basic.go View File

@@ -266,6 +266,17 @@ func registerSyncModelArtsTempJobs() {
})
}

func registerHandleCloudbrainDurationStatistic() {
RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{
Enabled: true,
RunAtStart: false,
Schedule: "1 0 * * * ?",
}, func(ctx context.Context, _ *models.User, _ Config) error {
repo.CloudbrainDurationStatisticHour()
return nil
})
}

func initBasicTasks() {
registerUpdateMirrorTask()
registerRepoHealthCheck()
@@ -293,4 +304,5 @@ func initBasicTasks() {
registerCloudbrainPointDeductTask()

registerHandleModelSafetyTask()
registerHandleCloudbrainDurationStatistic()
}

+ 5
- 0
routers/api/v1/api.go View File

@@ -599,6 +599,11 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData)
m.Get("/waitting_top_data", repo.GetWaittingTop)
m.Get("/running_top_data", repo.GetRunningTop)

m.Get("/overview_resource", repo.GetCloudbrainResourceOverview)
m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic)
m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail)
m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest)
})
}, operationReq)



+ 476
- 50
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"

@@ -120,9 +121,6 @@ func GetOverviewDuration(ctx *context.Context) {
recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix
now := time.Now()
endTime := now
page := 1
pagesize := 1000
count := pagesize
worker_server_num := 1
cardNum := 1
durationAllSum := int64(0)
@@ -138,54 +136,46 @@ func GetOverviewDuration(ctx *context.Context) {
c2NetDuration := int64(0)
cDCenterDuration := int64(0)

for count == pagesize && count != 0 {
cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: pagesize,
},
Type: models.TypeCloudBrainAll,
BeginTimeUnix: int64(recordBeginTime),
EndTimeUnix: endTime.Unix(),
})
if err != nil {
ctx.ServerError("Get cloudbrains failed:", err)
return
}
models.LoadSpecs4CloudbrainInfo(cloudbrains)

for _, cloudbrain := range cloudbrains {
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber
} else {
worker_server_num = 1
}
if cloudbrain.Cloudbrain.Spec == nil {
cardNum = 1
} else {
cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
}
duration := cloudbrain.Duration
durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum)
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration += duration
cloudBrainOneCardDuSum += durationSum
} else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration += duration
cloudBrainTwoCardDuSum += durationSum
} else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
c2NetDuration += duration
c2NetCardDuSum += durationSum
} else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
cDCenterDuration += duration
cDNetCardDuSum += durationSum
}
cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{
Type: models.TypeCloudBrainAll,
BeginTimeUnix: int64(recordBeginTime),
EndTimeUnix: endTime.Unix(),
})
if err != nil {
ctx.ServerError("Get cloudbrains failed:", err)
return
}
models.LoadSpecs4CloudbrainInfo(cloudbrains)

durationAllSum += duration
cardDuSum += durationSum
count = len(cloudbrains)
page += 1
for _, cloudbrain := range cloudbrains {
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber
} else {
worker_server_num = 1
}
if cloudbrain.Cloudbrain.Spec == nil {
cardNum = 1
} else {
cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
}
duration := cloudbrain.Duration
durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum)
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration += duration
cloudBrainOneCardDuSum += durationSum
} else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration += duration
cloudBrainTwoCardDuSum += durationSum
} else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
c2NetDuration += duration
c2NetCardDuSum += durationSum
} else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
cDCenterDuration += duration
cDNetCardDuSum += durationSum
}

durationAllSum += duration
cardDuSum += durationSum
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"cloudBrainOneCardDuSum": cloudBrainOneCardDuSum,
@@ -532,6 +522,21 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int

}

func getPageDateCloudbrainDuration(dateUsageStatistic []models.DateUsageStatistic, page int, pagesize int) []models.DateUsageStatistic {
begin := (page - 1) * pagesize
end := (page) * pagesize

if begin > len(dateUsageStatistic)-1 {
return nil
}
if end > len(dateUsageStatistic)-1 {
return dateUsageStatistic[begin:]
} else {
return dateUsageStatistic[begin:end]
}

}

func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
queryType := ctx.QueryTrim("type")
beginTimeStr := ctx.QueryTrim("beginTime")
@@ -545,7 +550,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0)
beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime)
if err != nil {
log.Error("Parameter is wrong", err)
log.Error("getCloudbrainTimePeroid error:", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
return
}
@@ -1403,3 +1408,424 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t

return beginTime, endTime, nil
}

func GetCloudbrainResourceOverview(ctx *context.Context) {
recordCloudbrainDuration, err := models.GetDurationRecordBeginTime()
if err != nil {
log.Error("Can not get GetDurationRecordBeginTime", err)
return
}
recordBeginTime := recordCloudbrainDuration[0].CreatedUnix
recordUpdateTime := time.Now().Unix()
resourceQueues, err := models.GetCanUseCardInfo()
if err != nil {
log.Info("GetCanUseCardInfo err: %v", err)
return
}
OpenIResourceDetail := []models.ResourceDetail{}
C2NetResourceDetail := []models.ResourceDetail{}
for _, resourceQueue := range resourceQueues {
if resourceQueue.Cluster == models.OpenICluster {
var resourceDetail models.ResourceDetail
resourceDetail.QueueCode = resourceQueue.QueueCode
resourceDetail.Cluster = resourceQueue.Cluster
resourceDetail.AiCenterCode = resourceQueue.AiCenterCode
resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode
resourceDetail.ComputeResource = resourceQueue.ComputeResource
resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")"
resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum
resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync
OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail)
}
if resourceQueue.Cluster == models.C2NetCluster {
var resourceDetail models.ResourceDetail
resourceDetail.QueueCode = resourceQueue.QueueCode
resourceDetail.Cluster = resourceQueue.Cluster
resourceDetail.AiCenterCode = resourceQueue.AiCenterCode
resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode
resourceDetail.ComputeResource = resourceQueue.ComputeResource
resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")"
resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum
resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync
C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail)
}
}
openIResourceNum := make(map[string]map[string]int)

for _, openIResourceDetail := range OpenIResourceDetail {
if _, ok := openIResourceNum[openIResourceDetail.AiCenterName]; !ok {
openIResourceNum[openIResourceDetail.AiCenterName] = make(map[string]int)
}
if _, ok := openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType]; !ok {
openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum
} else {
openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum
}
}

c2NetResourceNum := make(map[string]map[string]int)
for _, c2NetResourceDetail := range C2NetResourceDetail {
if _, ok := c2NetResourceNum[c2NetResourceDetail.AiCenterName]; !ok {
c2NetResourceNum[c2NetResourceDetail.AiCenterName] = make(map[string]int)
}
if _, ok := c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType]; !ok {
c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType] = c2NetResourceDetail.CardsTotalNum
} else {
c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType] += c2NetResourceDetail.CardsTotalNum
}

}

ctx.JSON(http.StatusOK, map[string]interface{}{
"openI": openIResourceNum,
"c2Net": c2NetResourceNum,
"recordUpdateTime": recordUpdateTime,
"recordBeginTime": recordBeginTime,
})
}

func GetCloudbrainResourceUsageDetail(ctx *context.Context) {
aiCenterCode := ctx.QueryTrim("aiCenterCode")
if aiCenterCode == "" {
aiCenterCode = models.AICenterOfCloudBrainOne
}
beginTime, endTime := getBeginAndEndTime(ctx)
dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode)
if err != nil {
log.Error("Can not query dayCloudbrainDuration.", err)
return
}
hourCloudbrainDuration, err := getHourCloudbrainDuration(beginTime, endTime, aiCenterCode)
if err != nil {
log.Error("Can not query hourCloudbrainDuration.", err)
return
}
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}
pagesize := ctx.QueryInt("pagesize")
if pagesize <= 0 {
pagesize = 36500
}
pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize)
ctx.JSON(http.StatusOK, map[string]interface{}{
"totalCount": count,
"pageDateCloudbrainDuration": pageDateCloudbrainDuration,
"hourCloudbrainDuration": hourCloudbrainDuration,
})
}

func GetDurationRateStatistic(ctx *context.Context) {
beginTime, endTime := getBeginAndEndTime(ctx)
OpenIDurationRate, C2NetDurationRate, totalUsageRate := getDurationStatistic(beginTime, endTime)

ctx.JSON(http.StatusOK, map[string]interface{}{
"openIDurationRate": OpenIDurationRate,
"c2NetDurationRate": C2NetDurationRate,
"totalUsageRate": totalUsageRate,
})

}

func CloudbrainDurationStatisticForTest(ctx *context.Context) {
repo.CloudbrainDurationStatisticHour()
ctx.JSON(http.StatusOK, map[string]interface{}{
"message": 0,
})
}

func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) {
queryType := ctx.QueryTrim("type")
now := time.Now()
beginTimeStr := ctx.QueryTrim("beginTime")
endTimeStr := ctx.QueryTrim("endTime")

var beginTime time.Time
var endTime time.Time
var err error
if queryType != "" {
if queryType == "all" {
recordCloudbrainDuration, err := models.GetDurationRecordBeginTime()
if err != nil {
log.Error("Can not get GetDurationRecordBeginTime", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
return beginTime, endTime
}
brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime()
beginTime = brainRecordBeginTime
endTime = now
} else if queryType == "today" {
beginTime = now.AddDate(0, 0, 0)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now

} else if queryType == "yesterday" {
beginTime = now.AddDate(0, 0, -1)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
} else if queryType == "last_7day" {
beginTime = now.AddDate(0, 0, -6)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now
} else if queryType == "last_30day" {
beginTime = now.AddDate(0, 0, -29)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now
} else if queryType == "current_month" {
endTime = now
beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location())

} else if queryType == "current_year" {
endTime = now
beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location())
} else if queryType == "last_month" {
lastMonthTime := now.AddDate(0, -1, 0)
beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location())
endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
}

} else {
if beginTimeStr == "" || endTimeStr == "" {
//如果查询类型和开始时间结束时间都未设置,按queryType=all处理
recordCloudbrainDuration, err := models.GetDurationRecordBeginTime()
if err != nil {
log.Error("Can not get recordCloudbrain", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
return beginTime, endTime
}
brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime()
beginTime = brainRecordBeginTime
endTime = now
} else {
beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local)
if err != nil {
log.Error("Can not ParseInLocation.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error"))
return beginTime, endTime
}
endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local)
if err != nil {
log.Error("Can not ParseInLocation.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error"))
return beginTime, endTime
}
if endTime.After(time.Now()) {
endTime = time.Now()
}
}

}
return beginTime, endTime
}

func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (int, int, float64) {
totalDuration := int(0)
usageDuration := int(0)
usageRate := float64(0)

for _, cloudbrainStatistic := range cloudbrainStatistics {
if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() {
totalDuration += cloudbrainStatistic.CardsTotalDuration
usageDuration += cloudbrainStatistic.CardsUseDuration
}
}
if totalDuration == 0 || usageDuration == 0 {
usageRate = 0
} else {
usageRate = float64(usageDuration) / float64(totalDuration)
}

return totalDuration, usageDuration, usageRate
}

func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float64) {
OpenITotalDuration := make(map[string]int)
OpenIUsageDuration := make(map[string]int)
OpenIUsageRate := make(map[string]float64)

C2NetTotalDuration := make(map[string]int)
C2NetUsageDuration := make(map[string]int)
OpenIDurationRate := models.DurationRateStatistic{}
C2NetDurationRate := models.DurationRateStatistic{}
cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
BeginTime: beginTime,
EndTime: endTime,
})
if err != nil {
log.Error("GetCardDurationStatistics error:", err)
return OpenIDurationRate, C2NetDurationRate, 0
}
for _, cloudbrainStatistic := range cardDurationStatistics {
if cloudbrainStatistic.Cluster == models.OpenICluster {
if _, ok := OpenITotalDuration[cloudbrainStatistic.AiCenterName]; !ok {
OpenITotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration
} else {
OpenITotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration
}
if _, ok := OpenIUsageDuration[cloudbrainStatistic.AiCenterName]; !ok {
OpenIUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration
} else {
OpenIUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration
}
}
if cloudbrainStatistic.Cluster == models.C2NetCluster {
if _, ok := C2NetTotalDuration[cloudbrainStatistic.AiCenterName]; !ok {
C2NetTotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration
} else {
C2NetTotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration
}
if _, ok := C2NetUsageDuration[cloudbrainStatistic.AiCenterName]; !ok {
C2NetUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration
} else {
C2NetUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration
}
}
}
ResourceAiCenterRes, err := models.GetResourceAiCenters()
if err != nil {
log.Error("Can not get ResourceAiCenterRes.", err)
return OpenIDurationRate, C2NetDurationRate, 0
}
for _, v := range ResourceAiCenterRes {
if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) {
if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok {
OpenIUsageDuration[v.AiCenterName] = 0
}
if _, ok := OpenITotalDuration[v.AiCenterName]; !ok {
OpenITotalDuration[v.AiCenterName] = 0
}
} else {
if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok {
C2NetUsageDuration[v.AiCenterName] = 0
}
}
}
totalCanUse := float64(0)
totalUse := float64(0)
totalUsageRate := float64(0)
for k, v := range OpenITotalDuration {
for i, j := range OpenIUsageDuration {
if k == i {
OpenIUsageRate[k] = float64(j) / float64(v)
}
}
}
for _, v := range OpenITotalDuration {
totalCanUse += float64(v)
}
for _, v := range OpenIUsageRate {
totalUse += float64(v)
}
if totalCanUse == 0 || totalUse == 0 {
totalUsageRate = 0
} else {
totalUsageRate = totalUse / totalCanUse
}

OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration
OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration
OpenIDurationRate.UsageRate = OpenIUsageRate
C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration
C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration
return OpenIDurationRate, C2NetDurationRate, totalUsageRate
}

func cutString(str string, lens int) string {
if len(str) < lens {
return str
}
return str[:lens]
}

func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) {
now := time.Now()
endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location())
if endTimeTemp.Equal(endTime) {
endTimeTemp = endTimeTemp.AddDate(0, 0, -1)
}
cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
BeginTime: beginTime,
EndTime: endTime,
AiCenterCode: aiCenterCode,
})
if err != nil {
log.Error("GetCardDurationStatistics error:", err)
return nil, 0, err
}

dayCloudbrainInfo := make([]models.DateUsageStatistic, 0)
count := 0
for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) {
TotalDuration, UsageDuration, UsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics)
dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateUsageStatistic{
Date: endTimeTemp.Format("2006/01/02"),
UsageDuration: UsageDuration,
TotalDuration: TotalDuration,
UsageRate: UsageRate,
})
endTime = endTimeTemp
endTimeTemp = endTimeTemp.AddDate(0, 0, -1)
if endTimeTemp.Before(beginTime) && beginTime.Before(endTime) {
endTimeTemp = beginTime
}
count += 1
}
return dayCloudbrainInfo, count, nil
}

func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) (models.HourTimeStatistic, error) {
hourTimeTotalDuration := make(map[string]int)
hourTimeUsageDuration := make(map[string]int)
hourTimeUsageRate := make(map[string]float64)
hourTimeStatistic := models.HourTimeStatistic{}

cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
BeginTime: beginTime,
EndTime: endTime,
})
if err != nil {
log.Error("GetCardDurationStatistics error:", err)
return hourTimeStatistic, err
}
for _, cloudbrainStatistic := range cardDurationStatistics {
if cloudbrainStatistic.AiCenterCode == aiCenterCode {
if _, ok := hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok {
hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration
} else {
hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration
}
if _, ok := hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok {
hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsUseDuration
} else {
hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsUseDuration
}
}
}
hourTimeList := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"}
for _, v := range hourTimeList {
if _, ok := hourTimeUsageDuration[v]; !ok {
hourTimeUsageDuration[v] = 0
}
if _, ok := hourTimeTotalDuration[v]; !ok {
hourTimeTotalDuration[v] = 0
}
}

for k, v := range hourTimeTotalDuration {
for i, j := range hourTimeUsageDuration {
if k == i {
if v == 0 || j == 0 {
hourTimeUsageRate[k] = 0
} else {
hourTimeUsageRate[k] = float64(j) / float64(v)
}
}
}
}

hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration
hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration
hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate
return hourTimeStatistic, nil
}

+ 169
- 0
routers/repo/cloudbrain_statistic.go View File

@@ -0,0 +1,169 @@
package repo

import (
"strings"
"time"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
)

func CloudbrainDurationStatisticHour() {

dateTime := time.Now().Format("2006-01-02 15:04:05")
dayTime := time.Now().Format("2006-01-02")
now := time.Now()

currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())

m, _ := time.ParseDuration("-1h")
beginTime := currentTime.Add(m).Unix()
endTime := currentTime.Unix()
hourTime := currentTime.Add(m).Hour()

ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime)
if err != nil {
log.Info("GetCloudbrainByTime err: %v", err)
return
}
specMap := make(map[string]*models.Specification)
models.LoadSpecs4CloudbrainInfo(ciTasks)
for _, cloudbrain := range ciTasks {
if _, ok := specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if cloudbrain.Cloudbrain.Spec != nil {
specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec
}
}
}

cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime)

resourceQueues, err := models.GetCanUseCardInfo()
if err != nil {
log.Info("GetCanUseCardInfo err: %v", err)
return
}
cardsTotalDurationMap := make(map[string]int)
for _, resourceQueue := range resourceQueues {
cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterName+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
}

for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo {
for cardType, cardDuration := range CardTypeInfo {
spec := specMap[centerCode+"/"+cardType]
if spec != nil {
if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil {
log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
return
}
if _, ok := cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource]; !ok {
cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource] = 0
}
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTime: dateTime,
DayTime: dayTime,
HourTime: hourTime,
Cluster: spec.Cluster,
AiCenterName: spec.AiCenterName,
AiCenterCode: centerCode,
AccCardType: cardType,
ComputeResource: spec.ComputeResource,
CardsUseDuration: cardDuration,
CardsTotalDuration: cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource],
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
delete(cardsTotalDurationMap, spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource)
}
}
}

for key, cardsTotalDuration := range cardsTotalDurationMap {
if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[2], strings.Split(key, "/")[3]); err != nil {
log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
return
}
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTime: dateTime,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: strings.Split(key, "/")[1],
AiCenterCode: strings.Split(key, "/")[2],
AccCardType: strings.Split(key, "/")[3],
ComputeResource: strings.Split(key, "/")[4],
CardsUseDuration: 0,
CardsTotalDuration: cardsTotalDuration,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
}

log.Info("finish summary cloudbrainDurationStat")
}

func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int {
var WorkServerNumber int
var AccCardsNum int
cloudBrainCenterCodeAndCardType := make(map[string]map[string]int)
for _, cloudbrain := range ciTasks {

if cloudbrain.Cloudbrain.StartTime == 0 {
cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
}
if cloudbrain.Cloudbrain.EndTime == 0 {
cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix
}
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber
} else {
WorkServerNumber = 1
}
if cloudbrain.Cloudbrain.Spec == nil {
AccCardsNum = 1
} else {
AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
}
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode]; !ok {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode] = make(map[string]int)
}

if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) {
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
}
} else {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
}
}
} else {
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
} else {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
}
} else {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
} else {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
}
}

}
}

return cloudBrainCenterCodeAndCardType
}

+ 6
- 5
routers/routes/routes.go View File

@@ -6,17 +6,18 @@ package routes

import (
"bytes"
"code.gitea.io/gitea/routers/badge"
"code.gitea.io/gitea/routers/reward/point"
"code.gitea.io/gitea/routers/task"
badge_service "code.gitea.io/gitea/services/badge"
"code.gitea.io/gitea/services/reward"
"encoding/gob"
"net/http"
"path"
"text/template"
"time"

"code.gitea.io/gitea/routers/badge"
"code.gitea.io/gitea/routers/reward/point"
"code.gitea.io/gitea/routers/task"
badge_service "code.gitea.io/gitea/services/badge"
"code.gitea.io/gitea/services/reward"

"code.gitea.io/gitea/routers/modelapp"

"code.gitea.io/gitea/modules/slideimage"


Loading…
Cancel
Save