diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index cb99f9ff5..a7678b267 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,7 +1,6 @@ package models import ( - "fmt" "strconv" "time" @@ -313,10 +312,6 @@ func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDu return xStatistic.Insert(cloudbrainDurationStatistic) } -func GetDurationStatisticByDate(date string, hour int, aiCenterCode string, accCardType string) (*CloudbrainDurationStatistic, error) { - cb := &CloudbrainDurationStatistic{DayTime: date, HourTime: hour, AiCenterCode: aiCenterCode, AccCardType: accCardType} - return getDurationStatistic(cb) -} func getDurationStatistic(cb *CloudbrainDurationStatistic) (*CloudbrainDurationStatistic, error) { has, err := x.Get(cb) if err != nil { @@ -327,26 +322,6 @@ func getDurationStatistic(cb *CloudbrainDurationStatistic) (*CloudbrainDurationS return cb, nil } -func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string) error { - sess := xStatistic.NewSession() - defer sess.Close() - if err := sess.Begin(); err != nil { - return fmt.Errorf("Begin: %v", err) - } - - if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ?", date, hour, aiCenterCode, accCardType).Delete(&CloudbrainDurationStatistic{}); err != nil { - return fmt.Errorf("Delete: %v", err) - } - - if err := sess.Commit(); err != nil { - sess.Close() - return fmt.Errorf("Commit: %v", err) - } - - sess.Close() - return nil -} - func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() @@ -402,11 +377,11 @@ func GetDurationRecordUpdateTime() ([]*CloudbrainDurationStatistic, error) { return CloudbrainDurationStatistics, nil } -func DeleteCloudbrainDurationStatistic() error { +func DeleteCloudbrainDurationStatistic(beginTime timeutil.TimeStamp, endTime timeutil.TimeStamp) error { sess := xStatistic.NewSession() defer sess.Close() - if _, err := sess.Exec("TRUNCATE TABLE cloudbrain_duration_statistic"); err != nil { - log.Info("TRUNCATE cloudbrain_duration_statistic error.") + if _, err := sess.Exec("DELETE FROM cloudbrain_duration_statistic WHERE cloudbrain_duration_statistic.date_time BETWEEN ? AND ?", beginTime, endTime); err != nil { + log.Info("DELETE cloudbrain_duration_statistic data error.") return err } return nil diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 446522fc2..09e65b007 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -123,8 +123,9 @@ func GetOverviewDuration(ctx *context.Context) { recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix now := time.Now() endTime := now - // worker_server_num := 1 - // cardNum := 1 + var workServerNumber int64 + var cardNum int64 + durationAllSum := int64(0) cardDuSum := int64(0) @@ -138,52 +139,60 @@ func GetOverviewDuration(ctx *context.Context) { c2NetDuration := int64(0) cDCenterDuration := int64(0) - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - models.LoadSpecs4CloudbrainInfo(cloudbrains) - - for _, cloudbrain := range cloudbrains { - cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) - CardDurationString := repo.GetCloudbrainCardDuration(cloudbrain.Cloudbrain) - CardDuration := models.ConvertStrToDuration(CardDurationString) - // if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { - // worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber - // } else { - // worker_server_num = 1 - // } - // if cloudbrain.Cloudbrain.Spec == nil { - // cardNum = 1 - // } else { - // cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum - // } - // duration := cloudbrain.Duration - // duration := cloudbrain.Duration - duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration) - // CardDuration := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) - - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration += duration - cloudBrainOneCardDuSum += CardDuration - } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration += duration - cloudBrainTwoCardDuSum += CardDuration - } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - c2NetDuration += duration - c2NetCardDuSum += CardDuration - } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { - cDCenterDuration += duration - cDNetCardDuSum += CardDuration + page := 1 + pagesize := 10000 + count := pagesize + // Each time a maximum of 10000 pieces of data are detected to the memory, batch processing + for count == pagesize && count != 0 { + cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pagesize, + }, + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return } + models.LoadSpecs4CloudbrainInfo(cloudbrains) + + for _, cloudbrain := range cloudbrains { + cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) + if cloudbrain.Cloudbrain.Spec != nil { + cardNum = int64(cloudbrain.Cloudbrain.Spec.AccCardsNum) + } else { + cardNum = 1 + } + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + workServerNumber = int64(cloudbrain.Cloudbrain.WorkServerNumber) + } else { + workServerNumber = 1 + } + duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration) + CardDuration := workServerNumber * int64(cardNum) * duration + + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration += duration + cloudBrainOneCardDuSum += CardDuration + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration + cloudBrainTwoCardDuSum += CardDuration + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration + c2NetCardDuSum += CardDuration + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDNetCardDuSum += CardDuration + } - durationAllSum += duration - cardDuSum += CardDuration + durationAllSum += duration + cardDuSum += CardDuration + } + count = len(cloudbrains) + page += 1 } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 23f7336b1..2ec12cd41 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -8,7 +8,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" cloudbrainService "code.gitea.io/gitea/services/cloudbrain" ) @@ -16,19 +15,23 @@ import ( func CloudbrainDurationStatisticHour() { var statisticTime time.Time var count int64 - recordBeginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", setting.Grampus.UsageRateBeginTime, time.Local) recordDurationUpdateTime, err := models.GetDurationRecordUpdateTime() if err != nil { log.Error("Can not get GetDurationRecordBeginTime", err) - return - } - if recordDurationUpdateTime == nil { - statisticTime = recordBeginTime - } else { - statisticTime = time.Unix(int64(recordDurationUpdateTime[0].DateTime), 0) } now := time.Now() currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()) + if err == nil && len(recordDurationUpdateTime) > 0 { + statisticTime = time.Unix(int64(recordDurationUpdateTime[0].DateTime), 0).Add(+1 * time.Hour) + } else { + statisticTime = currentTime + } + deleteBeginTime := time.Unix(int64(recordDurationUpdateTime[0].DateTime), 0) + + err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(deleteBeginTime.Unix()), timeutil.TimeStamp(currentTime.Unix())) + if err != nil { + log.Error("DeleteCloudbrainDurationStatistic failed", err) + } for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) { countEach := summaryDurationStat(statisticTime) @@ -37,13 +40,10 @@ func CloudbrainDurationStatisticHour() { } log.Info("summaryDurationStat count: %v", count) } -func UpdateDurationStatisticHistoryData() int64 { +func UpdateDurationStatisticHistoryData(beginTime time.Time, endTime time.Time) int64 { var count int64 - recordBeginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", setting.Grampus.UsageRateBeginTime, time.Local) - now := time.Now() - currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()) - statisticTime := recordBeginTime.Add(+1 * time.Hour) - + statisticTime := beginTime + currentTime := endTime for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) { countEach := summaryDurationStat(statisticTime) count += countEach @@ -107,13 +107,6 @@ func summaryDurationStat(statisticTime time.Time) int64 { for cardType, cardDuration := range CardTypes { cloudbrainTable := cloudbrainMap[centerCode+"/"+cardType] if cloudbrainTable != nil { - if _, err := models.GetDurationStatisticByDate(dayTime, hourTime, centerCode, cardType); err == nil { - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil { - log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) - return 0 - } - } - if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType]; !ok { cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType] = 0 } @@ -139,12 +132,6 @@ func summaryDurationStat(statisticTime time.Time) int64 { } for key, cardsTotalDuration := range cardsTotalDurationMap { - if _, err := models.GetDurationStatisticByDate(dayTime, hourTime, strings.Split(key, "/")[1], strings.Split(key, "/")[2]); err == nil { - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[1], strings.Split(key, "/")[2]); err != nil { - log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) - return 0 - } - } cloudbrainDurationStat := models.CloudbrainDurationStatistic{ DateTime: dateTime, DayTime: dayTime, @@ -257,8 +244,22 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be } func CloudbrainUpdateHistoryData(ctx *context.Context) { - err := models.DeleteCloudbrainDurationStatistic() - count := UpdateDurationStatisticHistoryData() + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var count int64 + var err error + if beginTimeStr != "" && endTimeStr != "" { + beginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", beginTimeStr, time.Local) + endTime, _ := time.ParseInLocation("2006-01-02 15:04:05", endTimeStr, time.Local) + if time.Now().Before(endTime) { + endTime = time.Now() + } + beginTimeUnix := timeutil.TimeStamp(beginTime.Unix()) + endTimeUnix := timeutil.TimeStamp(endTime.Unix()) + + err = models.DeleteCloudbrainDurationStatistic(beginTimeUnix, endTimeUnix) + count = UpdateDurationStatisticHistoryData(beginTime.Add(+1*time.Hour), endTime) + } ctx.JSON(http.StatusOK, map[string]interface{}{ "message": 0, "count": count,