@@ -1,41 +1,12 @@ | |||||
package models | package models | ||||
import ( | import ( | ||||
"fmt" | |||||
"strconv" | "strconv" | ||||
"time" | "time" | ||||
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
"code.gitea.io/gitea/modules/timeutil" | |||||
) | ) | ||||
// Cloudbrain statistic info of all CloudbrainTasks | |||||
type CloudbrainStatistic struct { | |||||
ID int64 `xorm:"pk autoincr" json:"-"` | |||||
Date string `xorm:"NOT NULL DEFAULT 0" json:"date"` | |||||
WhichHour int64 `xorm:"NOT NULL DEFAULT -1" json:"whichHour"` | |||||
NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` | |||||
NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` | |||||
NumTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"` | |||||
NumDubugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"` | |||||
NumTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"` | |||||
NumInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"` | |||||
NumCloudOneAll int64 `xorm:"NOT NULL DEFAULT 0" json:"numCloudOneAll"` | |||||
NumCloudTwoAll int64 `xorm:"NOT NULL DEFAULT 0" json:"numCloudTwoAll"` | |||||
DurationDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDubugOne"` | |||||
DurationBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationBenchmarkOne"` | |||||
DurationTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainOne"` | |||||
DurationDebugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDebugTwo"` | |||||
DurationTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainTwo"` | |||||
DurationInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationInferenceTwo"` | |||||
DurationCloudOneAll int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudOneAll"` | |||||
DurationCloudTwoAll int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudTwoAll"` | |||||
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"` | |||||
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` | |||||
} | |||||
func GetJobWaitingPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | func GetJobWaitingPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | ||||
countSql := "SELECT count(*) FROM " + | countSql := "SELECT count(*) FROM " + | ||||
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | ||||
@@ -244,56 +215,6 @@ func GetCreatorCount() (int64, error) { | |||||
return x.SQL(countSql).Count() | return x.SQL(countSql).Count() | ||||
} | } | ||||
func DeleteCloudbrainStatisticDaily(date string) error { | |||||
sess := xStatistic.NewSession() | |||||
defer sess.Close() | |||||
if err := sess.Begin(); err != nil { | |||||
return fmt.Errorf("Begin: %v", err) | |||||
} | |||||
if _, err := sess.Where("date = ?", date).Delete(&CloudbrainStatistic{}); err != nil { | |||||
return fmt.Errorf("Delete: %v", err) | |||||
} | |||||
if err := sess.Commit(); err != nil { | |||||
sess.Close() | |||||
return fmt.Errorf("Commit: %v", err) | |||||
} | |||||
sess.Close() | |||||
return nil | |||||
} | |||||
func GetHourStatTime(timeStr string, whichHour int64) (time.Time, time.Time) { | |||||
t, _ := time.Parse("2006-01-02", timeStr) | |||||
timeNumber := t.Unix() | |||||
beginTimeNumber := timeNumber - 8*60*60 + whichHour*60*60 | |||||
endTimeNumber := beginTimeNumber + 1*60*60 | |||||
beginTime := time.Unix(beginTimeNumber, 0) | |||||
endTime := time.Unix(endTimeNumber, 0) | |||||
log.Info("%s, %s", beginTime, endTime) | |||||
return beginTime, endTime | |||||
} | |||||
func GetDayStatTime(timeStr string) (time.Time, time.Time) { | |||||
t, _ := time.Parse("2006-01-02", timeStr) | |||||
timeNumber := t.Unix() | |||||
beginTimeNumber := timeNumber - 8*60*60 | |||||
endTimeNumber := beginTimeNumber + 16*60*60 | |||||
beginTime := time.Unix(beginTimeNumber, 0) | |||||
endTime := time.Unix(endTimeNumber, 0) | |||||
log.Info("%s, %s", beginTime, endTime) | |||||
return beginTime, endTime | |||||
} | |||||
func CreateCloudbrainStatistic(cloudbrainStat *CloudbrainStatistic) (err error) { | |||||
if _, err = xStatistic.Insert(cloudbrainStat); err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func GetJobWaitingCount() (int64, error) { | func GetJobWaitingCount() (int64, error) { | ||||
countSql := "SELECT count(*) FROM " + "public.cloudbrain where status ='" + string(JobWaiting) + "'" | countSql := "SELECT count(*) FROM " + "public.cloudbrain where status ='" + string(JobWaiting) + "'" | ||||
return x.SQL(countSql).Count() | return x.SQL(countSql).Count() | ||||
@@ -160,7 +160,6 @@ func init() { | |||||
new(UserLoginLog), | new(UserLoginLog), | ||||
new(UserMetrics), | new(UserMetrics), | ||||
new(UserAnalysisPara), | new(UserAnalysisPara), | ||||
new(CloudbrainStatistic), | |||||
) | ) | ||||
gonicNames := []string{"SSL", "UID"} | gonicNames := []string{"SSL", "UID"} | ||||
@@ -173,16 +173,6 @@ func registerHandleRepoAndUserStatistic() { | |||||
return nil | return nil | ||||
}) | }) | ||||
} | } | ||||
func registerHandleCloudbrainStatistic() { | |||||
RegisterTaskFatal("handle_cloudbrain_statistic", &BaseConfig{ | |||||
Enabled: true, | |||||
RunAtStart: false, | |||||
Schedule: "@daily", | |||||
}, func(ctx context.Context, _ *models.User, _ Config) error { | |||||
repo.CloudbrainStatisticAuto() | |||||
return nil | |||||
}) | |||||
} | |||||
func registerHandleSummaryStatistic() { | func registerHandleSummaryStatistic() { | ||||
RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ | RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ | ||||
@@ -233,7 +223,6 @@ func initBasicTasks() { | |||||
registerHandleBlockChainUnSuccessCommits() | registerHandleBlockChainUnSuccessCommits() | ||||
registerHandleRepoAndUserStatistic() | registerHandleRepoAndUserStatistic() | ||||
registerHandleCloudbrainStatistic() | |||||
registerHandleSummaryStatistic() | registerHandleSummaryStatistic() | ||||
registerSyncCloudbrainStatus() | registerSyncCloudbrainStatus() | ||||
@@ -1,121 +0,0 @@ | |||||
package repo | |||||
import ( | |||||
"time" | |||||
"code.gitea.io/gitea/models" | |||||
"code.gitea.io/gitea/modules/log" | |||||
"code.gitea.io/gitea/modules/setting" | |||||
"code.gitea.io/gitea/services/mailer" | |||||
) | |||||
func CloudbrainStatisticAuto() { | |||||
yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") | |||||
CloudbrainStatisticDaily(yesterday) | |||||
} | |||||
func CloudbrainStatisticDaily(date string) { | |||||
log.Info("%s", date) | |||||
log.Info("begin Cloudbrain Statistic") | |||||
warnEmailMessage := "云脑任务统计信息入库失败,请尽快定位。" | |||||
if err := models.DeleteCloudbrainStatisticDaily(date); err != nil { | |||||
log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error()) | |||||
mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) | |||||
return | |||||
} | |||||
//0 to 23 for each hour, 25 for all 24 hours | |||||
var WhichHour [25]int64 = [25]int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25} | |||||
for _, whichHour := range WhichHour { | |||||
log.Info("start statistic: %s", date) | |||||
var beginTime, endTime time.Time | |||||
beginTime, endTime = models.GetHourStatTime(date, whichHour) | |||||
if whichHour == 25 { | |||||
beginTime, endTime = models.GetDayStatTime(date) | |||||
} | |||||
numDubugOne, err := models.GetDebugOnePeriodCount(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetDebugOneCount failed(%s): %v", numDubugOne, err) | |||||
} | |||||
numBenchmarkOne, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetBenchmarkOneCount failed(%s): %v", numBenchmarkOne, err) | |||||
} | |||||
numTrainOne, err := models.GetTrainOnePeriodCount(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetTrainOneCount failed(%s): %v", numTrainOne, err) | |||||
} | |||||
numDebugTwo, err := models.GetDebugTwoPeriodCount(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetDebugTwoCount failed(%s): %v", numDebugTwo, err) | |||||
} | |||||
numTrainTwo, err := models.GetTrainTwoPeriodCount(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetTrainTwoCount failed(%s): %v", numTrainTwo, err) | |||||
} | |||||
numInferenceTwo, err := models.GetInferenceTwoPeriodCount(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetInferenceTwoCount failed(%s): %v", numInferenceTwo, err) | |||||
} | |||||
numCloudOneAll := numDubugOne + numBenchmarkOne + numTrainOne | |||||
numCloudTwoAll := numDebugTwo + numTrainTwo + numInferenceTwo | |||||
durationDubugOne, err := models.GetDebugOnePeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetDebugOneDuration failed(%s): %v", durationDubugOne, err) | |||||
} | |||||
durationBenchmarkOne, err := models.GetBenchmarkOnePeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetBenchmarkOneDuration failed(%s): %v", durationBenchmarkOne, err) | |||||
} | |||||
durationTrainOne, err := models.GetTrainOnePeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetTrainOneDuration failed(%s): %v", durationTrainOne, err) | |||||
} | |||||
durationDebugTwo, err := models.GetDebugTwoPeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetDebugTwoDuration failed(%s): %v", durationDebugTwo, err) | |||||
} | |||||
durationTrainTwo, err := models.GetTrainTwoPeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetTrainTwoDuration failed(%s): %v", durationTrainTwo, err) | |||||
} | |||||
durationInferenceTwo, err := models.GetInferenceTwoPeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetInferenceTwoDuration failed(%s): %v", durationInferenceTwo, err) | |||||
} | |||||
durationCloudOneAll, err := models.GetCloudBrainOnePeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetCloudBrainOnePeriodDuration failed(%s): %v", durationCloudOneAll, err) | |||||
} | |||||
durationCloudTwoAll, err := models.GetCloudBrainTwoPeriodDuration(beginTime, endTime) | |||||
if err != nil { | |||||
log.Error("GetCloudBrainTwoPeriodDuration failed(%s): %v", durationCloudTwoAll, err) | |||||
} | |||||
err = models.CreateCloudbrainStatistic(&models.CloudbrainStatistic{ | |||||
Date: date, | |||||
WhichHour: whichHour, | |||||
NumDubugOne: numDubugOne, | |||||
NumBenchmarkOne: numBenchmarkOne, | |||||
NumTrainOne: numTrainOne, | |||||
NumDubugTwo: numDebugTwo, | |||||
NumTrainTwo: numTrainTwo, | |||||
NumInferenceTwo: numInferenceTwo, | |||||
NumCloudOneAll: numCloudOneAll, | |||||
NumCloudTwoAll: numCloudTwoAll, | |||||
DurationDubugOne: durationDubugOne, | |||||
DurationBenchmarkOne: durationBenchmarkOne, | |||||
DurationTrainOne: durationTrainOne, | |||||
DurationDebugTwo: durationDebugTwo, | |||||
DurationTrainTwo: durationTrainTwo, | |||||
DurationInferenceTwo: durationInferenceTwo, | |||||
DurationCloudOneAll: durationCloudOneAll, | |||||
DurationCloudTwoAll: durationCloudTwoAll, | |||||
}) | |||||
if err != nil { | |||||
log.Error("CreateCloudbrainStatistic(%s) failed:%v", date, err.Error()) | |||||
return | |||||
} | |||||
} | |||||
} |
@@ -1033,7 +1033,6 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
}, context.RepoRef()) | }, context.RepoRef()) | ||||
m.Group("/cloudbrain", func() { | m.Group("/cloudbrain", func() { | ||||
m.Get("/test", repo.CloudbrainStatisticAuto) | |||||
m.Group("/:id", func() { | m.Group("/:id", func() { | ||||
m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) | m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) | ||||
m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDebug) | m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDebug) | ||||