From 17718c507fc3234609040ccf921fb17486727c40 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 22 Mar 2022 17:50:50 +0800 Subject: [PATCH 001/143] update --- models/cloudbrain_static.go | 5 ++ routers/api/v1/api.go | 15 ++++ routers/api/v1/repo/cloudbrain_dashboard.go | 119 ++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 models/cloudbrain_static.go create mode 100644 routers/api/v1/repo/cloudbrain_dashboard.go diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go new file mode 100644 index 000000000..9898ebf92 --- /dev/null +++ b/models/cloudbrain_static.go @@ -0,0 +1,5 @@ +package models + +func CountBrainStatByRawSql(sql string) (int64, error) { + return xStatistic.SQL(sql).Count() +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 306854af3..6118db265 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -554,6 +554,21 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday) m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) + //cloudbrain board + m.Group("/cloudbrainboard", func() { + m.Get("/restoreFork", repo.RestoreForkNumber) + m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) + m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) + m.Group("/cloudbrain", func() { + m.Get("", repo.GetAllCloudbrainsPeriodStatistics) + m.Group("/:id", func() { + m.Get("", repo.GetProjectLatestStatistics) + m.Get("/period", repo.GetProjectPeriodStatistics) + + }) + }) + }, operationReq) + // Users m.Group("/users", func() { m.Get("/search", user.Search) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go new file mode 100644 index 000000000..d39f5c373 --- /dev/null +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -0,0 +1,119 @@ +package repo + +import ( + "net/http" + "strconv" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pagesize") + if pageSize <= 0 { + pageSize = DEFAULT_PAGE_SIZE + } + orderBy := getOrderBy(ctx) + + latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + DebugOneCountSql := generateDebugOneCountSql(beginTime, endTime) + DebugOneTotal, err := models.CountBrainStatByRawSql(beginTime, endTime) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + DebugTwoCountSql := generateDebugTwoCountSql(beginTime, endTime) + DebugOneTotal, err := models.CountBrainStatByRawSql(DebugTwoCountSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + + sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) + + projectsPeriodData := ProjectsPeriodData{ + RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), + PageSize: pageSize, + TotalPage: getTotalPage(DebugOneTotal, pageSize), + TotalCount: DebugOneTotal, + LastUpdatedTime: latestUpdatedTime, + PageRecords: models.GetRepoStatisticByRawSql(sql), + } + + ctx.JSON(http.StatusOK, projectsPeriodData) + +} + +func generateDebugOneCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + " and type=" + "0" + + " group by job_id) " + return countSql +} +func generateBenchmarkOneCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "BENCHMARK" + + " and type=" + "0" + + " group by job_id) " + return countSql +} +func generateDebugTwoCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + " and type=" + "2" + + " group by job_id) " + return countSql +} +func generateTrainTwoCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "TRAIN" + + " and type=" + "1" + + " group by job_id) " + return countSql +} +func generateInferenceTwoCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "INFERENCE" + + " and type=" + "1" + + " group by job_id) " + return countSql +} From 851262f18a4a516cfdc54a5ed65395203934badd Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 23 Mar 2022 11:12:06 +0800 Subject: [PATCH 002/143] update --- models/cloudbrain_static.go | 47 +++++++++++- routers/api/v1/repo/cloudbrain_dashboard.go | 108 ++++++++++------------------ 2 files changed, 81 insertions(+), 74 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 9898ebf92..f388caae7 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,5 +1,48 @@ package models -func CountBrainStatByRawSql(sql string) (int64, error) { - return xStatistic.SQL(sql).Count() +import ( + "strconv" + "time" +) + +func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + return xStatistic.SQL(countSql).Count() +} + +func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "BENCHMARK" + + " and type=" + "0" + return xStatistic.SQL(countSql).Count() +} +func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + " and type=" + "2" + return xStatistic.SQL(countSql).Count() +} +func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "TRAIN" + + " and type=" + "1" + return xStatistic.SQL(countSql).Count() +} +func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "INFERENCE" + + " and type=" + "1" + return xStatistic.SQL(countSql).Count() } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index d39f5c373..66df42d88 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -2,14 +2,20 @@ package repo import ( "net/http" - "strconv" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" ) +type CloudbrainsPeriodData struct { + DebugOneCount int64 `json:"debugOneCount"` + BenchmarkOneCount int64 `json:"benchmarkOneCount"` + DebugTwoCount int64 `json:"debugTwoCount"` + TrainTwoCount int64 `json:"trainTwoCount"` + InferenceTwoCount int64 `json:"inferenceTwoCount"` +} + func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() @@ -24,7 +30,6 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } - q := ctx.QueryTrim("q") page := ctx.QueryInt("page") if page <= 0 { page = 1 @@ -33,87 +38,46 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { if pageSize <= 0 { pageSize = DEFAULT_PAGE_SIZE } - orderBy := getOrderBy(ctx) - latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime() + debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime) if err != nil { - log.Error("Can not query the last updated time.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + log.Error("Can not query debugOneCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) return } - - DebugOneCountSql := generateDebugOneCountSql(beginTime, endTime) - DebugOneTotal, err := models.CountBrainStatByRawSql(beginTime, endTime) + benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime) if err != nil { - log.Error("Can not query total count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + log.Error("Can not query benchmarkCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) return } - DebugTwoCountSql := generateDebugTwoCountSql(beginTime, endTime) - DebugOneTotal, err := models.CountBrainStatByRawSql(DebugTwoCountSql) + debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) if err != nil { - log.Error("Can not query total count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + log.Error("Can not query debugTwoCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error")) return } - - sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) - - projectsPeriodData := ProjectsPeriodData{ - RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), - PageSize: pageSize, - TotalPage: getTotalPage(DebugOneTotal, pageSize), - TotalCount: DebugOneTotal, - LastUpdatedTime: latestUpdatedTime, - PageRecords: models.GetRepoStatisticByRawSql(sql), + trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query DebugOneTotal count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error")) + return + } + inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query inferenceTwoCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error")) + return } - ctx.JSON(http.StatusOK, projectsPeriodData) + cloudbrainsPeriodData := CloudbrainsPeriodData{ + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + } -} + ctx.JSON(http.StatusOK, cloudbrainsPeriodData) -func generateDebugOneCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + - " and type=" + "0" + - " group by job_id) " - return countSql -} -func generateBenchmarkOneCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "BENCHMARK" + - " and type=" + "0" + - " group by job_id) " - return countSql -} -func generateDebugTwoCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + - " and type=" + "2" + - " group by job_id) " - return countSql -} -func generateTrainTwoCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "TRAIN" + - " and type=" + "1" + - " group by job_id) " - return countSql -} -func generateInferenceTwoCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "INFERENCE" + - " and type=" + "1" + - " group by job_id) " - return countSql } From 0034d1f23068e87061bdaad427f5942d553ee37d Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 23 Mar 2022 16:55:24 +0800 Subject: [PATCH 003/143] update --- models/cloudbrain_static.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index f388caae7..e6bdd7848 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -9,40 +9,41 @@ func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + " and job_type ='" + string(JobTypeDebug) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" - return xStatistic.SQL(countSql).Count() + return x.SQL(countSql).Count() } func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "BENCHMARK" + - " and type=" + "0" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeBenchmark) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + return x.SQL(countSql).Count() } func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + - " and type=" + "2" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeDebug) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() } func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "TRAIN" + - " and type=" + "1" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeTrain) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() } func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "INFERENCE" + - " and type=" + "1" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeInference) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() } From 43358dee5eca837cabfd59a41496e498ec63ef7e Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 24 Mar 2022 10:42:28 +0800 Subject: [PATCH 004/143] update --- routers/repo/repo_statistic.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 468e6fa85..8c7ea0fe5 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -17,6 +17,7 @@ import ( func StatisticAuto() { RepoStatisticAuto() TimingCountData() + CloudbrainStatisticAuto() } //auto daily @@ -334,3 +335,12 @@ func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string } return nil } + +func CloudbrainStatisticAuto() { + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + CloudbrainStatisticDaily(yesterday) +} + +func CloudbrainStatisticDaily(date string) { + return +} From 3ea149bada66466eadf721bed4ebf5dba6f180f3 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 25 Mar 2022 09:46:55 +0800 Subject: [PATCH 005/143] update --- models/cloudbrain_static.go | 17 +++++++++++++++++ modules/cron/tasks_basic.go | 11 +++++++++++ routers/repo/repo_statistic.go | 1 - 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index e6bdd7848..f646daf0c 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -3,8 +3,25 @@ package models import ( "strconv" "time" + + "code.gitea.io/gitea/modules/timeutil" ) +// Cloudbrain statistic info of all CloudbrainTasks +type CloudbrainStatistic struct { + ID int64 `xorm:"pk autoincr" json:"-"` + Date string `xorm:"unique(s) NOT NULL" json:"date"` + NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` + NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` + NumTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"` + NumDubugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"` + NumTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"` + NumInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"` + + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` +} + func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index b9838e66f..011cc8dba 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -173,6 +173,16 @@ func registerHandleRepoAndUserStatistic() { return nil }) } +func registerHandleCloudbrainStatistic() { + RegisterTaskFatal("handle_cloudbrain_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.CloudbrainStatisticAuto() + return nil + }) +} func registerHandleSummaryStatistic() { RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ @@ -212,6 +222,7 @@ func initBasicTasks() { registerHandleBlockChainUnSuccessCommits() registerHandleRepoAndUserStatistic() + registerHandleCloudbrainStatistic() registerHandleSummaryStatistic() registerSyncCloudbrainStatus() diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 8c7ea0fe5..b889eb951 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -17,7 +17,6 @@ import ( func StatisticAuto() { RepoStatisticAuto() TimingCountData() - CloudbrainStatisticAuto() } //auto daily From 6871c8c15e8b1add99775b8fc839ff22876b4ce8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 28 Mar 2022 18:03:42 +0800 Subject: [PATCH 006/143] update --- models/cloudbrain_static.go | 119 +++++++++++++++++++++++++++++++++++ routers/repo/cloudbrain_statistic.go | 115 +++++++++++++++++++++++++++++++++ routers/repo/repo_statistic.go | 9 --- 3 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 routers/repo/cloudbrain_statistic.go diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index f646daf0c..1b7bef01a 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,9 +1,11 @@ package models import ( + "fmt" "strconv" "time" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" ) @@ -11,12 +13,24 @@ import ( type CloudbrainStatistic struct { ID int64 `xorm:"pk autoincr" json:"-"` Date string `xorm:"unique(s) NOT NULL" json:"date"` + WhichHour int64 `xorm:"NOT NULL DEFAULT -1" json:"whichHour"` NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` NumTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"` NumDubugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"` NumTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"` NumInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"` + NumCloudOneAll int64 `xorm:"NOT NULL DEFAULT 0" json:"numCloudOneAll"` + NumCloudTwoAll int64 `xorm:"NOT NULL DEFAULT 0" json:"numCloudTwoAll"` + + DurationDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDubugOne"` + DurationBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationBenchmarkOne"` + DurationTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainOne"` + DurationDebugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDebugTwo"` + DurationTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainTwo"` + DurationInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationInferenceTwo"` + DurationCloudOneAll int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudOneAll"` + DurationCloudTwoAll int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudTwoAll"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` @@ -31,6 +45,32 @@ func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error return x.SQL(countSql).Count() } +func GetDebugOneDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} + +func GenerateTrainOneCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeTrain) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + + return x.SQL(countSql).Count() +} +func GetTrainOneDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + @@ -40,6 +80,14 @@ func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, e " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" return x.SQL(countSql).Count() } +func GetBenchmarkOneDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -48,6 +96,13 @@ func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" return x.SQL(countSql).Count() } +func GetDebugTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -56,6 +111,13 @@ func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" return x.SQL(countSql).Count() } +func GetTrainTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -64,3 +126,60 @@ func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, e " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" return x.SQL(countSql).Count() } +func GetInferenceTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} + +func DeleteCloudbrainStatisticDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&CloudbrainStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func GetHourStatTime(timeStr string, whichHour int64) (time.Time, time.Time) { + t, _ := time.Parse("2006-01-02", timeStr) + timeNumber := t.Unix() + beginTimeNumber := timeNumber - 8*60*60 + whichHour*60*60 + endTimeNumber := beginTimeNumber + 1*60*60 + beginTime := time.Unix(beginTimeNumber, 0) + endTime := time.Unix(endTimeNumber, 0) + log.Info("%s, %s", beginTime, endTime) + + return beginTime, endTime +} +func GetDayStatTime(timeStr string) (time.Time, time.Time) { + t, _ := time.Parse("2006-01-02", timeStr) + timeNumber := t.Unix() + beginTimeNumber := timeNumber - 8*60*60 + endTimeNumber := beginTimeNumber + 16*60*60 + beginTime := time.Unix(beginTimeNumber, 0) + endTime := time.Unix(endTimeNumber, 0) + log.Info("%s, %s", beginTime, endTime) + + return beginTime, endTime +} + +func CreateCloudbrainStatistic(cloudbrainStat *CloudbrainStatistic) (err error) { + if _, err = xStatistic.Insert(cloudbrainStat); err != nil { + return err + } + return nil +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go new file mode 100644 index 000000000..926fa68c7 --- /dev/null +++ b/routers/repo/cloudbrain_statistic.go @@ -0,0 +1,115 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer" +) + +func CloudbrainStatisticAuto() { + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + CloudbrainStatisticDaily(yesterday) +} + +func CloudbrainStatisticDaily(date string) { + log.Info("%s", date) + log.Info("begin Repo Statistic") + warnEmailMessage := "云脑任务统计信息入库失败，请尽快定位。" + if err := models.DeleteCloudbrainStatisticDaily(date); err != nil { + log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) + return + } + + //0 to 23 for each hour, 25 for all 24 hours + var WhichHour [25]int64 = [25]int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25} + for _, whichHour := range WhichHour { + log.Info("start statistic: %s", date) + var beginTime, endTime time.Time + beginTime, endTime = models.GetHourStatTime(date, whichHour) + if whichHour == 25 { + beginTime, endTime = models.GetDayStatTime(date) + } + numDubugOne, err := models.GenerateDebugOneCount(beginTime, endTime) + if err != nil { + log.Error("GenerateDebugOneCount failed(%s): %v", numDubugOne, err) + } + numBenchmarkOne, err := models.GenerateBenchmarkOneCount(beginTime, endTime) + if err != nil { + log.Error("GenerateBenchmarkOneCount failed(%s): %v", numBenchmarkOne, err) + } + numTrainOne, err := models.GenerateTrainOneCount(beginTime, endTime) + if err != nil { + log.Error("GenerateTrainOneCount failed(%s): %v", numTrainOne, err) + } + numDebugTwo, err := models.GenerateDebugTwoCount(beginTime, endTime) + if err != nil { + log.Error("GenerateDebugTwoCount failed(%s): %v", numDebugTwo, err) + } + numTrainTwo, err := models.GenerateTrainTwoCount(beginTime, endTime) + if err != nil { + log.Error("GenerateTrainTwoCount failed(%s): %v", numTrainTwo, err) + } + numInferenceTwo, err := models.GenerateInferenceTwoCount(beginTime, endTime) + if err != nil { + log.Error("GenerateInferenceTwoCount failed(%s): %v", numInferenceTwo, err) + } + numCloudOneAll := numDubugOne + numBenchmarkOne + numTrainOne + numCloudTwoAll := numDebugTwo + numTrainTwo + numInferenceTwo + + durationDubugOne, err := models.GetDebugOneDuration(beginTime, endTime) + if err != nil { + log.Error("GetDebugOneDuration failed(%s): %v", durationDubugOne, err) + } + durationBenchmarkOne, err := models.GetBenchmarkOneDuration(beginTime, endTime) + if err != nil { + log.Error("GetBenchmarkOneDuration failed(%s): %v", durationBenchmarkOne, err) + } + durationTrainOne, err := models.GetTrainOneDuration(beginTime, endTime) + if err != nil { + log.Error("GetTrainOneDuration failed(%s): %v", durationTrainOne, err) + } + durationDebugTwo, err := models.GetDebugTwoDuration(beginTime, endTime) + if err != nil { + log.Error("GetDebugTwoDuration failed(%s): %v", durationDebugTwo, err) + } + durationTrainTwo, err := models.GetTrainTwoDuration(beginTime, endTime) + if err != nil { + log.Error("GetTrainTwoDuration failed(%s): %v", durationTrainTwo, err) + } + durationInferenceTwo, err := models.GetInferenceTwoDuration(beginTime, endTime) + if err != nil { + log.Error("GetInferenceTwoDuration failed(%s): %v", durationInferenceTwo, err) + } + durationCloudOneAll := durationDubugOne + durationBenchmarkOne + durationTrainOne + durationCloudTwoAll := durationDebugTwo + durationTrainTwo + durationInferenceTwo + + err = models.CreateCloudbrainStatistic(&models.CloudbrainStatistic{ + Date: date, + WhichHour: whichHour, + NumDubugOne: numDubugOne, + NumBenchmarkOne: numBenchmarkOne, + NumTrainOne: numTrainOne, + NumDubugTwo: numDebugTwo, + NumTrainTwo: numTrainTwo, + NumInferenceTwo: numInferenceTwo, + NumCloudOneAll: numCloudOneAll, + NumCloudTwoAll: numCloudTwoAll, + DurationDubugOne: durationDubugOne, + DurationBenchmarkOne: durationBenchmarkOne, + DurationTrainOne: durationTrainOne, + DurationDebugTwo: durationDebugTwo, + DurationTrainTwo: durationTrainTwo, + DurationInferenceTwo: durationInferenceTwo, + DurationCloudOneAll: durationCloudOneAll, + DurationCloudTwoAll: durationCloudTwoAll, + }) + if err != nil { + log.Error("CreateCloudbrainStatistic(%s) failed:%v", date, err.Error()) + return + } + } +} diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index b889eb951..468e6fa85 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -334,12 +334,3 @@ func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string } return nil } - -func CloudbrainStatisticAuto() { - yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") - CloudbrainStatisticDaily(yesterday) -} - -func CloudbrainStatisticDaily(date string) { - return -} From 34cf97e9bb7d7e97407a12585d3928ec69f5f8b5 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 29 Mar 2022 09:42:48 +0800 Subject: [PATCH 007/143] update --- routers/api/v1/api.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6118db265..89914f688 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -556,16 +556,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) //cloudbrain board m.Group("/cloudbrainboard", func() { - m.Get("/restoreFork", repo.RestoreForkNumber) - m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) - m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) m.Group("/cloudbrain", func() { m.Get("", repo.GetAllCloudbrainsPeriodStatistics) - m.Group("/:id", func() { - m.Get("", repo.GetProjectLatestStatistics) - m.Get("/period", repo.GetProjectPeriodStatistics) - - }) }) }, operationReq) From 3696ce23c414dd9bfd56d82c82f74a9ee8badaa8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 31 Mar 2022 10:33:25 +0800 Subject: [PATCH 008/143] update --- models/cloudbrain_static.go | 2 +- models/models.go | 1 + routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 74 +++++++++++++++++++++++++++++ routers/repo/cloudbrain_statistic.go | 4 +- routers/routes/routes.go | 1 + 6 files changed, 80 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 1b7bef01a..4d0e7cfaa 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -12,7 +12,7 @@ import ( // Cloudbrain statistic info of all CloudbrainTasks type CloudbrainStatistic struct { ID int64 `xorm:"pk autoincr" json:"-"` - Date string `xorm:"unique(s) NOT NULL" json:"date"` + Date string `xorm:"NOT NULL DEFAULT 0" json:"date"` WhichHour int64 `xorm:"NOT NULL DEFAULT -1" json:"whichHour"` NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` diff --git a/models/models.go b/models/models.go index 0f4679b4f..02cfb3144 100755 --- a/models/models.go +++ b/models/models.go @@ -151,6 +151,7 @@ func init() { new(UserBusinessAnalysisCurrentWeek), new(UserBusinessAnalysisYesterday), new(UserLoginLog), + new(CloudbrainStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 89914f688..87af1d19b 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -556,6 +556,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) //cloudbrain board m.Group("/cloudbrainboard", func() { + m.Get("/downloadtable", repo.ServeCloudbrainPeriodStatisticsFile) m.Group("/cloudbrain", func() { m.Get("", repo.GetAllCloudbrainsPeriodStatistics) }) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 66df42d88..358aa6c37 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -2,10 +2,12 @@ package repo import ( "net/http" + "net/url" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "github.com/360EntSecGroup-Skylar/excelize/v2" ) type CloudbrainsPeriodData struct { @@ -81,3 +83,75 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } +func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 1000 + orderBy := getOrderBy(ctx) + + _, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + countSql := generateCountSql(beginTime, endTime, latestDate, q) + total, err := models.CountRepoStatByRawSql(countSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") + fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) + + totalPage := getTotalPage(total, pageSize) + + f := excelize.NewFile() + + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") + + for k, v := range allProjectsPeroidHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + + var row = 2 + for i := 0; i <= totalPage; i++ { + + pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + for _, record := range pageRecords { + + for k, v := range allProjectsPeroidValues(row, record, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + + } + + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) + +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 926fa68c7..16137ed2e 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -10,13 +10,13 @@ import ( ) func CloudbrainStatisticAuto() { - yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + yesterday := time.Now().AddDate(0, 0, 0).Format("2006-01-02") CloudbrainStatisticDaily(yesterday) } func CloudbrainStatisticDaily(date string) { log.Info("%s", date) - log.Info("begin Repo Statistic") + log.Info("begin Cloudbrain Statistic") warnEmailMessage := "云脑任务统计信息入库失败，请尽快定位。" if err := models.DeleteCloudbrainStatisticDaily(date); err != nil { log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error()) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 2d146c2c6..a2e969a60 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -990,6 +990,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/cloudbrain", func() { + m.Get("/test", repo.CloudbrainStatisticAuto) m.Group("/:id", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDebug) From 29fdbd5d580f6cd341255332616f0a2a3bea1070 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 6 Apr 2022 18:15:35 +0800 Subject: [PATCH 009/143] update --- models/cloudbrain_static.go | 10 + routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 272 +++++++++++++++++++++++++++- routers/repo/cloudbrain_statistic.go | 2 +- 4 files changed, 283 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 4d0e7cfaa..8e9329ce2 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -36,6 +36,16 @@ type CloudbrainStatistic struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` } +type TimeCloudbrainNum struct { + Time time.Time `json:"time"` + DebugOneCount int64 `json:"debugOneCount"` + BenchmarkOneCount int64 `json:"benchmarkOneCount"` + TrainOneCount int64 `json:"trainOneCount"` + DebugTwoCount int64 `json:"debugTwoCount"` + TrainTwoCount int64 `json:"trainTwoCount"` + InferenceTwoCount int64 `json:"inferenceTwoCount"` +} + func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index fded809f4..c03ffbe02 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -559,6 +559,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrainboard", func() { m.Get("/downloadtable", repo.ServeCloudbrainPeriodStatisticsFile) m.Group("/cloudbrain", func() { + m.Get("/trend", repo.GetAllCloudbrainsTrend) m.Get("", repo.GetAllCloudbrainsPeriodStatistics) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 358aa6c37..609783bfb 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1,8 +1,10 @@ package repo import ( + "fmt" "net/http" "net/url" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -13,10 +15,52 @@ import ( type CloudbrainsPeriodData struct { DebugOneCount int64 `json:"debugOneCount"` BenchmarkOneCount int64 `json:"benchmarkOneCount"` + TrainOneCount int64 `json:"trainOneCount"` DebugTwoCount int64 `json:"debugTwoCount"` TrainTwoCount int64 `json:"trainTwoCount"` InferenceTwoCount int64 `json:"inferenceTwoCount"` } +type TimeCloudbrainsNum struct { + TimeCloudbrainNum []models.TimeCloudbrainNum `json:"imeCloudbrainNum"` +} + +func GetAllCloudbrainsTrend(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + timeCloudbrainNum, beginTime, endTime, err := getCloudbrainTrendTime(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pagesize") + if pageSize <= 0 { + pageSize = DEFAULT_PAGE_SIZE + } + + cloudbrainsPeriodData := TimeCloudbrainsNum{ + TimeCloudbrainNum: timeCloudbrainNum, + } + now := time.Now() + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) + testtime := now.AddDate(0, -1, 1) + fmt.Printf("beginTime is: %s", beginTime) + fmt.Printf("endTime is: %s", endTime) + fmt.Printf("testtime is:%s", testtime) + + ctx.JSON(http.StatusOK, cloudbrainsPeriodData) + +} func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { @@ -53,6 +97,12 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) return } + trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query trainOneCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error")) + return + } debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query debugTwoCount.", err) @@ -75,10 +125,18 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { cloudbrainsPeriodData := CloudbrainsPeriodData{ DebugOneCount: debugOneCount, BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, DebugTwoCount: debugTwoCount, TrainTwoCount: trainTwoCount, InferenceTwoCount: inferenceTwoCount, } + now := time.Now() + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) + testtime := now.AddDate(0, -1, 1) + fmt.Printf("beginTime is: %s", beginTime) + fmt.Printf("endTime is: %s", endTime) + fmt.Printf("testtime is:%s", testtime) ctx.JSON(http.StatusOK, cloudbrainsPeriodData) @@ -91,7 +149,7 @@ func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } - beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) if err != nil { log.Error("Parameter is wrong", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) @@ -155,3 +213,215 @@ func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { f.WriteTo(ctx.Resp) } + +func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) { + queryType := ctx.QueryTrim("type") + now := time.Now() + recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var err error + if queryType != "" { + + if queryType == "all" { + beginTime = recordBeginTimeTemp + endTime = now + } else if queryType == "yesterday" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) + + } else { + return now, now, fmt.Errorf("The value of type parameter is wrong.") + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = recordBeginTimeTemp + endTime = now + + } else { + + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + return now, now, err + } + + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + return now, now, err + } + + beginTime = beginTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 1) + } + + } + + if beginTime.Before(recordBeginTimeTemp) { + beginTime = recordBeginTimeTemp + } + + return beginTime, endTime, nil + +} + +func getCloudbrainTrendTime(ctx *context.Context, recordBeginTime time.Time) ([]models.TimeCloudbrainNum, time.Time, time.Time, error) { + queryType := ctx.QueryTrim("type") + now := time.Now() + recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var err error + timeCloudbrainNum := make([]models.TimeCloudbrainNum, 0) + if queryType != "" { + + if queryType == "all" { + beginTime = recordBeginTimeTemp + endTime = now + } else if queryType == "yesterday" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime1 := beginTime.AddDate(0, 0, 1) + for endTime1.Before(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err := getCloudbrainCount(beginTime, endTime1) + if err != nil { + log.Error("Can not query debugOneCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) + } + timeCloudbrainNum = append(timeCloudbrainNum, models.TimeCloudbrainNum{ + Time: beginTime, + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + }) + beginTime = endTime1 + endTime1 = beginTime.AddDate(0, 0, 1) + } + return timeCloudbrainNum, beginTime, endTime, nil + + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) + + } else { + return timeCloudbrainNum, now, now, fmt.Errorf("The value of type parameter is wrong.") + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = recordBeginTimeTemp + endTime = now + + } else { + + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + return timeCloudbrainNum, now, now, err + } + + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + return timeCloudbrainNum, now, now, err + } + + beginTime = beginTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 1) + } + + } + + if beginTime.Before(recordBeginTimeTemp) { + beginTime = recordBeginTimeTemp + } + + return timeCloudbrainNum, beginTime, endTime, nil + +} + +func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, error) { + debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query debugOneCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query benchmarkCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query trainOneCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query debugTwoCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query DebugOneTotal count.", err) + return 0, 0, 0, 0, 0, 0, err + } + inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query inferenceTwoCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 16137ed2e..142b0c14e 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -10,7 +10,7 @@ import ( ) func CloudbrainStatisticAuto() { - yesterday := time.Now().AddDate(0, 0, 0).Format("2006-01-02") + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") CloudbrainStatisticDaily(yesterday) } From f7bff308e2b7ba9f43890da10f5da948faf3bc20 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 7 Apr 2022 17:01:30 +0800 Subject: [PATCH 010/143] update --- models/cloudbrain_static.go | 10 - modules/setting/setting.go | 26 +- routers/api/v1/repo/cloudbrain_dashboard.go | 528 +++++++++++++++++----------- 3 files changed, 334 insertions(+), 230 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 8e9329ce2..4d0e7cfaa 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -36,16 +36,6 @@ type CloudbrainStatistic struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` } -type TimeCloudbrainNum struct { - Time time.Time `json:"time"` - DebugOneCount int64 `json:"debugOneCount"` - BenchmarkOneCount int64 `json:"benchmarkOneCount"` - TrainOneCount int64 `json:"trainOneCount"` - DebugTwoCount int64 `json:"debugTwoCount"` - TrainTwoCount int64 `json:"trainTwoCount"` - InferenceTwoCount int64 `json:"inferenceTwoCount"` -} - func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 26f068193..19620f5db 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -452,18 +452,19 @@ var ( DecompressOBSTaskName string //cloudbrain config - CBAuthUser string - CBAuthPassword string - RestServerHost string - JobPath string - CBCodePathPrefix string - JobType string - GpuTypes string - DebugServerHost string - ResourceSpecs string - MaxDuration int64 - TrainGpuTypes string - TrainResourceSpecs string + CBAuthUser string + CBAuthPassword string + RestServerHost string + JobPath string + CBCodePathPrefix string + JobType string + GpuTypes string + DebugServerHost string + ResourceSpecs string + MaxDuration int64 + TrainGpuTypes string + TrainResourceSpecs string + BrainRecordBeginTime string //benchmark config IsBenchmarkEnabled bool @@ -1290,6 +1291,7 @@ func NewContext() { MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") + BrainRecordBeginTime = sec.Key("brain_record_beigin_time").MustString("2021-01-01") sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 609783bfb..85bea2189 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "github.com/360EntSecGroup-Skylar/excelize/v2" ) @@ -21,42 +22,172 @@ type CloudbrainsPeriodData struct { InferenceTwoCount int64 `json:"inferenceTwoCount"` } type TimeCloudbrainsNum struct { - TimeCloudbrainNum []models.TimeCloudbrainNum `json:"imeCloudbrainNum"` + TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"` +} +type DateCloudbrainNum struct { + Date string `json:"date"` + DebugOneCount int64 `json:"debugOneCount"` + BenchmarkOneCount int64 `json:"benchmarkOneCount"` + TrainOneCount int64 `json:"trainOneCount"` + DebugTwoCount int64 `json:"debugTwoCount"` + TrainTwoCount int64 `json:"trainTwoCount"` + InferenceTwoCount int64 `json:"inferenceTwoCount"` + CloudbrainOneCount int64 `json:"cloudbrainOneCount"` + CloudbrainTwoCount int64 `json:"cloudbrainTwoCount"` + CloudbrainCount int64 `json:"cloudbrainCount"` } func GetAllCloudbrainsTrend(ctx *context.Context) { - - recordBeginTime, err := getRecordBeginTime() + brainRecordBeginTime, err := getBrainRecordBeginTime() if err != nil { - log.Error("Can not get record begin time", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + log.Error("Can not get brain record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.brain_record_begintime_get_err")) return } - timeCloudbrainNum, beginTime, endTime, err := getCloudbrainTrendTime(ctx, recordBeginTime) - if err != nil { - log.Error("Parameter is wrong", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) - return - } - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pageSize := ctx.QueryInt("pagesize") - if pageSize <= 0 { - pageSize = DEFAULT_PAGE_SIZE - } + queryType := ctx.QueryTrim("type") + now := time.Now() + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var endTimeTemp time.Time + dateCloudbrainNum := make([]DateCloudbrainNum, 0) + if queryType != "" { + if queryType == "all" { + beginTime = brainRecordBeginTime + endTime = now + endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getYearCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.Add(time.Hour) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getHourCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) //begin from monday + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = brainRecordBeginTime + endTime = now + endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + days := (endTime.Unix() - beginTime.Unix()) / 3600 / 24 + if 1 < days { + endTimeTemp = beginTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 2) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if 0 < days || days <= 1 { + endTimeTemp = beginTime.Add(time.Hour) + dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getHourCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getHourCloudbrainNum_get_error")) + return + } + } else { + return + } + } + } cloudbrainsPeriodData := TimeCloudbrainsNum{ - TimeCloudbrainNum: timeCloudbrainNum, + TimeCloudbrainNum: dateCloudbrainNum, } - now := time.Now() - endTime = now - beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) - testtime := now.AddDate(0, -1, 1) - fmt.Printf("beginTime is: %s", beginTime) - fmt.Printf("endTime is: %s", endTime) - fmt.Printf("testtime is:%s", testtime) ctx.JSON(http.StatusOK, cloudbrainsPeriodData) @@ -141,79 +272,6 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } -func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { - - recordBeginTime, err := getRecordBeginTime() - if err != nil { - log.Error("Can not get record begin time", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return - } - beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) - if err != nil { - log.Error("Parameter is wrong", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) - return - } - q := ctx.QueryTrim("q") - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pageSize := 1000 - orderBy := getOrderBy(ctx) - - _, latestDate, err := models.GetRepoStatLastUpdatedTime() - if err != nil { - log.Error("Can not query the last updated time.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) - return - } - - countSql := generateCountSql(beginTime, endTime, latestDate, q) - total, err := models.CountRepoStatByRawSql(countSql) - if err != nil { - log.Error("Can not query total count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) - return - } - var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") - fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) - - totalPage := getTotalPage(total, pageSize) - - f := excelize.NewFile() - - index := f.NewSheet(projectAnalysis) - f.DeleteSheet("Sheet1") - - for k, v := range allProjectsPeroidHeader(ctx) { - f.SetCellValue(projectAnalysis, k, v) - } - - var row = 2 - for i := 0; i <= totalPage; i++ { - - pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) - for _, record := range pageRecords { - - for k, v := range allProjectsPeroidValues(row, record, ctx) { - f.SetCellValue(projectAnalysis, k, v) - } - row++ - - } - - } - f.SetActiveSheet(index) - - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - - f.WriteTo(ctx.Resp) - -} - func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) { queryType := ctx.QueryTrim("type") now := time.Now() @@ -292,136 +350,190 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t } -func getCloudbrainTrendTime(ctx *context.Context, recordBeginTime time.Time) ([]models.TimeCloudbrainNum, time.Time, time.Time, error) { - queryType := ctx.QueryTrim("type") - now := time.Now() - recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) - - beginTimeStr := ctx.QueryTrim("beginTime") - endTimeStr := ctx.QueryTrim("endTime") - var beginTime time.Time - var endTime time.Time - var err error - timeCloudbrainNum := make([]models.TimeCloudbrainNum, 0) - if queryType != "" { - - if queryType == "all" { - beginTime = recordBeginTimeTemp - endTime = now - } else if queryType == "yesterday" { - endTime = now - beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) - - } else if queryType == "current_week" { - beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday - beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) - endTime = now - } else if queryType == "current_month" { - endTime = now - beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) - endTime1 := beginTime.AddDate(0, 0, 1) - for endTime1.Before(endTime) { - debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err := getCloudbrainCount(beginTime, endTime1) - if err != nil { - log.Error("Can not query debugOneCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) - } - timeCloudbrainNum = append(timeCloudbrainNum, models.TimeCloudbrainNum{ - Time: beginTime, - DebugOneCount: debugOneCount, - BenchmarkOneCount: benchmarkOneCount, - TrainOneCount: trainOneCount, - DebugTwoCount: debugTwoCount, - TrainTwoCount: trainTwoCount, - InferenceTwoCount: inferenceTwoCount, - }) - beginTime = endTime1 - endTime1 = beginTime.AddDate(0, 0, 1) - } - return timeCloudbrainNum, beginTime, endTime, nil - - } else if queryType == "monthly" { - endTime = now - beginTime = now.AddDate(0, -1, 1) - beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) - - } else if queryType == "current_year" { - endTime = now - beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) - - } else if queryType == "last_month" { - - lastMonthTime := now.AddDate(0, -1, 0) - beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) - endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) - - } else { - return timeCloudbrainNum, now, now, fmt.Errorf("The value of type parameter is wrong.") - - } - - } else { - if beginTimeStr == "" || endTimeStr == "" { - //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 - beginTime = recordBeginTimeTemp - endTime = now - - } else { - - beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) - if err != nil { - return timeCloudbrainNum, now, now, err - } - - endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) - if err != nil { - return timeCloudbrainNum, now, now, err - } - - beginTime = beginTime.AddDate(0, 0, 1) - endTime = endTime.AddDate(0, 0, 1) - } - - } - - if beginTime.Before(recordBeginTimeTemp) { - beginTime = recordBeginTimeTemp - } - - return timeCloudbrainNum, beginTime, endTime, nil - -} - -func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, error) { +func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, int64, int64, int64, error) { debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime) if err != nil { log.Error("Can not query debugOneCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime) if err != nil { log.Error("Can not query benchmarkCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime) if err != nil { log.Error("Can not query trainOneCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query debugTwoCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query DebugOneTotal count.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query inferenceTwoCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } - return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err + cloudbrainOneCount := debugOneCount + benchmarkOneCount + trainOneCount + cloudbrainTwoCount := debugTwoCount + trainTwoCount + inferenceTwoCount + cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount + return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err +} +func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + dayCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + if err != nil { + log.Error("Can not query getCloudbrainCount.", err) + return nil, err + } + dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format(time.RFC3339), + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + CloudbrainOneCount: cloudbrainOneCount, + CloudbrainTwoCount: cloudbrainTwoCount, + CloudbrainCount: cloudbrainCount, + }) + beginTime = endTimeTemp + endTimeTemp = beginTime.Add(time.Hour) + } + return dayCloudbrainNum, nil +} +func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + dayCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + if err != nil { + log.Error("Can not query getCloudbrainCount.", err) + return nil, err + } + dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format("2006-01-02"), + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + CloudbrainOneCount: cloudbrainOneCount, + CloudbrainTwoCount: cloudbrainTwoCount, + CloudbrainCount: cloudbrainCount, + }) + beginTime = endTimeTemp + endTimeTemp = beginTime.AddDate(0, 0, 1) + } + return dayCloudbrainNum, nil +} +func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + yearCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + if err != nil { + log.Error("Can not query getCloudbrainCount.", err) + return nil, err + } + yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format("2006-01"), + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + CloudbrainOneCount: cloudbrainOneCount, + CloudbrainTwoCount: cloudbrainTwoCount, + CloudbrainCount: cloudbrainCount, + }) + beginTime = endTimeTemp + endTimeTemp = beginTime.AddDate(0, 1, 0) + } + return yearCloudbrainNum, nil +} +func getBrainRecordBeginTime() (time.Time, error) { + return time.ParseInLocation(DATE_FORMAT, setting.BrainRecordBeginTime, time.Local) +} + +func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 1000 + orderBy := getOrderBy(ctx) + + _, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + countSql := generateCountSql(beginTime, endTime, latestDate, q) + total, err := models.CountRepoStatByRawSql(countSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") + fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) + + totalPage := getTotalPage(total, pageSize) + + f := excelize.NewFile() + + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") + + for k, v := range allProjectsPeroidHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + + var row = 2 + for i := 0; i <= totalPage; i++ { + + pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + for _, record := range pageRecords { + + for k, v := range allProjectsPeroidValues(row, record, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + + } + + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) + } From 05ab115942601cc6716acb905e6f2a3ae482573c Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 8 Apr 2022 18:25:35 +0800 Subject: [PATCH 011/143] update --- models/cloudbrain.go | 65 +- options/locale/locale_zh-CN.ini | 2 + routers/api/v1/api.go | 5 +- routers/api/v1/repo/cloudbrain_dashboard.go | 123 ++++ web_src/js/components/BrainAnalysis.vue | 1008 +++++++++++++++++++++++++++ web_src/js/components/DataAnalysis.vue | 10 + 6 files changed, 1210 insertions(+), 3 deletions(-) create mode 100644 web_src/js/components/BrainAnalysis.vue diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1662dcd96..7f7b9bcd6 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,13 +1,14 @@ package models import ( - "code.gitea.io/gitea/modules/util" "encoding/json" "fmt" "strconv" "strings" "time" + "code.gitea.io/gitea/modules/util" + "xorm.io/builder" "xorm.io/xorm" @@ -1564,3 +1565,65 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { return nil } + +func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + if (opts.Type) >= 0 { + cond = cond.And( + builder.Eq{"cloudbrain.type": opts.Type}, + ) + } + + var count int64 + var err error + condition := "cloudbrain.user_id = `user`.id" + if len(opts.Keyword) == 0 { + count, err = sess.Where(cond).Count(new(Cloudbrain)) + } else { + lowerKeyWord := strings.ToLower(opts.Keyword) + + cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) + count, err = sess.Table(&Cloudbrain{}).Where(cond). + Join("left", "`user`", condition).Count(new(CloudbrainInfo)) + + } + + if err != nil { + return nil, 0, fmt.Errorf("Count: %v", err) + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + sess.Limit(opts.PageSize, start) + } + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Join("left", "`user`", condition). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + if opts.NeedRepoInfo { + var ids []int64 + for _, task := range cloudbrains { + ids = append(ids, task.RepoID) + } + repositoryMap, err := GetRepositoriesMapByIDs(ids) + if err == nil { + for _, task := range cloudbrains { + task.Repo = repositoryMap[task.RepoID] + } + } + + } + + return cloudbrains, count, nil +} diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index d26065363..06ea4bd7f 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1005,6 +1005,8 @@ modelarts.train_job.job_status=任务状态 modelarts.train_job.job_name=任务名称 modelarts.train_job.version=任务版本 modelarts.train_job.start_time=开始时间 +modelarts.train_job.end_time=结束时间 +modelarts.train_job.wait_time=等待时间 modelarts.train_job.dura_time=运行时长 modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c03ffbe02..14bd2e2d1 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -557,10 +557,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) //cloudbrain board m.Group("/cloudbrainboard", func() { - m.Get("/downloadtable", repo.ServeCloudbrainPeriodStatisticsFile) + m.Get("/downloadStatistics", repo.ServeCloudbrainPeriodStatisticsFile) + m.Get("/downloadAll", repo.DownloadCloudBrainBoard) m.Group("/cloudbrain", func() { m.Get("/trend", repo.GetAllCloudbrainsTrend) - m.Get("", repo.GetAllCloudbrainsPeriodStatistics) + m.Get("/statistics", repo.GetAllCloudbrainsPeriodStatistics) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 85bea2189..ab26d7dfc 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -537,3 +537,126 @@ func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { f.WriteTo(ctx.Resp) } +func DownloadCloudBrainBoard(ctx *context.Context) { + + page := 1 + + pageSize := 300 + + var cloudBrain = ctx.Tr("repo.cloudbrain") + fileName := getCloudbrainFileName(cloudBrain) + + _, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: 1, + }, + Type: models.TypeCloudBrainAll, + NeedRepoInfo: false, + }) + + if err != nil { + log.Warn("Can not get cloud brain info", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) + return + } + + totalPage := getTotalPage(total, pageSize) + fmt.Printf("total:%v", total) + fmt.Printf("totalPage:%v", totalPage) + + f := excelize.NewFile() + + index := f.NewSheet(cloudBrain) + f.DeleteSheet("Sheet1") + + for k, v := range allHeader(ctx) { + f.SetCellValue(cloudBrain, k, v) + } + + var row = 2 + for i := 0; i < totalPage; i++ { + + pageRecords, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pageSize, + }, + Type: models.TypeCloudBrainAll, + NeedRepoInfo: true, + }) + if err != nil { + log.Warn("Can not get cloud brain info", err) + continue + } + for _, record := range pageRecords { + + for k, v := range allValues(row, record, ctx) { + f.SetCellValue(cloudBrain, k, v) + } + row++ + + } + + page++ + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) +} + +func getCloudbrainFileName(baseName string) string { + return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx" + +} +func allHeader(ctx *context.Context) map[string]string { + + return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_task_type"), "C1": ctx.Tr("repo.modelarts.status"), + "D1": ctx.Tr("repo.modelarts.createtime"), "E1": ctx.Tr("repo.modelarts.train_job.dura_time"), "F1": ctx.Tr("repo.modelarts.computing_resources"), + "G1": ctx.Tr("repo.cloudbrain_creator"), "H1": ctx.Tr("repo.repo_name"), "I1": ctx.Tr("repo.cloudbrain_task_name"), "J1": ctx.Tr("repo.modelarts.train_job.start_time"), + "K1": ctx.Tr("repo.modelarts.train_job.end_time"), "L1": ctx.Tr("repo.modelarts.train_job.wait_time")} + +} +func allValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { + return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): rs.JobType, getCellName("C", row): rs.Status, + getCellName("D", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("E", row): rs.TrainJobDuration, + getCellName("F", row): rs.ComputeResource, getCellName("G", row): rs.Name, getCellName("H", row): getRepoPathName(rs), + getCellName("I", row): rs.JobName, getCellName("J", row): getBrainStartTime(rs), + getCellName("K", row): getBrainEndTime(rs), getCellName("L", row): getBrainWaitTime(rs), + } +} +func getRepoPathName(rs *models.CloudbrainInfo) string { + if rs.Repo != nil { + return rs.Repo.OwnerName + "/" + rs.Repo.Alias + } + return "" +} +func getBrainStartTime(rs *models.CloudbrainInfo) string { + timeString := time.Unix(int64(rs.Cloudbrain.StartTime), 0).Format(CREATE_TIME_FORMAT) + if timeString != "1970/01/01 08:00:00" { + return timeString + } else { + return "0" + } + +} +func getBrainEndTime(rs *models.CloudbrainInfo) string { + timeString := time.Unix(int64(rs.Cloudbrain.EndTime), 0).Format(CREATE_TIME_FORMAT) + if timeString != "1970/01/01 08:00:00" { + return timeString + } else { + return "0" + } + +} +func getBrainWaitTime(rs *models.CloudbrainInfo) string { + waitTime := rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix + if waitTime <= 0 { + return "0" + } else { + return models.ConvertDurationToStr(int64(waitTime)) + } +} diff --git a/web_src/js/components/BrainAnalysis.vue b/web_src/js/components/BrainAnalysis.vue new file mode 100644 index 000000000..4f096e024 --- /dev/null +++ b/web_src/js/components/BrainAnalysis.vue @@ -0,0 +1,1008 @@ + + + + + diff --git a/web_src/js/components/DataAnalysis.vue b/web_src/js/components/DataAnalysis.vue index ae536db28..6f80c3481 100755 --- a/web_src/js/components/DataAnalysis.vue +++ b/web_src/js/components/DataAnalysis.vue @@ -26,6 +26,14 @@ + + + + + + 云脑分析 + + @@ -33,12 +41,14 @@ - + \ No newline at end of file From 7669454ce941f7e80f391756b7f85ad701836eba Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 16 Jun 2022 19:40:16 +0800 Subject: [PATCH 119/143] fix-2208 --- routers/repo/dataset.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 133262bf3..8b8e4f784 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -616,8 +616,14 @@ func DatasetIsCollaborator(ctx *context.Context, dataset *models.Dataset) bool { repo.GetOwner() if ctx.User != nil { if repo.Owner.IsOrganization() { - if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { - for _, t := range repo.Owner.Teams { + org := repo.Owner + org.Teams, err = org.GetUserTeams(ctx.User.ID) + if err != nil { + log.Error("GetUserTeams error:", err.Error()) + return false + } + if org.IsUserPartOfOrg(ctx.User.ID) { + for _, t := range org.Teams { if t.IsMember(ctx.User.ID) && t.HasRepository(repo.ID) { return true } From 803e8749f440c5886445ca7aa558055a43b33671 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 17 Jun 2022 11:28:50 +0800 Subject: [PATCH 120/143] fix-2285, fix-2311 --- models/dataset.go | 8 ++++++++ routers/home.go | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/models/dataset.go b/models/dataset.go index b7186ac0b..53b63185f 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -181,6 +181,7 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { if len(opts.DatasetIDs) > 0 { subCon := builder.NewCond() subCon = subCon.And(builder.In("dataset.id", opts.DatasetIDs)) + subCon = generateFilterCond(opts, subCon) cond = cond.Or(subCon) } @@ -460,5 +461,12 @@ func GetCollaboratorDatasetIdsByUserID(userID int64) []int64 { _ = x.Table("dataset").Join("INNER", "collaboration", "dataset.repo_id = collaboration.repo_id and collaboration.mode>0 and collaboration.user_id=?", userID). Cols("dataset.id").Find(&datasets) return datasets +} +func GetTeamDatasetIdsByUserID(userID int64) []int64 { + var datasets []int64 + _ = x.Table("dataset").Join("INNER", "team_repo", "dataset.repo_id = team_repo.repo_id"). + Join("INNER", "team_user", "team_repo.team_id=team_user.team_id and team_user.uid=?", userID). + Cols("dataset.id").Find(&datasets) + return datasets } diff --git a/routers/home.go b/routers/home.go index ae184a72a..a147a84ea 100755 --- a/routers/home.go +++ b/routers/home.go @@ -346,7 +346,9 @@ func ExploreDatasets(ctx *context.Context) { var datasetsIds []int64 if ownerID > 0 { - datasetsIds = models.GetCollaboratorDatasetIdsByUserID(ownerID) + collaboratorDatasetsIds := models.GetCollaboratorDatasetIdsByUserID(ownerID) + teamDatasetsIds := models.GetTeamDatasetIdsByUserID(ownerID) + datasetsIds = append(collaboratorDatasetsIds, teamDatasetsIds...) } opts := &models.SearchDatasetOptions{ From afa22e930c255683943b25dfd56cce5b4fad25d6 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 17 Jun 2022 17:33:25 +0800 Subject: [PATCH 121/143] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- public/home/home.js | 42 +++--------------------------------------- routers/home.go | 26 ++------------------------ 2 files changed, 5 insertions(+), 63 deletions(-) diff --git a/public/home/home.js b/public/home/home.js index 9754de0cb..7343dba0b 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -217,9 +217,9 @@ function refresh3DInfo(record){ //console.log("cloudbrain two line length=" + lines.length); var span = $('.rotation3D__line').find("span")[1]; //console.log(span); - span.innerText =record.RefName; - //$('.rotation3D__line').find("span").eq(1).text(record.RefName) - //lines[1].find("span").text(record.RefName); + if(span != null){ + span.innerText =record.RefName; + } } } @@ -452,48 +452,12 @@ function queryRecommendData(){ displayOrg(json.org); displayRepo(json.repo); displayActivity(json.image); - displayCloudBrain(json.cloudbrain) }, error:function(response) { } }); - - // $.ajax({ - // type:"GET", - // url:"/recommend/repo", - // headers: { - // authorization:token, - // }, - // dataType:"json", - // async:false, - // success:function(json){ - // displayRepo(json); - // }, - // error:function(response) { - // } - // }); - - // $.ajax({ - // type:"GET", - // url:"/recommend/imageinfo", - // headers: { - // authorization:token, - // }, - // dataType:"json", - // async:false, - // success:function(json){ - // displayActivity(json); - // }, - // error:function(response) { - // } - // }); } -function displayCloudBrain(json){ - $('#completed_task').text(json.completed_task); - $('#running_task').text(json.running_task); - $('#wait_task').text(json.wait_task); -} function displayActivity(json){ var activityDiv = document.getElementById("recommendactivity"); diff --git a/routers/home.go b/routers/home.go index ae184a72a..0746fc486 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,7 +7,6 @@ package routers import ( "bytes" - "fmt" "net/http" "strconv" "strings" @@ -758,15 +757,6 @@ func GetRankUser(index string) ([]map[string]interface{}, error) { return resultOrg, nil } -// func GetImageInfoFromPromote(ctx *context.Context) { -// imageInfo, err := GetImageInfo() -// if err != nil { -// ctx.ServerError("500", err) -// return -// } -// ctx.JSON(200, imageInfo) -// } - func GetUserRankFromPromote(ctx *context.Context) { index := ctx.Params("index") resultUserRank, err := GetRankUser(index) @@ -790,27 +780,15 @@ func RecommendHomeInfo(ctx *context.Context) { if err != nil { log.Info("error." + err.Error()) } - resultCloudBrain, err := getCloudbrainNums() - if err != nil { - log.Info("error." + err.Error()) - } + mapInterface := make(map[string]interface{}) mapInterface["org"] = resultOrg mapInterface["repo"] = resultRepo mapInterface["image"] = resultImage - mapInterface["cloudbrain"] = resultCloudBrain + //mapInterface["cloudbrain"] = resultCloudBrain ctx.JSON(http.StatusOK, mapInterface) } -func getCloudbrainNums() (map[string]string, error) { - result := make(map[string]string) - cloudStatusMap := models.GetAllStatusCloudBrain() - result["completed_task"] = fmt.Sprint(cloudStatusMap["COMPLETED"]) - result["running_task"] = fmt.Sprint(cloudStatusMap["RUNNING"]) - result["wait_task"] = fmt.Sprint(cloudStatusMap["WAITING"]) - return result, nil -} - // func RecommendOrgFromPromote(ctx *context.Context) { // resultOrg, err := GetRecommendOrg() // if err != nil { From 0cd2223e532747e0f18f37ca44eaeaf0e40d3d5f Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 17 Jun 2022 17:39:19 +0800 Subject: [PATCH 122/143] =?UTF-8?q?=E4=BA=91=E8=84=91=E6=A6=9C=E5=8D=95?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain_static.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 27cccf415..03cd7d2bc 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -213,7 +213,7 @@ func GetWaittingTop() ([]*CloudbrainInfo, error) { cond = cond.And( builder.Eq{"cloudbrain.status": string(JobWaiting)}, ) - sess.OrderBy("(cloudbrain.start_time-cloudbrain.created_unix) DESC limit 10") + sess.OrderBy("cloudbrain.created_unix ASC limit 10") cloudbrains := make([]*CloudbrainInfo, 0, 10) if err := sess.Table(&Cloudbrain{}).Where(cond). Find(&cloudbrains); err != nil { @@ -228,7 +228,7 @@ func GetRunningTop() ([]*CloudbrainInfo, error) { cond = cond.And( builder.Eq{"cloudbrain.status": string(JobRunning)}, ) - sess.OrderBy("(cloudbrain.end_time-cloudbrain.start_time) DESC limit 10") + sess.OrderBy("cloudbrain.duration DESC limit 10") cloudbrains := make([]*CloudbrainInfo, 0, 10) if err := sess.Table(&Cloudbrain{}).Where(cond). Find(&cloudbrains); err != nil { From 09f66aa74366e8342afb9547eca33ed66288ece4 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 17 Jun 2022 17:42:54 +0800 Subject: [PATCH 123/143] =?UTF-8?q?=E4=BA=91=E8=84=91=E7=9C=8B=E6=9D=BF?= =?UTF-8?q?=E6=A6=82=E8=A7=88=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/api/v1/repo/cloudbrain_dashboard.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 07298ffdc..2e1c979b7 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -121,6 +121,13 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { } } + cloudBrainTypeList := []int{0, 1, 2} + for _, v := range cloudBrainTypeList { + if _, ok := cloudBrainNum[v]; !ok { + cloudBrainNum[v] = 0 + } + } + todayRunningCount := todayStatusResult[string(models.JobRunning)] todayCompletedCount := todayStatusResult[string(models.ModelArtsTrainJobCompleted)] + todayStatusResult[string(models.JobFailed)] + todayStatusResult[string(models.ModelArtsStartFailed)] + todayStatusResult[string(models.JobStopped)] + todayStatusResult[string(models.JobSucceeded)] + todayStatusResult[string(models.ModelArtsTrainJobKilled)] From 2d8f9d3ba5a48dc07d361c3fe91a150ca9947529 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 17 Jun 2022 17:49:35 +0800 Subject: [PATCH 124/143] =?UTF-8?q?=E6=8F=90=E9=AB=98=E9=A6=96=E9=A1=B5?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E9=80=9F=E5=BA=A6=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user.go | 14 +++++++++++++- routers/home.go | 27 +++++++-------------------- services/repository/repository.go | 5 ----- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/models/user.go b/models/user.go index 7d4c8ce34..dd5a6f1d2 100755 --- a/models/user.go +++ b/models/user.go @@ -6,7 +6,6 @@ package models import ( - "code.gitea.io/gitea/modules/blockchain" "container/list" "context" "crypto/md5" @@ -25,6 +24,8 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/modules/blockchain" + "code.gitea.io/gitea/modules/avatar" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/generate" @@ -1498,6 +1499,17 @@ func GetUsersByIDs(ids []int64) (UserList, error) { return ous, err } +func GetUsersByNames(names []string) (UserList, error) { + ous := make([]*User, 0, len(names)) + if len(names) == 0 { + return ous, nil + } + err := x.In("name", names). + Asc("name"). + Find(&ous) + return ous, err +} + // GetUserIDsByNames returns a slice of ids corresponds to names. func GetUserIDsByNames(names []string, ignoreNonExistent bool) ([]int64, error) { ids := make([]int64, 0, len(names)) diff --git a/routers/home.go b/routers/home.go index 0746fc486..5f62c97be 100755 --- a/routers/home.go +++ b/routers/home.go @@ -672,9 +672,14 @@ func getRecommendOrg() ([]map[string]interface{}, error) { if err != nil { return nil, err } - resultOrg := make([]map[string]interface{}, 0) + names := make([]string, 0) for _, userName := range result { - user, err := models.GetUserByName(userName) + names = append(names, userName) + } + users, _ := models.GetUsersByNames(names) + resultOrg := make([]map[string]interface{}, 0) + for i, _ := range result { + user := users[i] if err == nil { userMap := make(map[string]interface{}) userMap["Name"] = user.Name @@ -789,24 +794,6 @@ func RecommendHomeInfo(ctx *context.Context) { ctx.JSON(http.StatusOK, mapInterface) } -// func RecommendOrgFromPromote(ctx *context.Context) { -// resultOrg, err := GetRecommendOrg() -// if err != nil { -// ctx.ServerError("500", err) -// return -// } -// ctx.JSON(200, resultOrg) -// } - -func RecommendRepoFromPromote(ctx *context.Context) { - result, err := repository.GetRecommendRepoFromPromote("projects") - if err != nil { - ctx.ServerError("500", err) - } else { - ctx.JSON(200, result) - } -} - func HomeTerm(ctx *context.Context) { ctx.HTML(200, tplHomeTerm) } diff --git a/services/repository/repository.go b/services/repository/repository.go index 80518b666..6bf4ab283 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -131,11 +131,6 @@ func GetRecommendRepoFromPromote(filename string) ([]map[string]interface{}, err repoMap["ID"] = fmt.Sprint(repo.ID) repoMap["Name"] = repo.Name repoMap["Alias"] = repo.Alias - if repo.RepoType == models.RepoCourse { - //Load creator - repo.GetCreator() - repoMap["Creator"] = repo.Creator - } repoMap["OwnerName"] = repo.OwnerName repoMap["NumStars"] = repo.NumStars From d0124f4fa9997f243dffba9ebd0a7f3e41ccc7b7 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 17 Jun 2022 17:55:26 +0800 Subject: [PATCH 125/143] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelarts/trainjob/show.tmpl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index f22a2ce56..0facbc43b 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -597,8 +597,12 @@
- + +
+
+ + +
@@ -671,6 +675,14 @@ $('#JobName').val(obj.DisplayJobName).addClass('model_disabled') $('input[name="JobId"]').val(obj.JobID) $('input[name="VersionName"]').val(obj.VersionName).addClass('model_disabled') + if(obj.EngineID ==122){ + $('input[name="Engine_name"]').val("MindSpore").addClass('model_disabled'); + $('input[name="Engine"]').val(2); + } + if(obj.EngineID ==121){ + $('input[name="Engine_name"]').val("TensorFlow").addClass('model_disabled'); + $('input[name="Engine"]').val(1); + } $('.ui.dimmer').css({ "background-color": "rgb(136, 136, 136,0.7)" }) createModelName() }, From 0879e304debb0fab26390967a8fe4be2af523daa Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 20 Jun 2022 08:30:30 +0800 Subject: [PATCH 126/143] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=BC=B9=E7=99=BB?= =?UTF-8?q?=E5=BD=95=E6=A1=86=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- public/self/js/notebook/es5-shim.min.js | 1 - public/self/js/notebook/purify.min.js | 1 - 2 files changed, 2 deletions(-) diff --git a/public/self/js/notebook/es5-shim.min.js b/public/self/js/notebook/es5-shim.min.js index f301219d9..bd2883959 100644 --- a/public/self/js/notebook/es5-shim.min.js +++ b/public/self/js/notebook/es5-shim.min.js @@ -4,4 +4,3 @@ * see https://github.com/es-shims/es5-shim/blob/master/LICENSE */ (function(t,r){"use strict";if(typeof define==="function"&&define.amd){define(r)}else if(typeof exports==="object"){module.exports=r()}else{t.returnExports=r()}})(this,function(){var t=Array;var r=t.prototype;var e=Object;var n=e.prototype;var i=Function;var a=i.prototype;var o=String;var f=o.prototype;var u=Number;var l=u.prototype;var s=r.slice;var c=r.splice;var v=r.push;var h=r.unshift;var p=r.concat;var y=r.join;var d=a.call;var g=a.apply;var w=Math.max;var b=Math.min;var T=n.toString;var m=typeof Symbol==="function"&&typeof Symbol.toStringTag==="symbol";var D;var S=Function.prototype.toString,x=/^\s*class /,O=function isES6ClassFn(t){try{var r=S.call(t);var e=r.replace(/\/\/.*\n/g,"");var n=e.replace(/\/\*[.\s\S]*\*\//g,"");var i=n.replace(/\n/gm," ").replace(/ {2}/g," ");return x.test(i)}catch(a){return false}},E=function tryFunctionObject(t){try{if(O(t)){return false}S.call(t);return true}catch(r){return false}},j="[object Function]",I="[object GeneratorFunction]",D=function isCallable(t){if(!t){return false}if(typeof t!=="function"&&typeof t!=="object"){return false}if(m){return E(t)}if(O(t)){return false}var r=T.call(t);return r===j||r===I};var M;var U=RegExp.prototype.exec,$=function tryRegexExec(t){try{U.call(t);return true}catch(r){return false}},F="[object RegExp]";M=function isRegex(t){if(typeof t!=="object"){return false}return m?$(t):T.call(t)===F};var N;var C=String.prototype.valueOf,k=function tryStringObject(t){try{C.call(t);return true}catch(r){return false}},A="[object String]";N=function isString(t){if(typeof t==="string"){return true}if(typeof t!=="object"){return false}return m?k(t):T.call(t)===A};var R=e.defineProperty&&function(){try{var t={};e.defineProperty(t,"x",{enumerable:false,value:t});for(var r in t){return false}return t.x===t}catch(n){return false}}();var P=function(t){var r;if(R){r=function(t,r,n,i){if(!i&&r in t){return}e.defineProperty(t,r,{configurable:true,enumerable:false,writable:true,value:n})}}else{r=function(t,r,e,n){if(!n&&r in t){return}t[r]=e}}return function defineProperties(e,n,i){for(var a in n){if(t.call(n,a)){r(e,a,n[a],i)}}}}(n.hasOwnProperty);var J=function isPrimitive(t){var r=typeof t;return t===null||r!=="object"&&r!=="function"};var Y=u.isNaN||function isActualNaN(t){return t!==t};var z={ToInteger:function ToInteger(t){var r=+t;if(Y(r)){r=0}else if(r!==0&&r!==1/0&&r!==-(1/0)){r=(r>0||-1)*Math.floor(Math.abs(r))}return r},ToPrimitive:function ToPrimitive(t){var r,e,n;if(J(t)){return t}e=t.valueOf;if(D(e)){r=e.call(t);if(J(r)){return r}}n=t.toString;if(D(n)){r=n.call(t);if(J(r)){return r}}throw new TypeError},ToObject:function(t){if(t==null){throw new TypeError("can't convert "+t+" to object")}return e(t)},ToUint32:function ToUint32(t){return t>>>0}};var Z=function Empty(){};P(a,{bind:function bind(t){var r=this;if(!D(r)){throw new TypeError("Function.prototype.bind called on incompatible "+r)}var n=s.call(arguments,1);var a;var o=function(){if(this instanceof a){var i=g.call(r,this,p.call(n,s.call(arguments)));if(e(i)===i){return i}return this}else{return g.call(r,t,p.call(n,s.call(arguments)))}};var f=w(0,r.length-n.length);var u=[];for(var l=0;l0){r[e]=t[e]}return q(r,L(arguments,1))};B=function arraySliceApplyIE(t,r){return q(W(t),r)}}}var K=d.bind(f.slice);var Q=d.bind(f.split);var V=d.bind(f.indexOf);var _=d.bind(v);var tt=d.bind(n.propertyIsEnumerable);var rt=d.bind(r.sort);var et=t.isArray||function isArray(t){return H(t)==="[object Array]"};var nt=[].unshift(0)!==1;P(r,{unshift:function(){h.apply(this,arguments);return this.length}},nt);P(t,{isArray:et});var it=e("a");var at=it[0]!=="a"||!(0 in it);var ot=function properlyBoxed(t){var r=true;var e=true;var n=false;if(t){try{t.call("foo",function(t,e,n){if(typeof n!=="object"){r=false}});t.call([1],function(){"use strict";e=typeof this==="string"},"x")}catch(i){n=true}}return!!t&&!n&&r&&e};P(r,{forEach:function forEach(t){var r=z.ToObject(this);var e=at&&N(this)?Q(this,""):r;var n=-1;var i=z.ToUint32(e.length);var a;if(arguments.length>1){a=arguments[1]}if(!D(t)){throw new TypeError("Array.prototype.forEach callback must be a function")}while(++n1){o=arguments[1]}if(!D(r)){throw new TypeError("Array.prototype.map callback must be a function")}for(var f=0;f1){o=arguments[1]}if(!D(t)){throw new TypeError("Array.prototype.filter callback must be a function")}for(var f=0;f1){i=arguments[1]}if(!D(t)){throw new TypeError("Array.prototype.every callback must be a function")}for(var a=0;a1){i=arguments[1]}if(!D(t)){throw new TypeError("Array.prototype.some callback must be a function")}for(var a=0;a=2){a=arguments[1]}else{do{if(i in e){a=e[i++];break}if(++i>=n){throw new TypeError("reduce of empty array with no initial value")}}while(true)}for(;i=2){i=arguments[1]}else{do{if(a in e){i=e[a--];break}if(--a<0){throw new TypeError("reduceRight of empty array with no initial value")}}while(true)}if(a<0){return i}do{if(a in e){i=t(i,e[a],a,r)}}while(a--);return i}},!ut);var lt=r.indexOf&&[0,1].indexOf(1,2)!==-1;P(r,{indexOf:function indexOf(t){var r=at&&N(this)?Q(this,""):z.ToObject(this);var e=z.ToUint32(r.length);if(e===0){return-1}var n=0;if(arguments.length>1){n=z.ToInteger(arguments[1])}n=n>=0?n:w(0,e+n);for(;n1){n=b(n,z.ToInteger(arguments[1]))}n=n>=0?n:e-Math.abs(n);for(;n>=0;n--){if(n in r&&t===r[n]){return n}}return-1}},st);var ct=function(){var t=[1,2];var r=t.splice();return t.length===2&&et(r)&&r.length===0}();P(r,{splice:function splice(t,r){if(arguments.length===0){return[]}else{return c.apply(this,arguments)}}},!ct);var vt=function(){var t={};r.splice.call(t,0,0,1);return t.length===1}();P(r,{splice:function splice(t,r){if(arguments.length===0){return[]}var e=arguments;this.length=w(z.ToInteger(this.length),0);if(arguments.length>0&&typeof r!=="number"){e=W(arguments);if(e.length<2){_(e,this.length-t)}else{e[1]=z.ToInteger(r)}}return c.apply(this,e)}},!vt);var ht=function(){var r=new t(1e5);r[8]="x";r.splice(1,1);return r.indexOf("x")===7}();var pt=function(){var t=256;var r=[];r[t]="a";r.splice(t+1,0,"b");return r[t]==="a"}();P(r,{splice:function splice(t,r){var e=z.ToObject(this);var n=[];var i=z.ToUint32(e.length);var a=z.ToInteger(t);var f=a<0?w(i+a,0):b(a,i);var u=arguments.length===0?0:arguments.length===1?i-f:b(w(z.ToInteger(r),0),i-f);var l=0;var s;while(ly){delete e[l-1];l-=1}}else if(v>u){l=i-u;while(l>f){s=o(l+u-1);h=o(l+v-1);if(G(e,s)){e[h]=e[s]}else{delete e[h]}l-=1}}l=f;for(var d=0;d=0&&!et(t)&&D(t.callee)};var kt=Nt(arguments)?Nt:Ct;P(e,{keys:function keys(t){var r=D(t);var e=kt(t);var n=t!==null&&typeof t==="object";var i=n&&N(t);if(!n&&!r&&!e){throw new TypeError("Object.keys called on a non-object")}var a=[];var f=Ot&&r;if(i&&Et||e){for(var u=0;u11){return t+1}return t},getMonth:function getMonth(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=Bt(this);var r=Xt(this);if(t<0&&r>11){return 0}return r},getDate:function getDate(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=Bt(this);var r=Xt(this);var e=Lt(this);if(t<0&&r>11){if(r===12){return e}var n=ar(0,t+1);return n-e+1}return e},getUTCFullYear:function getUTCFullYear(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=qt(this);if(t<0&&Kt(this)>11){return t+1}return t},getUTCMonth:function getUTCMonth(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=qt(this);var r=Kt(this);if(t<0&&r>11){return 0}return r},getUTCDate:function getUTCDate(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=qt(this);var r=Kt(this);var e=Qt(this);if(t<0&&r>11){if(r===12){return e}var n=ar(0,t+1);return n-e+1}return e}},Jt);P(Date.prototype,{toUTCString:function toUTCString(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=Vt(this);var r=Qt(this);var e=Kt(this);var n=qt(this);var i=_t(this);var a=tr(this);var o=rr(this);return nr[t]+", "+(r<10?"0"+r:r)+" "+ir[e]+" "+n+" "+(i<10?"0"+i:i)+":"+(a<10?"0"+a:a)+":"+(o<10?"0"+o:o)+" GMT"}},Jt||Zt);P(Date.prototype,{toDateString:function toDateString(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=this.getDay();var r=this.getDate();var e=this.getMonth();var n=this.getFullYear();return nr[t]+" "+ir[e]+" "+(r<10?"0"+r:r)+" "+n}},Jt||Gt);if(Jt||Ht){Date.prototype.toString=function toString(){if(!this||!(this instanceof Date)){throw new TypeError("this is not a Date object.")}var t=this.getDay();var r=this.getDate();var e=this.getMonth();var n=this.getFullYear();var i=this.getHours();var a=this.getMinutes();var o=this.getSeconds();var f=this.getTimezoneOffset();var u=Math.floor(Math.abs(f)/60);var l=Math.floor(Math.abs(f)%60);return nr[t]+" "+ir[e]+" "+(r<10?"0"+r:r)+" "+n+" "+(i<10?"0"+i:i)+":"+(a<10?"0"+a:a)+":"+(o<10?"0"+o:o)+" GMT"+(f>0?"-":"+")+(u<10?"0"+u:u)+(l<10?"0"+l:l)};if(R){e.defineProperty(Date.prototype,"toString",{configurable:true,enumerable:false,writable:true})}}var or=-621987552e5;var fr="-000001";var ur=Date.prototype.toISOString&&new Date(or).toISOString().indexOf(fr)===-1;var lr=Date.prototype.toISOString&&new Date(-1).toISOString()!=="1969-12-31T23:59:59.999Z";var sr=d.bind(Date.prototype.getTime);P(Date.prototype,{toISOString:function toISOString(){if(!isFinite(this)||!isFinite(sr(this))){throw new RangeError("Date.prototype.toISOString called on non-finite value.")}var t=qt(this);var r=Kt(this);t+=Math.floor(r/12);r=(r%12+12)%12;var e=[r+1,Qt(this),_t(this),tr(this),rr(this)];t=(t<0?"-":t>9999?"+":"")+K("00000"+Math.abs(t),0<=t&&t<=9999?-4:-6);for(var n=0;n=7&&l>yr){var p=Math.floor(l/yr)*yr;var y=Math.floor(p/1e3);v+=y;h-=y*1e3}c=s===1&&o(e)===e?new t(r.parse(e)):s>=7?new t(e,n,i,a,f,v,h):s>=6?new t(e,n,i,a,f,v):s>=5?new t(e,n,i,a,f):s>=4?new t(e,n,i,a):s>=3?new t(e,n,i):s>=2?new t(e,n):s>=1?new t(e instanceof t?+e:e):new t}else{c=t.apply(this,arguments)}if(!J(c)){P(c,{constructor:r},true)}return c};var e=new RegExp("^"+"(\\d{4}|[+-]\\d{6})"+"(?:-(\\d{2})"+"(?:-(\\d{2})"+"(?:"+"T(\\d{2})"+":(\\d{2})"+"(?:"+":(\\d{2})"+"(?:(\\.\\d{1,}))?"+")?"+"("+"Z|"+"(?:"+"([-+])"+"(\\d{2})"+":(\\d{2})"+")"+")?)?)?)?"+"$");var n=[0,31,59,90,120,151,181,212,243,273,304,334,365];var i=function dayFromMonth(t,r){var e=r>1?1:0;return n[r]+Math.floor((t-1969+e)/4)-Math.floor((t-1901+e)/100)+Math.floor((t-1601+e)/400)+365*(t-1970)};var a=function toUTC(r){var e=0;var n=r;if(dr&&n>yr){var i=Math.floor(n/yr)*yr;var a=Math.floor(i/1e3);e+=a;n-=a*1e3}return u(new t(1970,0,1,0,0,e,n))};for(var f in t){if(G(t,f)){r[f]=t[f]}}P(r,{now:t.now,UTC:t.UTC},true);r.prototype=t.prototype;P(r.prototype,{constructor:r},true);var l=function parse(r){var n=e.exec(r);if(n){var o=u(n[1]),f=u(n[2]||1)-1,l=u(n[3]||1)-1,s=u(n[4]||0),c=u(n[5]||0),v=u(n[6]||0),h=Math.floor(u(n[7]||0)*1e3),p=Boolean(n[4]&&!n[8]),y=n[9]==="-"?1:-1,d=u(n[10]||0),g=u(n[11]||0),w;var b=c>0||v>0||h>0;if(s<(b?24:25)&&c<60&&v<60&&h<1e3&&f>-1&&f<12&&d<24&&g<60&&l>-1&&l=0){e+=wr.data[r];wr.data[r]=Math.floor(e/t);e=e%t*wr.base}},numToString:function numToString(){var t=wr.size;var r="";while(--t>=0){if(r!==""||t===0||wr.data[t]!==0){var e=o(wr.data[t]);if(r===""){r=e}else{r+=K("0000000",0,7-e.length)+e}}}return r},pow:function pow(t,r,e){return r===0?e:r%2===1?pow(t,r-1,e*t):pow(t*t,r/2,e)},log:function log(t){var r=0;var e=t;while(e>=4096){r+=12;e/=4096}while(e>=2){r+=1;e/=2}return r}};var br=function toFixed(t){var r,e,n,i,a,f,l,s;r=u(t);r=Y(r)?0:Math.floor(r);if(r<0||r>20){throw new RangeError("Number.toFixed called with invalid number of decimals")}e=u(this);if(Y(e)){return"NaN"}if(e<=-1e21||e>=1e21){return o(e)}n="";if(e<0){n="-";e=-e}i="0";if(e>1e-21){a=wr.log(e*wr.pow(2,69,1))-69;f=a<0?e*wr.pow(2,-a,1):e/wr.pow(2,a,1);f*=4503599627370496;a=52-a;if(a>0){wr.multiply(0,f);l=r;while(l>=7){wr.multiply(1e7,0);l-=7}wr.multiply(wr.pow(10,l,1),0);l=a-1;while(l>=23){wr.divide(1<<23);l-=23}wr.divide(1<0){s=i.length;if(s<=r){i=n+K("0.0000000000000000000",0,r-s+2)+i}else{i=n+K(i,0,s-r)+"."+K(i,s-r)}}else{i=n+i}return i};P(l,{toFixed:br},gr);var Tr=function(){try{return 1..toPrecision(undefined)==="1"}catch(t){return true}}();var mr=l.toPrecision;P(l,{toPrecision:function toPrecision(t){return typeof t==="undefined"?mr.call(this):mr.call(this,t)}},Tr);if("ab".split(/(?:ab)*/).length!==2||".".split(/(.?)(.?)/).length!==4||"tesst".split(/(s)*/)[1]==="t"||"test".split(/(?:)/,-1).length!==4||"".split(/.?/).length||".".split(/()()/).length>1){(function(){var t=typeof/()??/.exec("")[1]==="undefined";var r=Math.pow(2,32)-1;f.split=function(e,n){var i=String(this);if(typeof e==="undefined"&&n===0){return[]}if(!M(e)){return Q(this,e,n)}var a=[];var o=(e.ignoreCase?"i":"")+(e.multiline?"m":"")+(e.unicode?"u":"")+(e.sticky?"y":""),f=0,u,l,s,c;var h=new RegExp(e.source,o+"g");if(!t){u=new RegExp("^"+h.source+"$(?!\\s)",o)}var p=typeof n==="undefined"?r:z.ToUint32(n);l=h.exec(i);while(l){s=l.index+l[0].length;if(s>f){_(a,K(i,f,l.index));if(!t&&l.length>1){l[0].replace(u,function(){for(var t=1;t1&&l.index=p){break}}if(h.lastIndex===l.index){h.lastIndex++}l=h.exec(i)}if(f===i.length){if(c||!h.test("")){_(a,"")}}else{_(a,K(i,f))}return a.length>p?W(a,0,p):a}})()}else if("0".split(void 0,0).length){f.split=function split(t,r){if(typeof t==="undefined"&&r===0){return[]}return Q(this,t,r)}}var Dr=f.replace;var Sr=function(){var t=[];"x".replace(/x(.)?/g,function(r,e){_(t,e)});return t.length===1&&typeof t[0]==="undefined"}();if(!Sr){f.replace=function replace(t,r){var e=D(r);var n=M(t)&&/\)[*?]/.test(t.source);if(!e||!n){return Dr.call(this,t,r)}else{var i=function(e){var n=arguments.length;var i=t.lastIndex;t.lastIndex=0;var a=t.exec(e)||[];t.lastIndex=i;_(a,arguments[n-2],arguments[n-1]);return r.apply(this,a)};return Dr.call(this,t,i)}}}var xr=f.substr;var Or="".substr&&"0b".substr(-1)!=="b";P(f,{substr:function substr(t,r){var e=t;if(t<0){e=w(this.length+t,0)}return xr.call(this,e,r)}},Or);var Er="\t\n\x0B\f\r \xa0\u1680\u180e\u2000\u2001\u2002\u2003"+"\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\u2028"+"\u2029\ufeff";var jr="\u200b";var Ir="["+Er+"]";var Mr=new RegExp("^"+Ir+Ir+"*");var Ur=new RegExp(Ir+Ir+"*$");var $r=f.trim&&(Er.trim()||!jr.trim());P(f,{trim:function trim(){if(typeof this==="undefined"||this===null){throw new TypeError("can't convert "+this+" to object")}return o(this).replace(Mr,"").replace(Ur,"")}},$r);var Fr=d.bind(String.prototype.trim);var Nr=f.lastIndexOf&&"abc\u3042\u3044".lastIndexOf("\u3042\u3044",2)!==-1;P(f,{lastIndexOf:function lastIndexOf(t){if(typeof this==="undefined"||this===null){throw new TypeError("can't convert "+this+" to object")}var r=o(this);var e=o(t);var n=arguments.length>1?u(arguments[1]):NaN;var i=Y(n)?Infinity:z.ToInteger(n);var a=b(w(i,0),r.length);var f=e.length;var l=a+f;while(l>0){l=w(0,l-f);var s=V(K(r,l,a+f),e);if(s!==-1){return l+s}}return-1}},Nr);var Cr=f.lastIndexOf;P(f,{lastIndexOf:function lastIndexOf(t){return Cr.apply(this,arguments)}},f.lastIndexOf.length!==1);if(parseInt(Er+"08")!==8||parseInt(Er+"0x16")!==22){parseInt=function(t){var r=/^[-+]?0[xX]/;return function parseInt(e,n){if(typeof e==="symbol"){""+e}var i=Fr(String(e));var a=u(n)||(r.test(i)?16:10);return t(i,a)}}(parseInt)}if(1/parseFloat("-0")!==-Infinity){parseFloat=function(t){return function parseFloat(r){var e=Fr(String(r));var n=t(e);return n===0&&K(e,0,1)==="-"?-0:n}}(parseFloat)}if(String(new RangeError("test"))!=="RangeError: test"){var kr=function toString(){if(typeof this==="undefined"||this===null){throw new TypeError("can't convert "+this+" to object")}var t=this.name;if(typeof t==="undefined"){t="Error"}else if(typeof t!=="string"){t=o(t)}var r=this.message;if(typeof r==="undefined"){r=""}else if(typeof r!=="string"){r=o(r)}if(!t){return r}if(!r){return t}return t+": "+r};Error.prototype.toString=kr}if(R){var Ar=function(t,r){if(tt(t,r)){var e=Object.getOwnPropertyDescriptor(t,r);if(e.configurable){e.enumerable=false;Object.defineProperty(t,r,e)}}};Ar(Error.prototype,"message");if(Error.prototype.message!==""){Error.prototype.message=""}Ar(Error.prototype,"name")}if(String(/a/gim)!=="/a/gim"){var Rr=function toString(){var t="/"+this.source+"/";if(this.global){t+="g"}if(this.ignoreCase){t+="i"}if(this.multiline){t+="m"}return t};RegExp.prototype.toString=Rr}}); -//# sourceMappingURL=es5-shim.map diff --git a/public/self/js/notebook/purify.min.js b/public/self/js/notebook/purify.min.js index 8c121b491..30d29f6c2 100644 --- a/public/self/js/notebook/purify.min.js +++ b/public/self/js/notebook/purify.min.js @@ -1,3 +1,2 @@ /*! @license DOMPurify | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/2.2.2/LICENSE */ !function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):(e=e||self).DOMPurify=t()}(this,(function(){"use strict";var e=Object.hasOwnProperty,t=Object.setPrototypeOf,n=Object.isFrozen,r=Object.getPrototypeOf,o=Object.getOwnPropertyDescriptor,i=Object.freeze,a=Object.seal,l=Object.create,c="undefined"!=typeof Reflect&&Reflect,s=c.apply,u=c.construct;s||(s=function(e,t,n){return e.apply(t,n)}),i||(i=function(e){return e}),a||(a=function(e){return e}),u||(u=function(e,t){return new(Function.prototype.bind.apply(e,[null].concat(function(e){if(Array.isArray(e)){for(var t=0,n=Array(e.length);t1?n-1:0),o=1;o/gm),U=a(/^data-[\-\w.\u00B7-\uFFFF]/),j=a(/^aria-[\-\w]+$/),P=a(/^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i),B=a(/^(?:\w+script|data):/i),W=a(/[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205F\u3000]/g),G="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e};function q(e){if(Array.isArray(e)){for(var t=0,n=Array(e.length);t0&&void 0!==arguments[0]?arguments[0]:K(),n=function(t){return e(t)};if(n.version="2.2.6",n.removed=[],!t||!t.document||9!==t.document.nodeType)return n.isSupported=!1,n;var r=t.document,o=t.document,a=t.DocumentFragment,l=t.HTMLTemplateElement,c=t.Node,s=t.Element,u=t.NodeFilter,f=t.NamedNodeMap,x=void 0===f?t.NamedNodeMap||t.MozNamedAttrMap:f,Y=t.Text,X=t.Comment,$=t.DOMParser,Z=t.trustedTypes,J=s.prototype,Q=k(J,"cloneNode"),ee=k(J,"nextSibling"),te=k(J,"childNodes"),ne=k(J,"parentNode");if("function"==typeof l){var re=o.createElement("template");re.content&&re.content.ownerDocument&&(o=re.content.ownerDocument)}var oe=V(Z,r),ie=oe&&ze?oe.createHTML(""):"",ae=o,le=ae.implementation,ce=ae.createNodeIterator,se=ae.getElementsByTagName,ue=ae.createDocumentFragment,fe=r.importNode,me={};try{me=S(o).documentMode?o.documentMode:{}}catch(e){}var de={};n.isSupported=le&&void 0!==le.createHTMLDocument&&9!==me;var pe=z,ge=H,he=U,ye=j,ve=B,be=W,Te=P,Ae=null,xe=w({},[].concat(q(R),q(_),q(D),q(N),q(L))),we=null,Se=w({},[].concat(q(M),q(F),q(C),q(I))),ke=null,Re=null,_e=!0,De=!0,Ee=!1,Ne=!1,Oe=!1,Le=!1,Me=!1,Fe=!1,Ce=!1,Ie=!0,ze=!1,He=!0,Ue=!0,je=!1,Pe={},Be=w({},["annotation-xml","audio","colgroup","desc","foreignobject","head","iframe","math","mi","mn","mo","ms","mtext","noembed","noframes","noscript","plaintext","script","style","svg","template","thead","title","video","xmp"]),We=null,Ge=w({},["audio","video","img","source","image","track"]),qe=null,Ke=w({},["alt","class","for","id","label","name","pattern","placeholder","summary","title","value","style","xmlns"]),Ve=null,Ye=o.createElement("form"),Xe=function(e){Ve&&Ve===e||(e&&"object"===(void 0===e?"undefined":G(e))||(e={}),e=S(e),Ae="ALLOWED_TAGS"in e?w({},e.ALLOWED_TAGS):xe,we="ALLOWED_ATTR"in e?w({},e.ALLOWED_ATTR):Se,qe="ADD_URI_SAFE_ATTR"in e?w(S(Ke),e.ADD_URI_SAFE_ATTR):Ke,We="ADD_DATA_URI_TAGS"in e?w(S(Ge),e.ADD_DATA_URI_TAGS):Ge,ke="FORBID_TAGS"in e?w({},e.FORBID_TAGS):{},Re="FORBID_ATTR"in e?w({},e.FORBID_ATTR):{},Pe="USE_PROFILES"in e&&e.USE_PROFILES,_e=!1!==e.ALLOW_ARIA_ATTR,De=!1!==e.ALLOW_DATA_ATTR,Ee=e.ALLOW_UNKNOWN_PROTOCOLS||!1,Ne=e.SAFE_FOR_TEMPLATES||!1,Oe=e.WHOLE_DOCUMENT||!1,Fe=e.RETURN_DOM||!1,Ce=e.RETURN_DOM_FRAGMENT||!1,Ie=!1!==e.RETURN_DOM_IMPORT,ze=e.RETURN_TRUSTED_TYPE||!1,Me=e.FORCE_BODY||!1,He=!1!==e.SANITIZE_DOM,Ue=!1!==e.KEEP_CONTENT,je=e.IN_PLACE||!1,Te=e.ALLOWED_URI_REGEXP||Te,Ne&&(De=!1),Ce&&(Fe=!0),Pe&&(Ae=w({},[].concat(q(L))),we=[],!0===Pe.html&&(w(Ae,R),w(we,M)),!0===Pe.svg&&(w(Ae,_),w(we,F),w(we,I)),!0===Pe.svgFilters&&(w(Ae,D),w(we,F),w(we,I)),!0===Pe.mathMl&&(w(Ae,N),w(we,C),w(we,I))),e.ADD_TAGS&&(Ae===xe&&(Ae=S(Ae)),w(Ae,e.ADD_TAGS)),e.ADD_ATTR&&(we===Se&&(we=S(we)),w(we,e.ADD_ATTR)),e.ADD_URI_SAFE_ATTR&&w(qe,e.ADD_URI_SAFE_ATTR),Ue&&(Ae["#text"]=!0),Oe&&w(Ae,["html","head","body"]),Ae.table&&(w(Ae,["tbody"]),delete ke.tbody),i&&i(e),Ve=e)},$e=w({},["mi","mo","mn","ms","mtext"]),Ze=w({},["foreignobject","desc","title","annotation-xml"]),Je=w({},_);w(Je,D),w(Je,E);var Qe=w({},N);w(Qe,O);var et="http://www.w3.org/1998/Math/MathML",tt="http://www.w3.org/2000/svg",nt="http://www.w3.org/1999/xhtml",rt=function(e){var t=ne(e);t&&t.tagName||(t={namespaceURI:nt,tagName:"template"});var n=g(e.tagName),r=g(t.tagName);if(e.namespaceURI===tt)return t.namespaceURI===nt?"svg"===n:t.namespaceURI===et?"svg"===n&&("annotation-xml"===r||$e[r]):Boolean(Je[n]);if(e.namespaceURI===et)return t.namespaceURI===nt?"math"===n:t.namespaceURI===tt?"math"===n&&Ze[r]:Boolean(Qe[n]);if(e.namespaceURI===nt){if(t.namespaceURI===tt&&!Ze[r])return!1;if(t.namespaceURI===et&&!$e[r])return!1;var o=w({},["title","style","font","a","script"]);return!Qe[n]&&(o[n]||!Je[n])}return!1},ot=function(e){p(n.removed,{element:e});try{e.parentNode.removeChild(e)}catch(t){try{e.outerHTML=ie}catch(t){e.remove()}}},it=function(e,t){try{p(n.removed,{attribute:t.getAttributeNode(e),from:t})}catch(e){p(n.removed,{attribute:null,from:t})}t.removeAttribute(e)},at=function(e){var t=void 0,n=void 0;if(Me)e=""+e;else{var r=h(e,/^[\r\n\t ]+/);n=r&&r[0]}var i=oe?oe.createHTML(e):e;try{t=(new $).parseFromString(i,"text/html")}catch(e){}if(!t||!t.documentElement){var a=(t=le.createHTMLDocument("")).body;a.parentNode.removeChild(a.parentNode.firstElementChild),a.outerHTML=i}return e&&n&&t.body.insertBefore(o.createTextNode(n),t.body.childNodes[0]||null),se.call(t,Oe?"html":"body")[0]},lt=function(e){return ce.call(e.ownerDocument||e,e,u.SHOW_ELEMENT|u.SHOW_COMMENT|u.SHOW_TEXT,(function(){return u.FILTER_ACCEPT}),!1)},ct=function(e){return!(e instanceof Y||e instanceof X)&&!("string"==typeof e.nodeName&&"string"==typeof e.textContent&&"function"==typeof e.removeChild&&e.attributes instanceof x&&"function"==typeof e.removeAttribute&&"function"==typeof e.setAttribute&&"string"==typeof e.namespaceURI&&"function"==typeof e.insertBefore)},st=function(e){return"object"===(void 0===c?"undefined":G(c))?e instanceof c:e&&"object"===(void 0===e?"undefined":G(e))&&"number"==typeof e.nodeType&&"string"==typeof e.nodeName},ut=function(e,t,r){de[e]&&m(de[e],(function(e){e.call(n,t,r,Ve)}))},ft=function(e){var t=void 0;if(ut("beforeSanitizeElements",e,null),ct(e))return ot(e),!0;if(h(e.nodeName,/[\u0080-\uFFFF]/))return ot(e),!0;var r=g(e.nodeName);if(ut("uponSanitizeElement",e,{tagName:r,allowedTags:Ae}),!st(e.firstElementChild)&&(!st(e.content)||!st(e.content.firstElementChild))&&T(/<[/\w]/g,e.innerHTML)&&T(/<[/\w]/g,e.textContent))return ot(e),!0;if(!Ae[r]||ke[r]){if(Ue&&!Be[r])for(var o=ne(e),i=te(e),a=i.length-1;a>=0;--a)o.insertBefore(Q(i[a],!0),ee(e));return ot(e),!0}return e instanceof s&&!rt(e)?(ot(e),!0):"noscript"!==r&&"noembed"!==r||!T(/<\/no(script|embed)/i,e.innerHTML)?(Ne&&3===e.nodeType&&(t=e.textContent,t=y(t,pe," "),t=y(t,ge," "),e.textContent!==t&&(p(n.removed,{element:e.cloneNode()}),e.textContent=t)),ut("afterSanitizeElements",e,null),!1):(ot(e),!0)},mt=function(e,t,n){if(He&&("id"===t||"name"===t)&&(n in o||n in Ye))return!1;if(De&&T(he,t));else if(_e&&T(ye,t));else{if(!we[t]||Re[t])return!1;if(qe[t]);else if(T(Te,y(n,be,"")));else if("src"!==t&&"xlink:href"!==t&&"href"!==t||"script"===e||0!==v(n,"data:")||!We[e]){if(Ee&&!T(ve,y(n,be,"")));else if(n)return!1}else;}return!0},dt=function(e){var t=void 0,r=void 0,o=void 0,i=void 0;ut("beforeSanitizeAttributes",e,null);var a=e.attributes;if(a){var l={attrName:"",attrValue:"",keepAttr:!0,allowedAttributes:we};for(i=a.length;i--;){var c=t=a[i],s=c.name,u=c.namespaceURI;if(r=b(t.value),o=g(s),l.attrName=o,l.attrValue=r,l.keepAttr=!0,l.forceKeepAttr=void 0,ut("uponSanitizeAttribute",e,l),r=l.attrValue,!l.forceKeepAttr&&(it(s,e),l.keepAttr))if(T(/\/>/i,r))it(s,e);else{Ne&&(r=y(r,pe," "),r=y(r,ge," "));var f=e.nodeName.toLowerCase();if(mt(f,o,r))try{u?e.setAttributeNS(u,s,r):e.setAttribute(s,r),d(n.removed)}catch(e){}}}ut("afterSanitizeAttributes",e,null)}},pt=function e(t){var n=void 0,r=lt(t);for(ut("beforeSanitizeShadowDOM",t,null);n=r.nextNode();)ut("uponSanitizeShadowNode",n,null),ft(n)||(n.content instanceof a&&e(n.content),dt(n));ut("afterSanitizeShadowDOM",t,null)};return n.sanitize=function(e,o){var i=void 0,l=void 0,s=void 0,u=void 0,f=void 0;if(e||(e="\x3c!--\x3e"),"string"!=typeof e&&!st(e)){if("function"!=typeof e.toString)throw A("toString is not a function");if("string"!=typeof(e=e.toString()))throw A("dirty is not a string, aborting")}if(!n.isSupported){if("object"===G(t.toStaticHTML)||"function"==typeof t.toStaticHTML){if("string"==typeof e)return t.toStaticHTML(e);if(st(e))return t.toStaticHTML(e.outerHTML)}return e}if(Le||Xe(o),n.removed=[],"string"==typeof e&&(je=!1),je);else if(e instanceof c)1===(l=(i=at("\x3c!----\x3e")).ownerDocument.importNode(e,!0)).nodeType&&"BODY"===l.nodeName||"HTML"===l.nodeName?i=l:i.appendChild(l);else{if(!Fe&&!Ne&&!Oe&&-1===e.indexOf("<"))return oe&&ze?oe.createHTML(e):e;if(!(i=at(e)))return Fe?null:ie}i&&Me&&ot(i.firstChild);for(var m=lt(je?e:i);s=m.nextNode();)3===s.nodeType&&s===u||ft(s)||(s.content instanceof a&&pt(s.content),dt(s),u=s);if(u=null,je)return e;if(Fe){if(Ce)for(f=ue.call(i.ownerDocument);i.firstChild;)f.appendChild(i.firstChild);else f=i;return Ie&&(f=fe.call(r,f,!0)),f}var d=Oe?i.outerHTML:i.innerHTML;return Ne&&(d=y(d,pe," "),d=y(d,ge," ")),oe&&ze?oe.createHTML(d):d},n.setConfig=function(e){Xe(e),Le=!0},n.clearConfig=function(){Ve=null,Le=!1},n.isValidAttribute=function(e,t,n){Ve||Xe({});var r=g(e),o=g(t);return mt(r,o,n)},n.addHook=function(e,t){"function"==typeof t&&(de[e]=de[e]||[],p(de[e],t))},n.removeHook=function(e){de[e]&&d(de[e])},n.removeHooks=function(e){de[e]&&(de[e]=[])},n.removeAllHooks=function(){de={}},n}()})); -//# sourceMappingURL=purify.min.js.map From 708fbe03e3e13d838a299cfd1598f3ec263b8fbb Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 20 Jun 2022 09:26:25 +0800 Subject: [PATCH 127/143] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/home.go | 10 +++++++--- templates/repo/modelarts/trainjob/show.tmpl | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/routers/home.go b/routers/home.go index 430d0c223..f0cb5a667 100755 --- a/routers/home.go +++ b/routers/home.go @@ -679,10 +679,14 @@ func getRecommendOrg() ([]map[string]interface{}, error) { names = append(names, userName) } users, _ := models.GetUsersByNames(names) + userMap := make(map[string]*models.User, 0) + for _, user := range users { + userMap[user.Name] = user + } resultOrg := make([]map[string]interface{}, 0) - for i, _ := range result { - user := users[i] - if err == nil { + for _, userName := range result { + user := userMap[userName] + if user != nil { userMap := make(map[string]interface{}) userMap["Name"] = user.Name userMap["Description"] = user.Description diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 0facbc43b..674e7e859 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -602,7 +602,7 @@
- +
@@ -704,6 +704,8 @@ type: 'POST', data: data, success: function (res) { + $('input[name="Engine_name"]').val(""); + $('input[name="Engine"]').val(""); location.href = `/${userName}/${repoPath}/modelmanage/show_model` $('.ui.modal.second').modal('hide') }, From d72f55a3c42122f407525e9188f857c2f74ff5db Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 20 Jun 2022 09:29:44 +0800 Subject: [PATCH 128/143] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/home.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/home.go b/routers/home.go index f0cb5a667..8829e26f4 100755 --- a/routers/home.go +++ b/routers/home.go @@ -699,7 +699,7 @@ func getRecommendOrg() ([]map[string]interface{}, error) { userMap["NumMembers"] = user.NumMembers resultOrg = append(resultOrg, userMap) } else { - log.Info("query user error," + err.Error()) + log.Info("the user not exist," + userName) } } return resultOrg, nil From 3988538ea4d4cd79b2e55e2d68e5fb16d9412f5a Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 20 Jun 2022 09:43:28 +0800 Subject: [PATCH 129/143] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/dataset.go | 9 --------- routers/home.go | 4 +--- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 53b63185f..54f89acf8 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -181,7 +181,6 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { if len(opts.DatasetIDs) > 0 { subCon := builder.NewCond() subCon = subCon.And(builder.In("dataset.id", opts.DatasetIDs)) - subCon = generateFilterCond(opts, subCon) cond = cond.Or(subCon) } @@ -462,11 +461,3 @@ func GetCollaboratorDatasetIdsByUserID(userID int64) []int64 { Cols("dataset.id").Find(&datasets) return datasets } - -func GetTeamDatasetIdsByUserID(userID int64) []int64 { - var datasets []int64 - _ = x.Table("dataset").Join("INNER", "team_repo", "dataset.repo_id = team_repo.repo_id"). - Join("INNER", "team_user", "team_repo.team_id=team_user.team_id and team_user.uid=?", userID). - Cols("dataset.id").Find(&datasets) - return datasets -} diff --git a/routers/home.go b/routers/home.go index 8829e26f4..1ed5faaa8 100755 --- a/routers/home.go +++ b/routers/home.go @@ -345,9 +345,7 @@ func ExploreDatasets(ctx *context.Context) { var datasetsIds []int64 if ownerID > 0 { - collaboratorDatasetsIds := models.GetCollaboratorDatasetIdsByUserID(ownerID) - teamDatasetsIds := models.GetTeamDatasetIdsByUserID(ownerID) - datasetsIds = append(collaboratorDatasetsIds, teamDatasetsIds...) + datasetsIds = models.GetCollaboratorDatasetIdsByUserID(ownerID) } opts := &models.SearchDatasetOptions{ From fbbea0c3ecc629f5f7dfb75691c8b9a1534a3c95 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 20 Jun 2022 09:46:52 +0800 Subject: [PATCH 130/143] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/dataset.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/dataset.go b/models/dataset.go index 54f89acf8..b7186ac0b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -460,4 +460,5 @@ func GetCollaboratorDatasetIdsByUserID(userID int64) []int64 { _ = x.Table("dataset").Join("INNER", "collaboration", "dataset.repo_id = collaboration.repo_id and collaboration.mode>0 and collaboration.user_id=?", userID). Cols("dataset.id").Find(&datasets) return datasets + } From a983985d4cf8308a29fe154cbef54eba19d371a9 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 20 Jun 2022 10:51:48 +0800 Subject: [PATCH 131/143] =?UTF-8?q?=E5=BD=93=E6=B2=A1=E6=9C=89=E8=AE=AD?= =?UTF-8?q?=E7=BB=83=E4=BB=BB=E5=8A=A1=E6=97=B6=EF=BC=8C=E4=B8=8D=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E7=95=8C=E9=9D=A2=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 3a1781064..d951c8e7a 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -266,17 +266,18 @@ $.get(`${repolink}/modelmanage/query_train_job?repoId=${repoId}`, (data) => { const n_length = data.length - let train_html = '' - for (let i = 0; i < n_length; i++) { - train_html += `
${data[i].DisplayJobName}
` - train_html += '
' + if(n_length > 0){ + let train_html = '' + for (let i = 0; i < n_length; i++) { + train_html += `
${data[i].DisplayJobName}
` + train_html += '
' + } + $("#job-name").append(train_html) + $(".ui.dropdown.selection.search.width83").removeClass("loading") + $('#choice_model .default.text').text(data[0].DisplayJobName) + $('#choice_model input[name="JobId"]').val(data[0].JobID) + loadTrainVersion() } - $("#job-name").append(train_html) - $(".ui.dropdown.selection.search.width83").removeClass("loading") - $('#choice_model .default.text').text(data[0].DisplayJobName) - $('#choice_model input[name="JobId"]').val(data[0].JobID) - loadTrainVersion() - }) } function loadTrainVersion(value) { @@ -298,7 +299,6 @@ } $('#choice_version .default.text').text(versionName) $('#choice_version input[name="VersionName"]').val(versionName) - console.log("1111111111"); setEngine(data[0]) } From ac7f4ff7325f6ccb07fe562e79512ebea11466c3 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 20 Jun 2022 10:55:37 +0800 Subject: [PATCH 132/143] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=A4=B4=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/explore/repo_list.tmpl | 243 ++++++++++++++++++++++----------------- 1 file changed, 136 insertions(+), 107 deletions(-) diff --git a/templates/explore/repo_list.tmpl b/templates/explore/repo_list.tmpl index b6bb49da9..758fea14e 100755 --- a/templates/explore/repo_list.tmpl +++ b/templates/explore/repo_list.tmpl @@ -1,38 +1,46 @@
-
-
-
-
-
-
-
+
+
+
+
+
+
+
{{template "repo/header" .}} @@ -82,29 +87,41 @@
- - - + + + CPU/GPU - - - + + + Ascend NPU
- + {{.i18n.Tr "cloudbrain.job_name_rule"}}
- - + +
@@ -112,43 +129,45 @@
- - + +
- +
- {{range .engines}} - + {{end}}
- + {{range .engine_versions}} + + {{end}}
@@ -156,41 +175,49 @@
- - {{if .bootFile}} - - {{else}} - - {{end}} - - - - {{.i18n.Tr "cloudbrain.view_sample"}} + + {{if .bootFile}} + + {{else}} + + {{end}} + + + + {{.i18n.Tr "cloudbrain.view_sample"}}
- + {{template "custom/select_dataset_train" .}} - {{.i18n.Tr "cloudbrain.dataset_path_rule"}} + {{.i18n.Tr "cloudbrain.dataset_path_rule"}}
- {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} - + {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} +
- {{if ne 0 (len .params)}} + {{if ne 0 (len .params)}} {{range $k ,$v := .params}} -
-
- -
-
- -
- - - - +
+
+ +
+
+
+ + + + +
+ {{end}} {{end}} - {{end}}
@@ -199,13 +226,14 @@