package repo import ( "fmt" "net/http" "net/url" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "github.com/360EntSecGroup-Skylar/excelize/v2" "code.gitea.io/gitea/modules/setting" ) type CloudbrainsPeriodData struct { DebugOneCount int64 `json:"debugOneCount"` BenchmarkOneCount int64 `json:"benchmarkOneCount"` TrainOneCount int64 `json:"trainOneCount"` DebugTwoCount int64 `json:"debugTwoCount"` TrainTwoCount int64 `json:"trainTwoCount"` InferenceTwoCount int64 `json:"inferenceTwoCount"` JobWaitingCount int64 `json:"jobWaitingCount"` JobRunningCount int64 `json:"jobRunningCount"` JobSucceededCount int64 `json:"jobRunningCount"` } type TimeCloudbrainsNum struct { TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"` } type DateCloudbrainNum struct { Date string `json:"date"` DebugOneCount int64 `json:"debugOneCount"` BenchmarkOneCount int64 `json:"benchmarkOneCount"` TrainOneCount int64 `json:"trainOneCount"` DebugTwoCount int64 `json:"debugTwoCount"` TrainTwoCount int64 `json:"trainTwoCount"` InferenceTwoCount int64 `json:"inferenceTwoCount"` CloudbrainOneCount int64 `json:"cloudbrainOneCount"` CloudbrainTwoCount int64 `json:"cloudbrainTwoCount"` CloudbrainCount int64 `json:"cloudbrainCount"` } func GetAllCloudbrainsOverview(ctx *context.Context) { now := time.Now() //today overview beginTime := now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime := now jobWaitingCount, err := models.GetJobWaitingCount(beginTime, endTime) if err != nil { log.Error("Can not query jobWaitingCount.", err) } jobRunningCount, err := models.GetJobRunningCount(beginTime, endTime) if err != nil { log.Error("Can not query jobRunningCount.", err) } jobSucceededCount, err := models.GetJobSucceededCount(beginTime, endTime) if err != nil { log.Error("Can not query jobSucceededCount.", err) } cloudbrainsPeriodData := CloudbrainsPeriodData{ JobWaitingCount: jobWaitingCount, JobRunningCount: jobRunningCount, JobSucceededCount: jobSucceededCount, } ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } func GetAllCloudbrainsTrend(ctx *context.Context) { brainRecordBeginTime, err := getBrainRecordBeginTime() if err != nil { log.Error("Can not get brain record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.brain_record_begintime_get_err")) return } queryType := ctx.QueryTrim("type") now := time.Now() beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") var beginTime time.Time var endTime time.Time var endTimeTemp time.Time dateCloudbrainNum := make([]DateCloudbrainNum, 0) if queryType != "" { if queryType == "all" { beginTime = brainRecordBeginTime endTime = now endTimeTemp = beginTime.AddDate(0, 1, 0) dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getYearCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else if queryType == "yesterday" { beginTime = now.AddDate(0, 0, -1) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.Add(time.Hour) endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getHourCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else if queryType == "current_week" { beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) //begin from monday beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "current_month" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "monthly" { endTime = now beginTime = now.AddDate(0, -1, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "current_year" { endTime = now beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 1, 0) dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else if queryType == "last_month" { lastMonthTime := now.AddDate(0, -1, 0) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } } else { if beginTimeStr == "" || endTimeStr == "" { //如果查询类型和开始时间结束时间都未设置,按queryType=all处理 beginTime = brainRecordBeginTime endTime = now endTimeTemp = beginTime.AddDate(0, 1, 0) dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else { beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) return } endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) return } days := (endTime.Unix() - beginTime.Unix()) / 3600 / 24 if 1 < days { endTimeTemp = beginTime.AddDate(0, 0, 1) endTime = endTime.AddDate(0, 0, 2) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if 0 < days || days <= 1 { endTimeTemp = beginTime.Add(time.Hour) dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getHourCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getHourCloudbrainNum_get_error")) return } } else { return } } } cloudbrainsPeriodData := TimeCloudbrainsNum{ TimeCloudbrainNum: dateCloudbrainNum, } ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { recordBeginTime, err := getBrainRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) if err != nil { log.Error("Parameter is wrong", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } page := ctx.QueryInt("page") if page <= 0 { page = 1 } pageSize := ctx.QueryInt("pagesize") if pageSize <= 0 { pageSize = DEFAULT_PAGE_SIZE } debugOneCount, err := models.GetDebugOneCount(beginTime, endTime) if err != nil { log.Error("Can not query debugOneCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) return } benchmarkOneCount, err := models.GetBenchmarkOneCount(beginTime, endTime) if err != nil { log.Error("Can not query benchmarkCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) return } trainOneCount, err := models.GetTrainOneCount(beginTime, endTime) if err != nil { log.Error("Can not query trainOneCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error")) return } debugTwoCount, err := models.GetDebugTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query debugTwoCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error")) return } trainTwoCount, err := models.GetTrainTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query DebugOneTotal count.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error")) return } inferenceTwoCount, err := models.GetInferenceTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query inferenceTwoCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error")) return } cloudbrainsPeriodData := CloudbrainsPeriodData{ DebugOneCount: debugOneCount, BenchmarkOneCount: benchmarkOneCount, TrainOneCount: trainOneCount, DebugTwoCount: debugTwoCount, TrainTwoCount: trainTwoCount, InferenceTwoCount: inferenceTwoCount, } ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, int64, int64, int64, error) { debugOneCount, err := models.GetDebugOneCount(beginTime, endTime) if err != nil { log.Error("Can not query debugOneCount.", err) return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } benchmarkOneCount, err := models.GetBenchmarkOneCount(beginTime, endTime) if err != nil { log.Error("Can not query benchmarkCount.", err) return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } trainOneCount, err := models.GetTrainOneCount(beginTime, endTime) if err != nil { log.Error("Can not query trainOneCount.", err) return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } debugTwoCount, err := models.GetDebugTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query debugTwoCount.", err) return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } trainTwoCount, err := models.GetTrainTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query DebugOneTotal count.", err) return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } inferenceTwoCount, err := models.GetInferenceTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query inferenceTwoCount.", err) return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } cloudbrainOneCount := debugOneCount + benchmarkOneCount + trainOneCount cloudbrainTwoCount := debugTwoCount + trainTwoCount + inferenceTwoCount cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err } func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) if err != nil { log.Error("Can not query getCloudbrainCount.", err) return nil, err } dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format(time.RFC3339), DebugOneCount: debugOneCount, BenchmarkOneCount: benchmarkOneCount, TrainOneCount: trainOneCount, DebugTwoCount: debugTwoCount, TrainTwoCount: trainTwoCount, InferenceTwoCount: inferenceTwoCount, CloudbrainOneCount: cloudbrainOneCount, CloudbrainTwoCount: cloudbrainTwoCount, CloudbrainCount: cloudbrainCount, }) beginTime = endTimeTemp endTimeTemp = beginTime.Add(time.Hour) } return dayCloudbrainNum, nil } func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) { debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) if err != nil { log.Error("Can not query getCloudbrainCount.", err) return nil, err } dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006-01-02"), DebugOneCount: debugOneCount, BenchmarkOneCount: benchmarkOneCount, TrainOneCount: trainOneCount, DebugTwoCount: debugTwoCount, TrainTwoCount: trainTwoCount, InferenceTwoCount: inferenceTwoCount, CloudbrainOneCount: cloudbrainOneCount, CloudbrainTwoCount: cloudbrainTwoCount, CloudbrainCount: cloudbrainCount, }) beginTime = endTimeTemp endTimeTemp = beginTime.AddDate(0, 0, 1) } return dayCloudbrainNum, nil } func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { yearCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) { debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) if err != nil { log.Error("Can not query getCloudbrainCount.", err) return nil, err } yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006-01"), DebugOneCount: debugOneCount, BenchmarkOneCount: benchmarkOneCount, TrainOneCount: trainOneCount, DebugTwoCount: debugTwoCount, TrainTwoCount: trainTwoCount, InferenceTwoCount: inferenceTwoCount, CloudbrainOneCount: cloudbrainOneCount, CloudbrainTwoCount: cloudbrainTwoCount, CloudbrainCount: cloudbrainCount, }) beginTime = endTimeTemp endTimeTemp = beginTime.AddDate(0, 1, 0) } return yearCloudbrainNum, nil } func getBrainRecordBeginTime() (time.Time, error) { return time.ParseInLocation(DATE_FORMAT, setting.BrainRecordBeginTime, time.Local) } func DownloadCloudBrainBoard(ctx *context.Context) { page := 1 pageSize := 300 var cloudBrain = ctx.Tr("repo.cloudbrain") fileName := getCloudbrainFileName(cloudBrain) _, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: pageSize, }, Type: models.TypeCloudBrainAll, NeedRepoInfo: false, }) if err != nil { log.Warn("Can not get cloud brain info", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) return } totalPage := getTotalPage(total, pageSize) fmt.Printf("total:%v", total) fmt.Printf("totalPage:%v", totalPage) f := excelize.NewFile() index := f.NewSheet(cloudBrain) f.DeleteSheet("Sheet1") for k, v := range allCloudbrainHeader(ctx) { f.SetCellValue(cloudBrain, k, v) } var row = 2 for i := 0; i < totalPage; i++ { pageRecords, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: pageSize, }, Type: models.TypeCloudBrainAll, NeedRepoInfo: true, }) if err != nil { log.Warn("Can not get cloud brain info", err) continue } for _, record := range pageRecords { for k, v := range allCloudbrainValues(row, record, ctx) { f.SetCellValue(cloudBrain, k, v) } row++ } page++ } f.SetActiveSheet(index) ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) ctx.Resp.Header().Set("Content-Type", "application/octet-stream") f.WriteTo(ctx.Resp) } func getCloudbrainFileName(baseName string) string { return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx" } func allCloudbrainHeader(ctx *context.Context) map[string]string { return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_task_type"), "C1": ctx.Tr("repo.modelarts.status"), "D1": ctx.Tr("repo.modelarts.createtime"), "E1": ctx.Tr("repo.modelarts.train_job.wait_time"), "F1": ctx.Tr("repo.modelarts.train_job.dura_time"), "G1": ctx.Tr("repo.modelarts.train_job.start_time"), "H1": ctx.Tr("repo.modelarts.train_job.end_time"), "I1": ctx.Tr("repo.modelarts.computing_resources"), "J1": ctx.Tr("repo.cloudbrain_creator"), "K1": ctx.Tr("repo.repo_name"), "L1": ctx.Tr("repo.cloudbrain_task_name")} } func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): rs.JobType, getCellName("C", row): rs.Status, getCellName("D", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("E", row): getBrainWaitTime(rs), getCellName("F", row): rs.TrainJobDuration, getCellName("G", row): getBrainStartTime(rs), getCellName("H", row): getBrainEndTime(rs), getCellName("I", row): rs.ComputeResource, getCellName("J", row): rs.Name, getCellName("K", row): getBrainRepo(rs), getCellName("L", row): rs.JobName, } } func getBrainRepo(rs *models.CloudbrainInfo) string { if rs.Repo != nil { return rs.Repo.OwnerName + "/" + rs.Repo.Alias } return "" } func getBrainStartTime(rs *models.CloudbrainInfo) string { timeString := time.Unix(int64(rs.Cloudbrain.StartTime), 0).Format(CREATE_TIME_FORMAT) if timeString != "1970/01/01 08:00:00" { return timeString } else { return "0" } } func getBrainEndTime(rs *models.CloudbrainInfo) string { timeString := time.Unix(int64(rs.Cloudbrain.EndTime), 0).Format(CREATE_TIME_FORMAT) if timeString != "1970/01/01 08:00:00" { return timeString } else { return "0" } } func getBrainWaitTime(rs *models.CloudbrainInfo) string { waitTime := rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix if waitTime <= 0 { return "0" } else { return models.ConvertDurationToStr(int64(waitTime)) } }