aiforge/models/cloudbrain_static.go

package models

import (
	"strconv"
	"time"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/timeutil"
	"code.gitea.io/gitea/modules/util"
	"xorm.io/builder"
)

type TaskDetail struct {
	ID              int64              `json:"ID"`
	JobID           string             `json:"JobID"`
	JobName         string             `json:"JobName"`
	DisplayJobName  string             `json:"DisplayJobName"`
	Status          string             `json:"Status"`
	JobType         string             `json:"JobType"`
	CreatedUnix     timeutil.TimeStamp `json:"CreatedUnix"`
	WaitTime        string             `json:"WaitTime"`
	RunTime         string             `json:"RunTime"`
	StartTime       timeutil.TimeStamp `json:"StartTime"`
	EndTime         timeutil.TimeStamp `json:"EndTime"`
	ComputeResource string             `json:"ComputeResource"`
	Type            int                `json:"Type"`
	UserName        string             `json:"UserName"`
	RepoName        string             `json:"RepoName"`
	RepoAlias       string             `json:"RepoAlias"`
	RepoID          int64              `json:"RepoID"`
	IsDelete        bool               `json:"IsDelete"`
	CardNum         int                `json:"CardNum"`
	CardType        string             `json:"CardType"`
	CardDuration    string             `json:"CardDuration"`
	AiCenter        string             `json:"AiCenter"`
	FlavorName      string             `json:"FlavorName"`
	WorkServerNum   int64              `json:"WorkServerNum"`
	Spec            *Specification     `json:"Spec"`
}

type CloudbrainDurationStatistic struct {
	ID              int64 `xorm:"pk autoincr"`
	Cluster         string
	AiCenterCode    string `xorm:"INDEX"`
	AiCenterName    string
	ComputeResource string
	AccCardType     string `xorm:"INDEX"`

	DateTime           timeutil.TimeStamp `xorm:"INDEX DEFAULT 0"`
	DateTimeUnix       timeutil.TimeStamp `xorm:"INDEX DEFAULT 0"`
	DayTime            string             `xorm:"INDEX"`
	HourTime           int                `xorm:"INDEX"`
	CardsUseDuration   int
	CardsTotalDuration int
	CardsTotalNum      int

	DeletedUnix timeutil.TimeStamp `xorm:"deleted"`
	CreatedUnix timeutil.TimeStamp `xorm:"created"`
	UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
type DurationStatisticOptions struct {
	BeginTime    time.Time
	EndTime      time.Time
	AiCenterCode string
}

type DurationRateStatistic struct {
	AiCenterTotalDurationStat map[string]int     `json:"aiCenterTotalDurationStat"`
	AiCenterUsageDurationStat map[string]int     `json:"aiCenterUsageDurationStat"`
	UsageRate                 map[string]float64 `json:"UsageRate"`
}
type ResourceDetail struct {
	QueueCode       string
	Cluster         string `xorm:"notnull"`
	AiCenterCode    string
	AiCenterName    string
	ComputeResource string
	AccCardType     string
	CardsTotalNum   int
	IsAutomaticSync bool
}

type DateUsageStatistic struct {
	Date          string  `json:"date"`
	UsageDuration int     `json:"usageDuration"`
	TotalDuration int     `json:"totalDuration"`
	UsageRate     float64 `json:"usageRate"`
}

type HourTimeStatistic struct {
	HourTimeUsageDuration map[string]int     `json:"hourTimeUsageDuration"`
	HourTimeTotalDuration map[string]int     `json:"hourTimeTotalDuration"`
	HourTimeUsageRate     map[string]float64 `json:"hourTimeUsageRate"`
}
type CloudbrainTypeDuration []struct {
	Type            int `xorm:"type"`
	DurationSum     int `xorm:"durationSum"`
	CardDurationSum int `xorm:"cardDurationSum"`
	Count           int `xorm:"count"`
}
type CloudbrainAllDuration struct {
	DurationSum     int `xorm:"durationSum"`
	CardDurationSum int `xorm:"cardDurationSum"`
	Count           int `xorm:"count"`
}

func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
	countSql := "SELECT count(distinct user_id) FROM  " +
		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)
	return x.SQL(countSql).Count()
}
func GetTodayCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, error) {
	countSql := "SELECT count(*) FROM  " +
		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10)
	return x.SQL(countSql).Count()
}
func GetTodayRunningCount(beginTime time.Time, endTime time.Time) (int64, error) {
	countSql := "SELECT count(*) FROM  " +
		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + " and (status='" + string(JobRunning) + "'" +
		" or status='" + string(ModelArtsTrainJobInit) + "')"
	return x.SQL(countSql).Count()
}
func GetTodayWaitingCount(beginTime time.Time, endTime time.Time) (int64, error) {
	countSql := "SELECT count(*) FROM  " +
		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + " and status='" + string(JobWaiting) + "'"
	return x.SQL(countSql).Count()
}

func GetCreatorCount() (int64, error) {
	countSql := "SELECT count(distinct user_id) FROM  public.cloudbrain"
	return x.SQL(countSql).Count()
}

func GetCloudbrainTypeCount() ([]map[string]string, error) {
	countSql := "SELECT type,count(*) num FROM public.cloudbrain group by type order by num desc"
	return x.QueryString(countSql)
}

func GetCloudbrainStatusCount() ([]map[string]string, error) {
	countSql := "SELECT status,count(*) num FROM public.cloudbrain group by status order by num desc"
	return x.QueryString(countSql)
}

func GetCloudbrainTpyeDurationSum() ([]map[string]string, error) {
	countSql := "SELECT type,sum(duration) FROM public.cloudbrain group by type order by sum(duration) desc"
	return x.QueryString(countSql)
}

func GetRecordBeginTime() ([]*CloudbrainInfo, error) {
	sess := x.NewSession()
	defer sess.Close()
	sess.OrderBy("cloudbrain.id ASC limit 1")
	cloudbrains := make([]*CloudbrainInfo, 0)
	if err := sess.Table(&Cloudbrain{}).Unscoped().
		Find(&cloudbrains); err != nil {
		log.Info("find error.")
	}
	return cloudbrains, nil
}

func GetAllStatusCloudBrain() map[string]int {
	sess := x.NewSession()
	defer sess.Close()
	cloudbrains := make([]*CloudbrainInfo, 0)
	if err := sess.Table(&Cloudbrain{}).Unscoped().
		Find(&cloudbrains); err != nil {
		log.Info("find error.")
	}
	cloudBrainStatusResult := make(map[string]int)
	for _, cloudbrain := range cloudbrains {
		if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok {
			cloudBrainStatusResult[cloudbrain.Status] = 1
		} else {
			cloudBrainStatusResult[cloudbrain.Status] += 1
		}
	}
	return cloudBrainStatusResult
}

func GetWaittingTop() ([]*CloudbrainInfo, error) {
	sess := x.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()
	cond = cond.And(
		builder.Eq{"cloudbrain.status": string(JobWaiting)},
	)
	sess.OrderBy("cloudbrain.created_unix ASC limit 10")
	cloudbrains := make([]*CloudbrainInfo, 0, 10)
	if err := sess.Table(&Cloudbrain{}).Where(cond).
		Find(&cloudbrains); err != nil {
		log.Info("find error.")
	}

	var ids []int64
	for _, task := range cloudbrains {
		ids = append(ids, task.RepoID)
	}
	repositoryMap, err := GetRepositoriesMapByIDs(ids)
	if err == nil {
		for _, task := range cloudbrains {
			task.Repo = repositoryMap[task.RepoID]
		}
	}
	return cloudbrains, nil
}

func GetRunningTop() ([]*CloudbrainInfo, error) {
	sess := x.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()
	cond = cond.And(
		builder.Eq{"cloudbrain.status": string(JobRunning)},
	)
	sess.OrderBy("cloudbrain.duration DESC limit 10")
	cloudbrains := make([]*CloudbrainInfo, 0, 10)
	if err := sess.Table(&Cloudbrain{}).Where(cond).
		Find(&cloudbrains); err != nil {
		log.Info("find error.")
	}
	var ids []int64
	for _, task := range cloudbrains {
		ids = append(ids, task.RepoID)
	}
	repositoryMap, err := GetRepositoriesMapByIDs(ids)
	if err == nil {
		for _, task := range cloudbrains {
			task.Repo = repositoryMap[task.RepoID]
		}
	}
	return cloudbrains, nil
}

func getCreatePeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) {
	countSql := "SET TIME ZONE 'PRC'; SELECT count(*) FROM  " +
		"public.cloudbrain where to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '" + dateBeginTime +
		"' and to_char(to_timestamp(created_unix), 'YYYY-MM-DD') < '" + dateEndTime +
		"' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') >= '" + hourBeginTime +
		"' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') < '" + hourEndTime + "'"
	return x.SQL(countSql).Count()
}

func getRunPeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) {
	countSql := "SET TIME ZONE 'PRC'; SELECT count(*) FROM  " +
		"public.cloudbrain where ((to_char(to_timestamp(start_time), ' HH24:MI:SS') < '" + hourEndTime +
		"') and (to_char(to_timestamp(CASE WHEN start_time is not null AND end_time isnull  THEN COALESCE(end_time,EXTRACT(epoch FROM now())) ELSE end_time END), 'HH24:MI:SS') >= '" + hourBeginTime + "'))" +
		" and (to_char(to_timestamp(start_time), 'YYYY-MM-DD') < '" + dateEndTime +
		"' and to_char(to_timestamp(end_time), 'YYYY-MM-DD') >= '" + dateBeginTime + "')"
	return x.SQL(countSql).Count()
}

func GetCreateHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) {
	//0 to 23 for each hour,
	dateHourMap := make(map[string]interface{})
	var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}
	for key, value := range slice {
		hourBeginHour := util.AddZero(value) + ":00:00"
		hourEndHour := util.AddZero(value+1) + ":00:00"
		cout, err := getCreatePeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour)
		if err != nil {
			log.Error("Can not query getCreatePeriodCount.", err)
			return nil, nil
		}
		dateHourMap[strconv.Itoa(key)] = cout
	}
	return dateHourMap, nil
}

func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]int64, error) {
	dateHourMap := make(map[string]int64)
	var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}
	for key, value := range slice {
		hourBeginHour := util.AddZero(value) + ":00:00"
		hourEndHour := util.AddZero(value+1) + ":00:00"
		cout, err := getRunPeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour)
		if err != nil {
			log.Error("Can not query getRunPeriodCount.", err)
			return nil, nil
		}
		dateHourMap[strconv.Itoa(key)] = cout
	}
	return dateHourMap, nil
}

func GetCloudbrainRunning() ([]*CloudbrainInfo, error) {
	sess := x.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()
	cond = cond.And(
		builder.Eq{"cloudbrain.status": string(JobRunning)},
	)
	sess.OrderBy("cloudbrain.created_unix ASC")
	cloudbrains := make([]*CloudbrainInfo, 0, 10)
	if err := sess.Table(&Cloudbrain{}).Where(cond).
		Find(&cloudbrains); err != nil {
		log.Info("find error.")
	}
	return cloudbrains, nil
}

func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) {
	sess := x.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()
	cond = cond.Or(
		builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.start_time": beginTime}, builder.Gt{"cloudbrain.start_time": 0}),
	)
	cond = cond.Or(
		builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}, builder.Gt{"cloudbrain.start_time": 0}),
	)
	cond = cond.Or(
		builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}, builder.Lte{"cloudbrain.start_time": beginTime}),
	)
	sess.OrderBy("cloudbrain.id ASC")
	cloudbrains := make([]*CloudbrainInfo, 0, 10)
	if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond).
		Find(&cloudbrains); err != nil {
		log.Error("find error.")
		return nil, err
	}
	return cloudbrains, nil
}

func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*CloudbrainSpec, error) {
	sess := x.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()
	cond = cond.And(
		builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}),
	)
	cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10)
	if err := sess.Table(&CloudbrainSpec{}).Where(cond).
		Find(&cloudbrainSpecs); err != nil {
		log.Error("find error.")
		return nil, err
	}
	return cloudbrainSpecs, nil
}

func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) {
	return xStatistic.Insert(cloudbrainDurationStatistic)
}

func getDurationStatistic(cb *CloudbrainDurationStatistic) (*CloudbrainDurationStatistic, error) {
	has, err := x.Get(cb)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrJobNotExist{}
	}
	return cb, nil
}

func GetCanUseCardInfo() ([]*ResourceQueue, error) {
	sess := x.NewSession()
	defer sess.Close()
	sess.OrderBy("resource_queue.cluster DESC, resource_queue.ai_center_code ASC")
	ResourceQueues := make([]*ResourceQueue, 0, 10)
	if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil {
		log.Error("find error.")
		return nil, err
	}
	return ResourceQueues, nil
}

func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) {
	sess := xStatistic.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()
	if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 {
		cond = cond.And(
			builder.And(builder.Gte{"cloudbrain_duration_statistic.date_time_unix": opts.BeginTime.Unix()}, builder.Lt{"cloudbrain_duration_statistic.date_time_unix": opts.EndTime.Unix()}),
		)
	}
	if opts.AiCenterCode != "" {
		cond = cond.And(
			builder.Eq{"cloudbrain_duration_statistic.ai_center_code": opts.AiCenterCode},
		)
	}
	CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10)
	if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond).
		Find(&CloudbrainDurationStatistics); err != nil {
		log.Error("find error.")
		return nil, err
	}
	return CloudbrainDurationStatistics, nil
}

func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) {
	sess := xStatistic.NewSession()
	defer sess.Close()

	var cond = builder.NewCond()

	cond = cond.And(
		builder.Gt{"cloudbrain_duration_statistic.date_time_unix": 0},
	)

	sess.OrderBy("cloudbrain_duration_statistic.date_time_unix ASC limit 1")
	CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0)
	if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond).Find(&CloudbrainDurationStatistics); err != nil {
		log.Error("find error.")
		return nil, err
	}
	return CloudbrainDurationStatistics, nil
}

func GetDurationRecordUpdateTime() ([]*CloudbrainDurationStatistic, error) {
	sess := xStatistic.NewSession()
	defer sess.Close()
	var cond = builder.NewCond()

	cond = cond.And(
		builder.Gt{"cloudbrain_duration_statistic.date_time_unix": 1577808000},
	)
	sess.OrderBy("cloudbrain_duration_statistic.date_time_unix DESC limit 1")
	CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0)
	if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond).Find(&CloudbrainDurationStatistics); err != nil {
		log.Error("find error.")
		return nil, err
	}
	return CloudbrainDurationStatistics, nil
}

func DeleteCloudbrainDurationStatistic(beginTime timeutil.TimeStamp, endTime timeutil.TimeStamp) error {
	sess := xStatistic.NewSession()
	defer sess.Close()
	if _, err := sess.Exec("DELETE FROM cloudbrain_duration_statistic WHERE cloudbrain_duration_statistic.date_time_unix BETWEEN ? AND ?", beginTime, endTime); err != nil {
		log.Error("DELETE cloudbrain_duration_statistic data error.")
		return err
	}
	return nil
}

func GetCloudbrainTypeCardDuration() (CloudbrainTypeDuration, error) {
	query := `
    SELECT
		cloudbrain.type,
		SUM(cloudbrain.duration) as durationSum,
        SUM(
            COALESCE(cloudbrain.duration *
				CASE
				WHEN cloudbrain.work_server_number = 0 THEN 1
				ELSE COALESCE(cloudbrain.work_server_number, 1)
				END *
				COALESCE(cloudbrain_spec.acc_cards_num, 1), 0)
   ) as cardDurationSum,
        COUNT(*) as count
    FROM cloudbrain
    LEFT JOIN cloudbrain_spec
        ON cloudbrain.id = cloudbrain_spec.cloudbrain_id
	GROUP BY cloudbrain.type
    `
	// 执行查询
	var results CloudbrainTypeDuration
	if err := x.SQL(query).Find(&results); err != nil {
		panic(err)
	}
	return results, nil
}

func GetCloudbrainAllCardDuration() (CloudbrainAllDuration, error) {
	query := `
    SELECT
		SUM(cloudbrain.duration) as durationSum,
        SUM(
            COALESCE(cloudbrain.duration *
				CASE
				WHEN cloudbrain.work_server_number = 0 THEN 1
				ELSE COALESCE(cloudbrain.work_server_number, 1)
				END *
				COALESCE(cloudbrain_spec.acc_cards_num, 1), 0)
   ) as cardDurationSum,
        COUNT(*) as count
    FROM cloudbrain
    LEFT JOIN cloudbrain_spec
        ON cloudbrain.id = cloudbrain_spec.cloudbrain_id
    `
	// 执行查询
	var result CloudbrainAllDuration
	if _, err := x.SQL(query).Get(&result); err != nil {
		panic(err)
	}
	return result, nil
}