diff --git a/models/attachment.go b/models/attachment.go index 684a38b21..96594bbb3 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) { return total, nil } + +func GetAllAttachmentSize() (int64, error) { + return x.SumInt(&Attachment{}, "size") +} diff --git a/models/models.go b/models/models.go index 412148235..0cdcee4fd 100755 --- a/models/models.go +++ b/models/models.go @@ -137,6 +137,7 @@ func init() { tablesStatistic = append(tablesStatistic, new(RepoStatistic), + new(SummaryStatistic), new(UserBusinessAnalysis), ) diff --git a/models/repo.go b/models/repo.go index 7f4bfebba..dd36bf71d 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) { } +func GetAllRepositoriesCount() (int64, error) { + repo := new(Repository) + return x.Count(repo) +} + +func GetAllRepositoriesSize() (int64, error) { + return x.SumInt(&Repository{}, "size") +} + func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) { repo.LowerName = strings.ToLower(repo.Name) diff --git a/models/summary_statistic.go b/models/summary_statistic.go new file mode 100644 index 000000000..0addd472b --- /dev/null +++ b/models/summary_statistic.go @@ -0,0 +1,69 @@ +package models + +import ( + "fmt" + + "code.gitea.io/gitea/modules/timeutil" +) + +var DomainMap = map[string]int{ + "大模型": 0, + "ai开发工具": 1, + "计算机视觉": 2, + "自然语言处理": 3, + "机器学习": 4, + "神经网络": 5, + "自动驾驶": 6, + "机器人": 7, + "联邦学习": 8, + "数据挖掘": 9, + "risc-v开发": 10, +} + +type SummaryStatistic struct { + ID int64 `xorm:"pk autoincr"` + Date string `xorm:"unique(s) NOT NULL"` + NumUsers int64 `xorm:"NOT NULL DEFAULT 0"` + RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` + NumOrganizations int64 `xorm:"NOT NULL DEFAULT 0"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepos int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepoBigModel int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAI int `xorm:"NOT NULL DEFAULT 0"` + NumRepoVision int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNLP int `xorm:"NOT NULL DEFAULT 0"` + NumRepoML int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNN int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAutoDrive int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRobot int `xorm:"NOT NULL DEFAULT 0"` + NumRepoLeagueLearn int `xorm:"NOT NULL DEFAULT 0"` + NumRepoDataMining int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRISC int `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func DeleteSummaryStatisticDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) { + return xStatistic.Insert(summaryStatistic) +} diff --git a/models/topic.go b/models/topic.go index 397edd0c3..5533da7bc 100644 --- a/models/topic.go +++ b/models/topic.go @@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) { return &topic, nil } +func GetAllUsedTopics() ([]*Topic, error) { + topics := make([]*Topic, 0) + err := x.Where("repo_count > ?", 0).Find(&topics) + return topics, err + +} + // addTopicByNameToRepo adds a topic name to a repo and increments the topic count. // Returns topic after the addition func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) { diff --git a/models/user.go b/models/user.go index 78ab4627a..1ee20d74c 100755 --- a/models/user.go +++ b/models/user.go @@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { return nil } +func GetUsersCount() (int64, error) { + user := new(User) + return x.Where("type=0").Count(user) + +} + +func GetOrganizationsCount() (int64, error) { + user := new(User) + return x.Where("type=1").Count(user) + +} + func GetBlockChainUnSuccessUsers() ([]*User, error) { users := make([]*User, 0, 10) err := x.Where("public_key = ''"). diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 26cd16778..ed9829cef 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -174,6 +174,16 @@ func registerHandleRepoStatistic() { }) } +func registerHandleSummaryStatistic() { + RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SummaryStatistic() + return nil + }) +} func registerHandleUserStatistic() { RegisterTaskFatal("handle_user_statistic", &BaseConfig{ Enabled: true, @@ -202,4 +212,5 @@ func initBasicTasks() { registerHandleRepoStatistic() registerHandleUserStatistic() + registerHandleSummaryStatistic() } diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go new file mode 100644 index 000000000..53270664c --- /dev/null +++ b/routers/repo/repo_summary_statistic.go @@ -0,0 +1,94 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +func SummaryStatistic() { + log.Info("Generate summary statistic begin") + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + SummaryStatisticDaily(yesterday) + log.Info("Generate summary statistic end") +} + +func SummaryStatisticDaily(date string) { + log.Info("%s", date) + if err := models.DeleteSummaryStatisticDaily(date); err != nil { + log.Error("DeleteRepoStatDaily failed: %v", err.Error()) + return + } + + //user number + userNumber, err := models.GetUsersCount() + if err != nil { + log.Error("can not get user number", err) + userNumber = 0 + } + //organization number + organizationNumber, err := models.GetOrganizationsCount() + if err != nil { + log.Error("can not get orgnazition number", err) + organizationNumber = 0 + } + // repository number + repositoryNumer, err := models.GetAllRepositoriesCount() + if err != nil { + log.Error("can not get repository number", err) + repositoryNumer = 0 + } + //repository size + repositorySize, err := models.GetAllRepositoriesSize() + if err != nil { + log.Error("can not get repository size", err) + repositorySize = 0 + } + // dataset size + allDatasetSize, err := models.GetAllAttachmentSize() + if err != nil { + log.Error("can not get dataset size", err) + allDatasetSize = 0 + } + //topic repo number + topics, err := models.GetAllUsedTopics() + if err != nil { + log.Error("can not get topics", err) + } + var topicsCount [11]int + for _, topic := range topics { + + index, exists := models.DomainMap[topic.Name] + if exists { + topicsCount[index] = topic.RepoCount + } + + } + + summaryStat := models.SummaryStatistic{ + Date: date, + NumUsers: userNumber, + RepoSize: repositorySize, + DatasetSize: allDatasetSize, + NumOrganizations: organizationNumber, + NumRepos: repositoryNumer, + NumRepoBigModel: topicsCount[0], + NumRepoAI: topicsCount[1], + NumRepoVision: topicsCount[2], + NumRepoNLP: topicsCount[3], + NumRepoML: topicsCount[4], + NumRepoNN: topicsCount[5], + NumRepoAutoDrive: topicsCount[6], + NumRepoRobot: topicsCount[7], + NumRepoLeagueLearn: topicsCount[8], + NumRepoDataMining: topicsCount[9], + NumRepoRISC: topicsCount[10], + } + + if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil { + log.Error("Insert summary Stat failed: %v", err.Error()) + } + + log.Info("finish summary statistic") +}