diff --git a/models/attachment.go b/models/attachment.go
index 684a38b21..96594bbb3 100755
--- a/models/attachment.go
+++ b/models/attachment.go
@@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) {
return total, nil
}
+
+func GetAllAttachmentSize() (int64, error) {
+ return x.SumInt(&Attachment{}, "size")
+}
diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index e63e58bda..11defbb21 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -151,23 +151,42 @@ type TaskPod struct {
TaskRoleStatus struct {
Name string `json:"name"`
} `json:"taskRoleStatus"`
- TaskStatuses []struct {
- TaskIndex int `json:"taskIndex"`
- PodUID string `json:"podUid"`
- PodIP string `json:"podIp"`
- PodName string `json:"podName"`
- ContainerID string `json:"containerId"`
- ContainerIP string `json:"containerIp"`
- ContainerGpus string `json:"containerGpus"`
- State string `json:"state"`
- StartAt time.Time `json:"startAt"`
- FinishedAt time.Time `json:"finishedAt"`
- ExitCode int `json:"exitCode"`
- ExitDiagnostics string `json:"exitDiagnostics"`
- RetriedCount int `json:"retriedCount"`
- StartTime string
- FinishedTime string
- } `json:"taskStatuses"`
+ //TaskStatuses []struct {
+ // TaskIndex int `json:"taskIndex"`
+ // PodUID string `json:"podUid"`
+ // PodIP string `json:"podIp"`
+ // PodName string `json:"podName"`
+ // ContainerID string `json:"containerId"`
+ // ContainerIP string `json:"containerIp"`
+ // ContainerGpus string `json:"containerGpus"`
+ // State string `json:"state"`
+ // StartAt time.Time `json:"startAt"`
+ // FinishedAt time.Time `json:"finishedAt"`
+ // ExitCode int `json:"exitCode"`
+ // ExitDiagnostics string `json:"exitDiagnostics"`
+ // RetriedCount int `json:"retriedCount"`
+ // StartTime string
+ // FinishedTime string
+ //} `json:"taskStatuses"`
+ TaskStatuses []TaskStatuses `json:"taskStatuses"`
+}
+
+type TaskStatuses struct {
+ TaskIndex int `json:"taskIndex"`
+ PodUID string `json:"podUid"`
+ PodIP string `json:"podIp"`
+ PodName string `json:"podName"`
+ ContainerID string `json:"containerId"`
+ ContainerIP string `json:"containerIp"`
+ ContainerGpus string `json:"containerGpus"`
+ State string `json:"state"`
+ StartAt time.Time `json:"startAt"`
+ FinishedAt time.Time `json:"finishedAt"`
+ ExitCode int `json:"exitCode"`
+ ExitDiagnostics string `json:"exitDiagnostics"`
+ RetriedCount int `json:"retriedCount"`
+ StartTime string
+ FinishedTime string
}
type TaskInfo struct {
@@ -679,7 +698,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
}
func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
- if !isSigned || job.Status != string(JobStopped) {
+ if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){
return false
}
repo, err := GetRepositoryByID(job.RepoID)
diff --git a/models/models.go b/models/models.go
index 412148235..0cdcee4fd 100755
--- a/models/models.go
+++ b/models/models.go
@@ -137,6 +137,7 @@ func init() {
tablesStatistic = append(tablesStatistic,
new(RepoStatistic),
+ new(SummaryStatistic),
new(UserBusinessAnalysis),
)
diff --git a/models/repo.go b/models/repo.go
index 7f4bfebba..dd36bf71d 100755
--- a/models/repo.go
+++ b/models/repo.go
@@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) {
}
+func GetAllRepositoriesCount() (int64, error) {
+ repo := new(Repository)
+ return x.Count(repo)
+}
+
+func GetAllRepositoriesSize() (int64, error) {
+ return x.SumInt(&Repository{}, "size")
+}
+
func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) {
repo.LowerName = strings.ToLower(repo.Name)
diff --git a/models/summary_statistic.go b/models/summary_statistic.go
new file mode 100644
index 000000000..0addd472b
--- /dev/null
+++ b/models/summary_statistic.go
@@ -0,0 +1,69 @@
+package models
+
+import (
+ "fmt"
+
+ "code.gitea.io/gitea/modules/timeutil"
+)
+
+var DomainMap = map[string]int{
+ "大模型": 0,
+ "ai开发工具": 1,
+ "计算机视觉": 2,
+ "自然语言处理": 3,
+ "机器学习": 4,
+ "神经网络": 5,
+ "自动驾驶": 6,
+ "机器人": 7,
+ "联邦学习": 8,
+ "数据挖掘": 9,
+ "risc-v开发": 10,
+}
+
+type SummaryStatistic struct {
+ ID int64 `xorm:"pk autoincr"`
+ Date string `xorm:"unique(s) NOT NULL"`
+ NumUsers int64 `xorm:"NOT NULL DEFAULT 0"`
+ RepoSize int64 `xorm:"NOT NULL DEFAULT 0"`
+ DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"`
+ NumOrganizations int64 `xorm:"NOT NULL DEFAULT 0"`
+ NumModels int64 `xorm:"NOT NULL DEFAULT 0"`
+ NumRepos int64 `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoBigModel int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoAI int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoVision int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoNLP int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoML int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoNN int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoAutoDrive int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoRobot int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoLeagueLearn int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoDataMining int `xorm:"NOT NULL DEFAULT 0"`
+ NumRepoRISC int `xorm:"NOT NULL DEFAULT 0"`
+ CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
+ UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
+}
+
+func DeleteSummaryStatisticDaily(date string) error {
+ sess := xStatistic.NewSession()
+ defer sess.Close()
+ if err := sess.Begin(); err != nil {
+ return fmt.Errorf("Begin: %v", err)
+ }
+
+ if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil {
+ return fmt.Errorf("Delete: %v", err)
+ }
+
+ if err := sess.Commit(); err != nil {
+ sess.Close()
+ return fmt.Errorf("Commit: %v", err)
+ }
+
+ sess.Close()
+ return nil
+}
+
+func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) {
+ return xStatistic.Insert(summaryStatistic)
+}
diff --git a/models/topic.go b/models/topic.go
index 397edd0c3..5533da7bc 100644
--- a/models/topic.go
+++ b/models/topic.go
@@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) {
return &topic, nil
}
+func GetAllUsedTopics() ([]*Topic, error) {
+ topics := make([]*Topic, 0)
+ err := x.Where("repo_count > ?", 0).Find(&topics)
+ return topics, err
+
+}
+
// addTopicByNameToRepo adds a topic name to a repo and increments the topic count.
// Returns topic after the addition
func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) {
diff --git a/models/user.go b/models/user.go
index 78ab4627a..1ee20d74c 100755
--- a/models/user.go
+++ b/models/user.go
@@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error {
return nil
}
+func GetUsersCount() (int64, error) {
+ user := new(User)
+ return x.Where("type=0").Count(user)
+
+}
+
+func GetOrganizationsCount() (int64, error) {
+ user := new(User)
+ return x.Where("type=1").Count(user)
+
+}
+
func GetBlockChainUnSuccessUsers() ([]*User, error) {
users := make([]*User, 0, 10)
err := x.Where("public_key = ''").
diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go
index 26cd16778..ed9829cef 100755
--- a/modules/cron/tasks_basic.go
+++ b/modules/cron/tasks_basic.go
@@ -174,6 +174,16 @@ func registerHandleRepoStatistic() {
})
}
+func registerHandleSummaryStatistic() {
+ RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
+ Enabled: true,
+ RunAtStart: false,
+ Schedule: "@daily",
+ }, func(ctx context.Context, _ *models.User, _ Config) error {
+ repo.SummaryStatistic()
+ return nil
+ })
+}
func registerHandleUserStatistic() {
RegisterTaskFatal("handle_user_statistic", &BaseConfig{
Enabled: true,
@@ -202,4 +212,5 @@ func initBasicTasks() {
registerHandleRepoStatistic()
registerHandleUserStatistic()
+ registerHandleSummaryStatistic()
}
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index 6dc0d410c..9ccf9c5c6 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -776,6 +776,7 @@ cloudbrain_creator=创建者
cloudbrain_task=任务名称
cloudbrain_operate=操作
cloudbrain_status_createtime=状态/创建时间
+cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。
template.items=模板选项
template.git_content=Git数据(默认分支)
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index dedbd028c..ccf2b879d 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -40,6 +40,8 @@ var (
categories *models.Categories
)
+var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
+
// MustEnableDataset check if repository enable internal cb
func MustEnableCloudbrain(ctx *context.Context) {
if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
@@ -200,6 +202,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
gpuQueue := setting.JobType
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
resourceSpecId := form.ResourceSpecId
+
+ if !jobNamePattern.MatchString(jobName) {
+ ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
+ return
+ }
if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
@@ -281,17 +288,30 @@ func CloudBrainShow(ctx *context.Context) {
if result != nil {
jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
- ctx.Data["result"] = jobRes
taskRoles := jobRes.TaskRoles
- taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
- ctx.Data["taskRes"] = taskRes
- task.Status = taskRes.TaskStatuses[0].State
- task.ContainerID = taskRes.TaskStatuses[0].ContainerID
- task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
- err = models.UpdateJob(task)
- if err != nil {
- ctx.Data["error"] = err.Error()
+ if jobRes.JobStatus.State != string(models.JobFailed) {
+ taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
+ ctx.Data["taskRes"] = taskRes
+ task.Status = taskRes.TaskStatuses[0].State
+ task.ContainerID = taskRes.TaskStatuses[0].ContainerID
+ task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
+ err = models.UpdateJob(task)
+ if err != nil {
+ ctx.Data["error"] = err.Error()
+ }
+ } else {
+ task.Status = jobRes.JobStatus.State
+ taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{
+ {
+ State: jobRes.JobStatus.State,
+ },
+ }}
+ ctx.Data["taskRes"] = taskRes
+ jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05")
+ jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05")
}
+
+ ctx.Data["result"] = jobRes
}
ctx.Data["task"] = task
@@ -351,7 +371,7 @@ func CloudBrainStop(ctx *context.Context) {
return
}
- if task.Status == string(models.JobStopped) {
+ if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) {
log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
ctx.ServerError("the job has been stopped", errors.New("the job has been stopped"))
return
@@ -454,7 +474,7 @@ func CloudBrainDel(ctx *context.Context) {
return
}
- if task.Status != string(models.JobStopped) {
+ if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed){
log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
return
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 080c36377..ce6cfb4d5 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -100,7 +100,10 @@ func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) {
uuid := form.Attachment
description := form.Description
//repo := ctx.Repo.Repository
-
+ if !jobNamePattern.MatchString(jobName) {
+ ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
+ return
+ }
err := modelarts.GenerateTask(ctx, jobName, uuid, description)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form)
diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go
new file mode 100644
index 000000000..53270664c
--- /dev/null
+++ b/routers/repo/repo_summary_statistic.go
@@ -0,0 +1,94 @@
+package repo
+
+import (
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/log"
+)
+
+func SummaryStatistic() {
+ log.Info("Generate summary statistic begin")
+ yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+ SummaryStatisticDaily(yesterday)
+ log.Info("Generate summary statistic end")
+}
+
+func SummaryStatisticDaily(date string) {
+ log.Info("%s", date)
+ if err := models.DeleteSummaryStatisticDaily(date); err != nil {
+ log.Error("DeleteRepoStatDaily failed: %v", err.Error())
+ return
+ }
+
+ //user number
+ userNumber, err := models.GetUsersCount()
+ if err != nil {
+ log.Error("can not get user number", err)
+ userNumber = 0
+ }
+ //organization number
+ organizationNumber, err := models.GetOrganizationsCount()
+ if err != nil {
+ log.Error("can not get orgnazition number", err)
+ organizationNumber = 0
+ }
+ // repository number
+ repositoryNumer, err := models.GetAllRepositoriesCount()
+ if err != nil {
+ log.Error("can not get repository number", err)
+ repositoryNumer = 0
+ }
+ //repository size
+ repositorySize, err := models.GetAllRepositoriesSize()
+ if err != nil {
+ log.Error("can not get repository size", err)
+ repositorySize = 0
+ }
+ // dataset size
+ allDatasetSize, err := models.GetAllAttachmentSize()
+ if err != nil {
+ log.Error("can not get dataset size", err)
+ allDatasetSize = 0
+ }
+ //topic repo number
+ topics, err := models.GetAllUsedTopics()
+ if err != nil {
+ log.Error("can not get topics", err)
+ }
+ var topicsCount [11]int
+ for _, topic := range topics {
+
+ index, exists := models.DomainMap[topic.Name]
+ if exists {
+ topicsCount[index] = topic.RepoCount
+ }
+
+ }
+
+ summaryStat := models.SummaryStatistic{
+ Date: date,
+ NumUsers: userNumber,
+ RepoSize: repositorySize,
+ DatasetSize: allDatasetSize,
+ NumOrganizations: organizationNumber,
+ NumRepos: repositoryNumer,
+ NumRepoBigModel: topicsCount[0],
+ NumRepoAI: topicsCount[1],
+ NumRepoVision: topicsCount[2],
+ NumRepoNLP: topicsCount[3],
+ NumRepoML: topicsCount[4],
+ NumRepoNN: topicsCount[5],
+ NumRepoAutoDrive: topicsCount[6],
+ NumRepoRobot: topicsCount[7],
+ NumRepoLeagueLearn: topicsCount[8],
+ NumRepoDataMining: topicsCount[9],
+ NumRepoRISC: topicsCount[10],
+ }
+
+ if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil {
+ log.Error("Insert summary Stat failed: %v", err.Error())
+ }
+
+ log.Info("finish summary statistic")
+}
diff --git a/templates/explore/dataset_list.tmpl b/templates/explore/dataset_list.tmpl
index 48ae78127..7abc03363 100755
--- a/templates/explore/dataset_list.tmpl
+++ b/templates/explore/dataset_list.tmpl
@@ -29,8 +29,12 @@
{{.Repo.OwnerName}} / {{.Title}}
- {{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}}
- {{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}}
+ {{if .Task}}
+ {{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}}
+ {{end}}
+ {{if .Category}}
+ {{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}}
+ {{end}}
{{svg "octicon-flame" 16}} {{.DownloadTimes}}
diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl
index b3c6ff835..3d9c080db 100755
--- a/templates/repo/cloudbrain/index.tmpl
+++ b/templates/repo/cloudbrain/index.tmpl
@@ -337,9 +337,9 @@
调试
-