From 51e23eade9adf71ae00df448b906c24499af07ba Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 20 Oct 2021 15:35:18 +0800 Subject: [PATCH 1/7] failed info --- models/cloudbrain.go | 53 ++++++++++++++++++++++++------------ routers/repo/cloudbrain.go | 33 +++++++++++++++------- templates/repo/cloudbrain/index.tmpl | 4 +-- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e63e58bda..de70f7905 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -151,23 +151,42 @@ type TaskPod struct { TaskRoleStatus struct { Name string `json:"name"` } `json:"taskRoleStatus"` - TaskStatuses []struct { - TaskIndex int `json:"taskIndex"` - PodUID string `json:"podUid"` - PodIP string `json:"podIp"` - PodName string `json:"podName"` - ContainerID string `json:"containerId"` - ContainerIP string `json:"containerIp"` - ContainerGpus string `json:"containerGpus"` - State string `json:"state"` - StartAt time.Time `json:"startAt"` - FinishedAt time.Time `json:"finishedAt"` - ExitCode int `json:"exitCode"` - ExitDiagnostics string `json:"exitDiagnostics"` - RetriedCount int `json:"retriedCount"` - StartTime string - FinishedTime string - } `json:"taskStatuses"` + //TaskStatuses []struct { + // TaskIndex int `json:"taskIndex"` + // PodUID string `json:"podUid"` + // PodIP string `json:"podIp"` + // PodName string `json:"podName"` + // ContainerID string `json:"containerId"` + // ContainerIP string `json:"containerIp"` + // ContainerGpus string `json:"containerGpus"` + // State string `json:"state"` + // StartAt time.Time `json:"startAt"` + // FinishedAt time.Time `json:"finishedAt"` + // ExitCode int `json:"exitCode"` + // ExitDiagnostics string `json:"exitDiagnostics"` + // RetriedCount int `json:"retriedCount"` + // StartTime string + // FinishedTime string + //} `json:"taskStatuses"` + TaskStatuses []TaskStatuses `json:"taskStatuses"` +} + +type TaskStatuses struct { + TaskIndex int `json:"taskIndex"` + PodUID string `json:"podUid"` + PodIP string `json:"podIp"` + PodName string `json:"podName"` + ContainerID string `json:"containerId"` + ContainerIP string `json:"containerIp"` + ContainerGpus string `json:"containerGpus"` + State string `json:"state"` + StartAt time.Time `json:"startAt"` + FinishedAt time.Time `json:"finishedAt"` + ExitCode int `json:"exitCode"` + ExitDiagnostics string `json:"exitDiagnostics"` + RetriedCount int `json:"retriedCount"` + StartTime string + FinishedTime string } type TaskInfo struct { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index dedbd028c..f7bff584a 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -281,17 +281,30 @@ func CloudBrainShow(ctx *context.Context) { if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB") - ctx.Data["result"] = jobRes taskRoles := jobRes.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - ctx.Data["taskRes"] = taskRes - task.Status = taskRes.TaskStatuses[0].State - task.ContainerID = taskRes.TaskStatuses[0].ContainerID - task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - err = models.UpdateJob(task) - if err != nil { - ctx.Data["error"] = err.Error() + if jobRes.JobStatus.State != string(models.JobFailed) { + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + ctx.Data["taskRes"] = taskRes + task.Status = taskRes.TaskStatuses[0].State + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + } + } else { + task.Status = jobRes.JobStatus.State + taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{ + { + State: jobRes.JobStatus.State, + }, + }} + ctx.Data["taskRes"] = taskRes + jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05") + jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05") } + + ctx.Data["result"] = jobRes } ctx.Data["task"] = task @@ -351,7 +364,7 @@ func CloudBrainStop(ctx *context.Context) { return } - if task.Status == string(models.JobStopped) { + if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) { log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) ctx.ServerError("the job has been stopped", errors.New("the job has been stopped")) return diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index b3c6ff835..57c592383 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -337,9 +337,9 @@ 调试 -
+ {{$.CsrfTokenHtml}} - + 停止
From 85e2558d4980e341a2eef4bd6362d28863cb268c Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 20 Oct 2021 15:49:44 +0800 Subject: [PATCH 2/7] update candel states --- models/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e63e58bda..3f0c5af20 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -679,7 +679,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { } func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { - if !isSigned || job.Status != string(JobStopped) { + if !isSigned || (job.Status != string(JobStopped) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){ return false } repo, err := GetRepositoryByID(job.RepoID) From 9ffb38bdde0c4f87bc80c9c31a1ba4e491c006cf Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 20 Oct 2021 16:38:30 +0800 Subject: [PATCH 3/7] =?UTF-8?q?fix-473=E5=90=8E=E7=AB=AF=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_zh-CN.ini | 1 + routers/repo/cloudbrain.go | 7 +++++++ routers/repo/modelarts.go | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 6dc0d410c..9ccf9c5c6 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -776,6 +776,7 @@ cloudbrain_creator=创建者 cloudbrain_task=任务名称 cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 +cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。 template.items=模板选项 template.git_content=Git数据(默认分支) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f7bff584a..0c83c99bd 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -40,6 +40,8 @@ var ( categories *models.Categories ) +var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) + // MustEnableDataset check if repository enable internal cb func MustEnableCloudbrain(ctx *context.Context) { if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { @@ -200,6 +202,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuQueue := setting.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId + + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form) + return + } if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 080c36377..ce6cfb4d5 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -100,7 +100,10 @@ func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) { uuid := form.Attachment description := form.Description //repo := ctx.Repo.Repository - + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form) + return + } err := modelarts.GenerateTask(ctx, jobName, uuid, description) if err != nil { ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form) From 6b243f593aa9a6170e97c1c9f1f0e612cfa41fa1 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 20 Oct 2021 17:13:26 +0800 Subject: [PATCH 4/7] cloudbrain --- models/cloudbrain.go | 2 +- templates/repo/cloudbrain/index.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6c084e33d..11defbb21 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -698,7 +698,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { } func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { - if !isSigned || (job.Status != string(JobStopped) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){ + if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){ return false } repo, err := GetRepositoryByID(job.RepoID) diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 57c592383..3d9c080db 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -337,9 +337,9 @@ 调试 -
+ {{$.CsrfTokenHtml}} - + 停止
From 42bad058a2aa15affe179379c7129b7b1e2ae6e5 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 21 Oct 2021 09:55:01 +0800 Subject: [PATCH 5/7] =?UTF-8?q?fix-359=20=E5=B9=B3=E5=8F=B0=E6=80=BB?= =?UTF-8?q?=E4=BD=93=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE=E5=85=A5=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 4 ++ models/models.go | 1 + models/repo.go | 9 ++++ models/summary_statistic.go | 69 +++++++++++++++++++++++++ models/topic.go | 7 +++ models/user.go | 12 +++++ modules/cron/tasks_basic.go | 11 ++++ routers/repo/repo_summary_statistic.go | 94 ++++++++++++++++++++++++++++++++++ 8 files changed, 207 insertions(+) create mode 100644 models/summary_statistic.go create mode 100644 routers/repo/repo_summary_statistic.go diff --git a/models/attachment.go b/models/attachment.go index 684a38b21..96594bbb3 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) { return total, nil } + +func GetAllAttachmentSize() (int64, error) { + return x.SumInt(&Attachment{}, "size") +} diff --git a/models/models.go b/models/models.go index 412148235..0cdcee4fd 100755 --- a/models/models.go +++ b/models/models.go @@ -137,6 +137,7 @@ func init() { tablesStatistic = append(tablesStatistic, new(RepoStatistic), + new(SummaryStatistic), new(UserBusinessAnalysis), ) diff --git a/models/repo.go b/models/repo.go index 7f4bfebba..dd36bf71d 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) { } +func GetAllRepositoriesCount() (int64, error) { + repo := new(Repository) + return x.Count(repo) +} + +func GetAllRepositoriesSize() (int64, error) { + return x.SumInt(&Repository{}, "size") +} + func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) { repo.LowerName = strings.ToLower(repo.Name) diff --git a/models/summary_statistic.go b/models/summary_statistic.go new file mode 100644 index 000000000..0addd472b --- /dev/null +++ b/models/summary_statistic.go @@ -0,0 +1,69 @@ +package models + +import ( + "fmt" + + "code.gitea.io/gitea/modules/timeutil" +) + +var DomainMap = map[string]int{ + "大模型": 0, + "ai开发工具": 1, + "计算机视觉": 2, + "自然语言处理": 3, + "机器学习": 4, + "神经网络": 5, + "自动驾驶": 6, + "机器人": 7, + "联邦学习": 8, + "数据挖掘": 9, + "risc-v开发": 10, +} + +type SummaryStatistic struct { + ID int64 `xorm:"pk autoincr"` + Date string `xorm:"unique(s) NOT NULL"` + NumUsers int64 `xorm:"NOT NULL DEFAULT 0"` + RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` + NumOrganizations int64 `xorm:"NOT NULL DEFAULT 0"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepos int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepoBigModel int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAI int `xorm:"NOT NULL DEFAULT 0"` + NumRepoVision int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNLP int `xorm:"NOT NULL DEFAULT 0"` + NumRepoML int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNN int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAutoDrive int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRobot int `xorm:"NOT NULL DEFAULT 0"` + NumRepoLeagueLearn int `xorm:"NOT NULL DEFAULT 0"` + NumRepoDataMining int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRISC int `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func DeleteSummaryStatisticDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) { + return xStatistic.Insert(summaryStatistic) +} diff --git a/models/topic.go b/models/topic.go index 397edd0c3..5533da7bc 100644 --- a/models/topic.go +++ b/models/topic.go @@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) { return &topic, nil } +func GetAllUsedTopics() ([]*Topic, error) { + topics := make([]*Topic, 0) + err := x.Where("repo_count > ?", 0).Find(&topics) + return topics, err + +} + // addTopicByNameToRepo adds a topic name to a repo and increments the topic count. // Returns topic after the addition func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) { diff --git a/models/user.go b/models/user.go index 78ab4627a..1ee20d74c 100755 --- a/models/user.go +++ b/models/user.go @@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { return nil } +func GetUsersCount() (int64, error) { + user := new(User) + return x.Where("type=0").Count(user) + +} + +func GetOrganizationsCount() (int64, error) { + user := new(User) + return x.Where("type=1").Count(user) + +} + func GetBlockChainUnSuccessUsers() ([]*User, error) { users := make([]*User, 0, 10) err := x.Where("public_key = ''"). diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 26cd16778..ed9829cef 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -174,6 +174,16 @@ func registerHandleRepoStatistic() { }) } +func registerHandleSummaryStatistic() { + RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SummaryStatistic() + return nil + }) +} func registerHandleUserStatistic() { RegisterTaskFatal("handle_user_statistic", &BaseConfig{ Enabled: true, @@ -202,4 +212,5 @@ func initBasicTasks() { registerHandleRepoStatistic() registerHandleUserStatistic() + registerHandleSummaryStatistic() } diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go new file mode 100644 index 000000000..53270664c --- /dev/null +++ b/routers/repo/repo_summary_statistic.go @@ -0,0 +1,94 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +func SummaryStatistic() { + log.Info("Generate summary statistic begin") + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + SummaryStatisticDaily(yesterday) + log.Info("Generate summary statistic end") +} + +func SummaryStatisticDaily(date string) { + log.Info("%s", date) + if err := models.DeleteSummaryStatisticDaily(date); err != nil { + log.Error("DeleteRepoStatDaily failed: %v", err.Error()) + return + } + + //user number + userNumber, err := models.GetUsersCount() + if err != nil { + log.Error("can not get user number", err) + userNumber = 0 + } + //organization number + organizationNumber, err := models.GetOrganizationsCount() + if err != nil { + log.Error("can not get orgnazition number", err) + organizationNumber = 0 + } + // repository number + repositoryNumer, err := models.GetAllRepositoriesCount() + if err != nil { + log.Error("can not get repository number", err) + repositoryNumer = 0 + } + //repository size + repositorySize, err := models.GetAllRepositoriesSize() + if err != nil { + log.Error("can not get repository size", err) + repositorySize = 0 + } + // dataset size + allDatasetSize, err := models.GetAllAttachmentSize() + if err != nil { + log.Error("can not get dataset size", err) + allDatasetSize = 0 + } + //topic repo number + topics, err := models.GetAllUsedTopics() + if err != nil { + log.Error("can not get topics", err) + } + var topicsCount [11]int + for _, topic := range topics { + + index, exists := models.DomainMap[topic.Name] + if exists { + topicsCount[index] = topic.RepoCount + } + + } + + summaryStat := models.SummaryStatistic{ + Date: date, + NumUsers: userNumber, + RepoSize: repositorySize, + DatasetSize: allDatasetSize, + NumOrganizations: organizationNumber, + NumRepos: repositoryNumer, + NumRepoBigModel: topicsCount[0], + NumRepoAI: topicsCount[1], + NumRepoVision: topicsCount[2], + NumRepoNLP: topicsCount[3], + NumRepoML: topicsCount[4], + NumRepoNN: topicsCount[5], + NumRepoAutoDrive: topicsCount[6], + NumRepoRobot: topicsCount[7], + NumRepoLeagueLearn: topicsCount[8], + NumRepoDataMining: topicsCount[9], + NumRepoRISC: topicsCount[10], + } + + if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil { + log.Error("Insert summary Stat failed: %v", err.Error()) + } + + log.Info("finish summary statistic") +} From a00349f3e0a307f486f94425aa4a5caa9cf4e7c9 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 21 Oct 2021 10:51:38 +0800 Subject: [PATCH 6/7] del --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 0c83c99bd..ccf2b879d 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -474,7 +474,7 @@ func CloudBrainDel(ctx *context.Context) { return } - if task.Status != string(models.JobStopped) { + if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed){ log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) return From 1d29fce90a4b82dd029b37d32cf49322e75f9bd3 Mon Sep 17 00:00:00 2001 From: wangjr Date: Thu, 21 Oct 2021 11:52:15 +0800 Subject: [PATCH 7/7] fix issue #384 --- templates/explore/dataset_list.tmpl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templates/explore/dataset_list.tmpl b/templates/explore/dataset_list.tmpl index 48ae78127..7abc03363 100755 --- a/templates/explore/dataset_list.tmpl +++ b/templates/explore/dataset_list.tmpl @@ -29,8 +29,12 @@ {{.Repo.OwnerName}} / {{.Title}}
- {{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}} - {{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}} + {{if .Task}} + {{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}} + {{end}} + {{if .Category}} + {{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}} + {{end}} {{svg "octicon-flame" 16}} {{.DownloadTimes}}