From 5a7904fe3b978856678efe16f3425654f70790f2 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Sat, 9 Oct 2021 09:43:24 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D#333?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 12 +++++++++++ routers/repo/cloudbrain.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++ routers/repo/setting.go | 1 + routers/user/auth.go | 3 +++ 4 files changed, 70 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index edd5f102a..5be07f741 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -620,6 +620,18 @@ func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } +func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) { + cloudBrains := make([]*Cloudbrain, 0) + err := x.Cols("job_id", "status", "type").Where("user_id=? AND (status =? OR status=?)", userID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains) + return cloudBrains, err +} + +func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) { + cloudBrains := make([]*Cloudbrain, 0) + err := x.Cols("job_id", "status", "type").Where("repo_id=? AND (status =? OR status=?)", repoID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains) + return cloudBrains, err +} + func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err error) { cb := &Cloudbrain{JobID: jobID, Status: string(status)} _, err = x.Cols("status").Where("cloudbrain.job_id=?", jobID).Update(cb) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index deac7b312..c30906af4 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/storage" @@ -361,6 +363,58 @@ func CloudBrainStop(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") } +func StopJobsByUserID(userID int64) { + cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID) + if err != nil { + log.Warn("Failed to get cloudBrain info", err) + return + } + StopJobs(cloudBrains) + +} + +func StopJobsByRepoID(repoID int64) { + cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID) + if err != nil { + log.Warn("Failed to get cloudBrain info", err) + return + } + StopJobs(cloudBrains) +} + +/** + + */ +func StopJobs(cloudBrains []*models.Cloudbrain) { + + for _, taskInfo := range cloudBrains { + + if taskInfo.Type == models.TypeCloudBrainOne { + err := cloudbrain.StopJob(taskInfo.JobID) + logErrorAndUpdateJobStatus(err, taskInfo) + } else { + param := models.NotebookAction{ + Action: models.ActionStop, + } + _, err := modelarts.StopJob(taskInfo.JobID, param) + logErrorAndUpdateJobStatus(err, taskInfo) + } + + } +} + +func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { + if err != nil { + log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err) + } else { + taskInfo.Status = string(models.JobStopped) + err = models.UpdateJob(taskInfo) + if err != nil { + log.Warn("UpdateJob failed", err) + } + } +} + func CloudBrainDel(ctx *context.Context) { var jobID = ctx.Params(":jobid") task, err := models.GetCloudbrainByJobID(jobID) diff --git a/routers/repo/setting.go b/routers/repo/setting.go index b2ca042de..bf11f9e5a 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -440,6 +440,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { return } log.Trace("Repository deleted: %s/%s", ctx.Repo.Owner.Name, repo.Name) + go StopJobsByRepoID(repo.ID) ctx.Flash.Success(ctx.Tr("repo.settings.deletion_success")) ctx.Redirect(ctx.Repo.Owner.DashboardLink()) diff --git a/routers/user/auth.go b/routers/user/auth.go index 13e338565..126d0a4c8 100755 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -11,6 +11,8 @@ import ( "net/http" "strings" + "code.gitea.io/gitea/routers/repo" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/auth/oauth2" @@ -1056,6 +1058,7 @@ func SignOut(ctx *context.Context) { }) } HandleSignOut(ctx) + go repo.StopJobsByUserID(ctx.User.ID) ctx.Redirect(setting.AppSubURL + "/") } From a612e06c70eb3b63bd08c0e24b692840dd9977c4 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Sat, 9 Oct 2021 09:58:54 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E7=BB=9F=E8=AE=A1commit,?= =?UTF-8?q?commit=E8=A1=8C=E6=95=B0=EF=BC=8C=E6=96=B0=E8=B4=A1=E7=8C=AE?= =?UTF-8?q?=E8=80=85=E5=90=8E=E7=AB=AF=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/repo.go | 6 + models/repo_activity_custom.go | 39 ++++++ modules/git/repo_stats_custom.go | 258 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 303 insertions(+) create mode 100644 models/repo_activity_custom.go create mode 100644 modules/git/repo_stats_custom.go diff --git a/models/repo.go b/models/repo.go index 2742c3e31..7f4bfebba 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1424,6 +1424,12 @@ func GetAllRepositories() ([]*Repository, error) { return getALLRepositories(x) } +func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) { + repos := make([]*Repository, 0, 1000) + return repos, x.Cols(columns...).Find(&repos) + +} + func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) { repo.LowerName = strings.ToLower(repo.Name) diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go new file mode 100644 index 000000000..44d20abb3 --- /dev/null +++ b/models/repo_activity_custom.go @@ -0,0 +1,39 @@ +package models + +import "code.gitea.io/gitea/modules/git" + +func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) { + return git.GetRepoKPIStats(repo.RepoPath()) +} + +func GetAllUserKPIStats() (map[string]*git.UserKPIStats, error) { + authors := make(map[string]*git.UserKPIStats) + repositorys, err := GetAllRepositoriesByFilterCols("owner_name", "name") + if err != nil { + return nil, err + } + + for _, repository := range repositorys { + authorsOneRepo, err1 := git.GetUserKPIStats(repository.RepoPath()) + if err1 != nil { + return nil, err + } + + for key, value := range authorsOneRepo { + if _, ok := authors[key]; !ok { + authors[key] = &git.UserKPIStats{ + + Name: value.Name, + Email: value.Email, + Commits: 0, + CommitLines: 0, + } + } + authors[key].Commits += value.Commits + authors[key].CommitLines += value.CommitLines + + } + + } + return authors, nil +} diff --git a/modules/git/repo_stats_custom.go b/modules/git/repo_stats_custom.go new file mode 100644 index 000000000..95f1086ad --- /dev/null +++ b/modules/git/repo_stats_custom.go @@ -0,0 +1,258 @@ +package git + +import ( + "bufio" + "bytes" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +type RepoKPIStats struct { + Contributors int64 + KeyContributors int64 + ContributorsAdded int64 + CommitsAdded int64 + CommitLinesModified int64 + Authors []*UserKPITypeStats +} + +type UserKPIStats struct { + Name string + Email string + Commits int64 + CommitLines int64 +} +type UserKPITypeStats struct { + UserKPIStats + isNewContributor bool //是否是4个月内的新增贡献者 +} + +func GetRepoKPIStats(repoPath string) (*RepoKPIStats, error) { + stats := &RepoKPIStats{} + + contributors, err := GetContributors(repoPath) + if err != nil { + return nil, err + } + timeUntil := time.Now() + fourMonthAgo := timeUntil.AddDate(0, -4, 0) + recentlyContributors, err := getContributors(repoPath, fourMonthAgo) + newContributersDict := make(map[string]struct{}) + if err != nil { + return nil, err + } + + if contributors != nil { + stats.Contributors = int64(len(contributors)) + for _, contributor := range contributors { + if contributor.CommitCnt >= 3 { + stats.KeyContributors++ + } + + if recentlyContributors != nil { + for _, recentlyContributor := range recentlyContributors { + if recentlyContributor.Email == contributor.Email && recentlyContributor.CommitCnt == contributor.CommitCnt { + stats.ContributorsAdded++ + newContributersDict[recentlyContributor.Email] = struct{}{} + } + + } + } + + } + + } + + err = setRepoKPIStats(repoPath, fourMonthAgo, stats, newContributersDict) + + if err != nil { + return nil, fmt.Errorf("FillFromGit: %v", err) + } + return stats, nil + +} + +//获取一天内的用户贡献指标 +func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { + timeUntil := time.Now() + oneDayAgo := timeUntil.AddDate(0, 0, -1) + since := oneDayAgo.Format(time.RFC3339) + args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} + stdout, err := NewCommand(args...).RunInDirBytes(repoPath) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(stdout)) + scanner.Split(bufio.ScanLines) + usersKPIStatses := make(map[string]*UserKPIStats) + var author string + p := 0 + var email string + for scanner.Scan() { + l := strings.TrimSpace(scanner.Text()) + if l == "---" { + p = 1 + } else if p == 0 { + continue + } else { + p++ + } + if p > 4 && len(l) == 0 { + continue + } + switch p { + case 1: // Separator + case 2: // Commit sha-1 + case 3: // Author + author = l + case 4: // E-mail + email = strings.ToLower(l) + if _, ok := usersKPIStatses[email]; !ok { + usersKPIStatses[email] = &UserKPIStats{ + Name: author, + Email: email, + Commits: 0, + CommitLines: 0, + } + } + + usersKPIStatses[email].Commits++ + default: // Changed file + if parts := strings.Fields(l); len(parts) >= 3 { + if parts[0] != "-" { + if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil { + usersKPIStatses[email].CommitLines += c + } + } + if parts[1] != "-" { + if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil { + usersKPIStatses[email].CommitLines += c + } + } + + } + } + } + + return usersKPIStatses, nil + +} + +func setRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { + since := fromTime.Format(time.RFC3339) + args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} + + stdout, err := NewCommand(args...).RunInDirBytes(repoPath) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewReader(stdout)) + scanner.Split(bufio.ScanLines) + + authors := make(map[string]*UserKPITypeStats) + + var author string + p := 0 + var email string + for scanner.Scan() { + l := strings.TrimSpace(scanner.Text()) + if l == "---" { + p = 1 + } else if p == 0 { + continue + } else { + p++ + } + if p > 4 && len(l) == 0 { + continue + } + switch p { + case 1: // Separator + case 2: // Commit sha-1 + stats.CommitsAdded++ + case 3: // Author + author = l + case 4: // E-mail + email = strings.ToLower(l) + if _, ok := authors[email]; !ok { + authors[email] = &UserKPITypeStats{ + UserKPIStats: UserKPIStats{ + Name: author, + Email: email, + Commits: 0, + CommitLines: 0, + }, + isNewContributor: false, + } + } + if _, ok := newContributers[email]; ok { + authors[email].isNewContributor = true + } + + authors[email].Commits++ + default: // Changed file + if parts := strings.Fields(l); len(parts) >= 3 { + if parts[0] != "-" { + if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil { + stats.CommitLinesModified += c + authors[email].CommitLines += c + } + } + if parts[1] != "-" { + if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil { + stats.CommitLinesModified += c + authors[email].CommitLines += c + } + } + + } + } + } + + a := make([]*UserKPITypeStats, 0, len(authors)) + for _, v := range authors { + a = append(a, v) + } + // Sort authors descending depending on commit count + sort.Slice(a, func(i, j int) bool { + return a[i].Commits > a[j].Commits + }) + + stats.Authors = a + return nil + +} + +func getContributors(repoPath string, fromTime time.Time) ([]Contributor, error) { + since := fromTime.Format(time.RFC3339) + cmd := NewCommand("shortlog", "-sne", "--all", fmt.Sprintf("--since='%s'", since)) + stdout, err := cmd.RunInDir(repoPath) + if err != nil { + return nil, err + } + stdout = strings.Trim(stdout, "\n") + contributorRows := strings.Split(stdout, "\n") + if len(contributorRows) > 0 { + contributorsInfo := make([]Contributor, len(contributorRows)) + for i := 0; i < len(contributorRows); i++ { + var oneCount string = strings.Trim(contributorRows[i], " ") + if strings.Index(oneCount, "\t") < 0 { + continue + } + number := oneCount[0:strings.Index(oneCount, "\t")] + commitCnt, _ := strconv.Atoi(number) + committer := oneCount[strings.Index(oneCount, "\t")+1 : strings.LastIndex(oneCount, " ")] + committer = strings.Trim(committer, " ") + email := oneCount[strings.Index(oneCount, "<")+1 : strings.Index(oneCount, ">")] + contributorsInfo[i] = Contributor{ + commitCnt, committer, email, + } + } + return contributorsInfo, nil + } + return nil, nil +}