Merge branch 'V20211018' into zouap

models/cloudbrain.go

@@ -620,6 +620,18 @@ func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) {
 	return getRepoCloudBrain(cb)
 }
 
+func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) {
+	cloudBrains := make([]*Cloudbrain, 0)
+	err := x.Cols("job_id", "status", "type").Where("user_id=? AND (status =? OR status=?)", userID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains)
+	return cloudBrains, err
+}
+
+func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) {
+	cloudBrains := make([]*Cloudbrain, 0)
+	err := x.Cols("job_id", "status", "type").Where("repo_id=? AND (status =? OR status=?)", repoID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains)
+	return cloudBrains, err
+}
+
 func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err error) {
 	cb := &Cloudbrain{JobID: jobID, Status: string(status)}
 	_, err = x.Cols("status").Where("cloudbrain.job_id=?", jobID).Update(cb)

models/repo.go

@@ -1424,6 +1424,12 @@ func GetAllRepositories() ([]*Repository, error) {
 	return getALLRepositories(x)
 }
 
+func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) {
+	repos := make([]*Repository, 0, 1000)
+	return repos, x.Cols(columns...).Find(&repos)
+
+}
+
 func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) {
 	repo.LowerName = strings.ToLower(repo.Name)
 

models/repo_activity_custom.go

@@ -0,0 +1,39 @@
+package models
+
+import "code.gitea.io/gitea/modules/git"
+
+func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) {
+	return git.GetRepoKPIStats(repo.RepoPath())
+}
+
+func GetAllUserKPIStats() (map[string]*git.UserKPIStats, error) {
+	authors := make(map[string]*git.UserKPIStats)
+	repositorys, err := GetAllRepositoriesByFilterCols("owner_name", "name")
+	if err != nil {
+		return nil, err
+	}
+
+	for _, repository := range repositorys {
+		authorsOneRepo, err1 := git.GetUserKPIStats(repository.RepoPath())
+		if err1 != nil {
+			return nil, err
+		}
+
+		for key, value := range authorsOneRepo {
+			if _, ok := authors[key]; !ok {
+				authors[key] = &git.UserKPIStats{
+
+					Name:        value.Name,
+					Email:       value.Email,
+					Commits:     0,
+					CommitLines: 0,
+				}
+			}
+			authors[key].Commits += value.Commits
+			authors[key].CommitLines += value.CommitLines
+
+		}
+
+	}
+	return authors, nil
+}

modules/git/repo_stats_custom.go

@@ -0,0 +1,258 @@
+package git
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RepoKPIStats struct {
+	Contributors        int64
+	KeyContributors     int64
+	ContributorsAdded   int64
+	CommitsAdded        int64
+	CommitLinesModified int64
+	Authors             []*UserKPITypeStats
+}
+
+type UserKPIStats struct {
+	Name        string
+	Email       string
+	Commits     int64
+	CommitLines int64
+}
+type UserKPITypeStats struct {
+	UserKPIStats
+	isNewContributor bool //是否是4个月内的新增贡献者
+}
+
+func GetRepoKPIStats(repoPath string) (*RepoKPIStats, error) {
+	stats := &RepoKPIStats{}
+
+	contributors, err := GetContributors(repoPath)
+	if err != nil {
+		return nil, err
+	}
+	timeUntil := time.Now()
+	fourMonthAgo := timeUntil.AddDate(0, -4, 0)
+	recentlyContributors, err := getContributors(repoPath, fourMonthAgo)
+	newContributersDict := make(map[string]struct{})
+	if err != nil {
+		return nil, err
+	}
+
+	if contributors != nil {
+		stats.Contributors = int64(len(contributors))
+		for _, contributor := range contributors {
+			if contributor.CommitCnt >= 3 {
+				stats.KeyContributors++
+			}
+
+			if recentlyContributors != nil {
+				for _, recentlyContributor := range recentlyContributors {
+					if recentlyContributor.Email == contributor.Email && recentlyContributor.CommitCnt == contributor.CommitCnt {
+						stats.ContributorsAdded++
+						newContributersDict[recentlyContributor.Email] = struct{}{}
+					}
+
+				}
+			}
+
+		}
+
+	}
+
+	err = setRepoKPIStats(repoPath, fourMonthAgo, stats, newContributersDict)
+
+	if err != nil {
+		return nil, fmt.Errorf("FillFromGit: %v", err)
+	}
+	return stats, nil
+
+}
+
+//获取一天内的用户贡献指标
+func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) {
+	timeUntil := time.Now()
+	oneDayAgo := timeUntil.AddDate(0, 0, -1)
+	since := oneDayAgo.Format(time.RFC3339)
+	args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)}
+	stdout, err := NewCommand(args...).RunInDirBytes(repoPath)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewReader(stdout))
+	scanner.Split(bufio.ScanLines)
+	usersKPIStatses := make(map[string]*UserKPIStats)
+	var author string
+	p := 0
+	var email string
+	for scanner.Scan() {
+		l := strings.TrimSpace(scanner.Text())
+		if l == "---" {
+			p = 1
+		} else if p == 0 {
+			continue
+		} else {
+			p++
+		}
+		if p > 4 && len(l) == 0 {
+			continue
+		}
+		switch p {
+		case 1: // Separator
+		case 2: // Commit sha-1
+		case 3: // Author
+			author = l
+		case 4: // E-mail
+			email = strings.ToLower(l)
+			if _, ok := usersKPIStatses[email]; !ok {
+				usersKPIStatses[email] = &UserKPIStats{
+					Name:        author,
+					Email:       email,
+					Commits:     0,
+					CommitLines: 0,
+				}
+			}
+
+			usersKPIStatses[email].Commits++
+		default: // Changed file
+			if parts := strings.Fields(l); len(parts) >= 3 {
+				if parts[0] != "-" {
+					if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil {
+						usersKPIStatses[email].CommitLines += c
+					}
+				}
+				if parts[1] != "-" {
+					if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil {
+						usersKPIStatses[email].CommitLines += c
+					}
+				}
+
+			}
+		}
+	}
+
+	return usersKPIStatses, nil
+
+}
+
+func setRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error {
+	since := fromTime.Format(time.RFC3339)
+	args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)}
+
+	stdout, err := NewCommand(args...).RunInDirBytes(repoPath)
+	if err != nil {
+		return err
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(stdout))
+	scanner.Split(bufio.ScanLines)
+
+	authors := make(map[string]*UserKPITypeStats)
+
+	var author string
+	p := 0
+	var email string
+	for scanner.Scan() {
+		l := strings.TrimSpace(scanner.Text())
+		if l == "---" {
+			p = 1
+		} else if p == 0 {
+			continue
+		} else {
+			p++
+		}
+		if p > 4 && len(l) == 0 {
+			continue
+		}
+		switch p {
+		case 1: // Separator
+		case 2: // Commit sha-1
+			stats.CommitsAdded++
+		case 3: // Author
+			author = l
+		case 4: // E-mail
+			email = strings.ToLower(l)
+			if _, ok := authors[email]; !ok {
+				authors[email] = &UserKPITypeStats{
+					UserKPIStats: UserKPIStats{
+						Name:        author,
+						Email:       email,
+						Commits:     0,
+						CommitLines: 0,
+					},
+					isNewContributor: false,
+				}
+			}
+			if _, ok := newContributers[email]; ok {
+				authors[email].isNewContributor = true
+			}
+
+			authors[email].Commits++
+		default: // Changed file
+			if parts := strings.Fields(l); len(parts) >= 3 {
+				if parts[0] != "-" {
+					if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil {
+						stats.CommitLinesModified += c
+						authors[email].CommitLines += c
+					}
+				}
+				if parts[1] != "-" {
+					if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil {
+						stats.CommitLinesModified += c
+						authors[email].CommitLines += c
+					}
+				}
+
+			}
+		}
+	}
+
+	a := make([]*UserKPITypeStats, 0, len(authors))
+	for _, v := range authors {
+		a = append(a, v)
+	}
+	// Sort authors descending depending on commit count
+	sort.Slice(a, func(i, j int) bool {
+		return a[i].Commits > a[j].Commits
+	})
+
+	stats.Authors = a
+	return nil
+
+}
+
+func getContributors(repoPath string, fromTime time.Time) ([]Contributor, error) {
+	since := fromTime.Format(time.RFC3339)
+	cmd := NewCommand("shortlog", "-sne", "--all", fmt.Sprintf("--since='%s'", since))
+	stdout, err := cmd.RunInDir(repoPath)
+	if err != nil {
+		return nil, err
+	}
+	stdout = strings.Trim(stdout, "\n")
+	contributorRows := strings.Split(stdout, "\n")
+	if len(contributorRows) > 0 {
+		contributorsInfo := make([]Contributor, len(contributorRows))
+		for i := 0; i < len(contributorRows); i++ {
+			var oneCount string = strings.Trim(contributorRows[i], " ")
+			if strings.Index(oneCount, "\t") < 0 {
+				continue
+			}
+			number := oneCount[0:strings.Index(oneCount, "\t")]
+			commitCnt, _ := strconv.Atoi(number)
+			committer := oneCount[strings.Index(oneCount, "\t")+1 : strings.LastIndex(oneCount, " ")]
+			committer = strings.Trim(committer, " ")
+			email := oneCount[strings.Index(oneCount, "<")+1 : strings.Index(oneCount, ">")]
+			contributorsInfo[i] = Contributor{
+				commitCnt, committer, email,
+			}
+		}
+		return contributorsInfo, nil
+	}
+	return nil, nil
+}

routers/repo/cloudbrain.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 	"time"
 
+	"code.gitea.io/gitea/modules/modelarts"
+
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/storage"
 
@@ -361,6 +363,58 @@ func CloudBrainStop(ctx *context.Context) {
 	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
 }
 
+func StopJobsByUserID(userID int64) {
+	cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
+	if err != nil {
+		log.Warn("Failed to get cloudBrain info", err)
+		return
+	}
+	StopJobs(cloudBrains)
+
+}
+
+func StopJobsByRepoID(repoID int64) {
+	cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
+	if err != nil {
+		log.Warn("Failed to get cloudBrain info", err)
+		return
+	}
+	StopJobs(cloudBrains)
+}
+
+/**
+
+ */
+func StopJobs(cloudBrains []*models.Cloudbrain) {
+
+	for _, taskInfo := range cloudBrains {
+
+		if taskInfo.Type == models.TypeCloudBrainOne {
+			err := cloudbrain.StopJob(taskInfo.JobID)
+			logErrorAndUpdateJobStatus(err, taskInfo)
+		} else {
+			param := models.NotebookAction{
+				Action: models.ActionStop,
+			}
+			_, err := modelarts.StopJob(taskInfo.JobID, param)
+			logErrorAndUpdateJobStatus(err, taskInfo)
+		}
+
+	}
+}
+
+func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
+	if err != nil {
+		log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
+	} else {
+		taskInfo.Status = string(models.JobStopped)
+		err = models.UpdateJob(taskInfo)
+		if err != nil {
+			log.Warn("UpdateJob failed", err)
+		}
+	}
+}
+
 func CloudBrainDel(ctx *context.Context) {
 	var jobID = ctx.Params(":jobid")
 	task, err := models.GetCloudbrainByJobID(jobID)

routers/repo/setting.go

@@ -440,6 +440,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
 			return
 		}
 		log.Trace("Repository deleted: %s/%s", ctx.Repo.Owner.Name, repo.Name)
+		go StopJobsByRepoID(repo.ID)
 
 		ctx.Flash.Success(ctx.Tr("repo.settings.deletion_success"))
 		ctx.Redirect(ctx.Repo.Owner.DashboardLink())

routers/user/auth.go

@@ -11,6 +11,8 @@ import (
 	"net/http"
 	"strings"
 
+	"code.gitea.io/gitea/routers/repo"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/auth/oauth2"
@@ -1056,6 +1058,7 @@ func SignOut(ctx *context.Context) {
 		})
 	}
 	HandleSignOut(ctx)
+	go repo.StopJobsByUserID(ctx.User.ID)
 	ctx.Redirect(setting.AppSubURL + "/")
 }