Merge pull request '优化项目分析' (#966) from fix-828 into V20211115

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/966
Reviewed-by: lewis <747342561@qq.com>

models/repo_activity_custom.go

@@ -23,18 +23,22 @@ func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) {
 	if repo.HasWiki() {
 		wikiPath = repo.WikiPath()
 	}
-	return getRepoKPIStats(repo.RepoPath(), wikiPath)
+	repoCreated := time.Unix(int64(repo.CreatedUnix), 0)
+	return getRepoKPIStats(repo.RepoPath(), repoCreated, wikiPath)
 }
 
-func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error) {
+func getRepoKPIStats(repoPath string, repoCreated time.Time, wikiPath string) (*git.RepoKPIStats, error) {
 	stats := &git.RepoKPIStats{}
 
-	contributors, err := git.GetContributors(repoPath)
+	contributors, err := git.GetContributorsDetail(repoPath, repoCreated)
 	if err != nil {
 		return nil, err
 	}
 	timeUntil := time.Now()
 	fourMonthAgo := timeUntil.AddDate(0, -4, 0)
+	if fourMonthAgo.Before(repoCreated) {
+		fourMonthAgo = repoCreated
+	}
 	recentlyContributors, err := git.GetContributorsDetail(repoPath, fourMonthAgo)
 	newContributersDict := make(map[string]struct{})
 	if err != nil {
@@ -44,7 +48,7 @@ func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error
 	if contributors != nil {
 		contributorDistinctDict := make(map[string]int, 0)
 		keyContributorsDict := make(map[string]struct{}, 0)
-
+		var commitsCount int64
 		for _, contributor := range contributors {
 			if strings.Compare(contributor.Email, "") == 0 {
 				continue
@@ -70,6 +74,8 @@ func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error
 				setKeyContributerDict(contributorDistinctDict, contributor.Email, keyContributorsDict)
 			}
 
+			commitsCount += int64(contributor.CommitCnt)
+
 		}
 
 		if recentlyContributors != nil {
@@ -110,10 +116,10 @@ func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error
 
 		stats.Contributors = int64(len(contributorDistinctDict))
 		stats.KeyContributors = int64(len(keyContributorsDict))
-
+		stats.Commits = int64(commitsCount)
 	}
 
-	err = git.SetDevelopAge(repoPath, stats)
+	err = git.SetDevelopAge(repoPath, stats, repoCreated)
 	if err != nil {
 		return nil, fmt.Errorf("FillFromGit: %v", err)
 	}

models/repo_statistic.go

@@ -14,6 +14,7 @@ type RepoStatistic struct {
 	Name                 string  `xorm:"INDEX"  json:"name"`
 	OwnerName            string  `json:"ownerName"`
 	IsPrivate            bool    `json:"isPrivate"`
+	IsMirror             bool    `json:"isMirror"`
 	Date                 string  `xorm:"unique(s) NOT NULL" json:"date"`
 	NumWatches           int64   `xorm:"NOT NULL DEFAULT 0" json:"watch"`
 	NumWatchesAdded      int64   `xorm:"NOT NULL DEFAULT 0" json:"-"`

modules/git/repo_stats_custom.go

@@ -18,6 +18,7 @@ type RepoKPIStats struct {
 	KeyContributors     int64
 	DevelopAge          int64
 	ContributorsAdded   int64
+	Commits             int64
 	CommitsAdded        int64
 	CommitLinesModified int64
 	WikiPages           int64
@@ -35,8 +36,9 @@ type UserKPITypeStats struct {
 	isNewContributor bool //是否是4个月内的新增贡献者
 }
 
-func SetDevelopAge(repoPath string, stats *RepoKPIStats) error {
-	args := []string{"log", "--no-merges", "--branches=*", "--format=%cd", "--date=short"}
+func SetDevelopAge(repoPath string, stats *RepoKPIStats, fromTime time.Time) error {
+	since := fromTime.Format(time.RFC3339)
+	args := []string{"log", "--no-merges", "--branches=*", "--format=%cd", "--date=short", fmt.Sprintf("--since='%s'", since)}
 	stdout, err := NewCommand(args...).RunInDirBytes(repoPath)
 	if err != nil {
 		return err

modules/setting/setting.go

@@ -547,6 +547,7 @@ var (
 		GrowthCommit       float64
 		GrowthComments     float64
 		RecordBeginTime    string
+		IgnoreMirrorRepo   bool
 	}{}
 	
 	Warn_Notify_Mails []string
@@ -1333,6 +1334,7 @@ func SetRadarMapConfig() {
 	RadarMap.GrowthCommit = sec.Key("growth_commit").MustFloat64(0.2)
 	RadarMap.GrowthComments = sec.Key("growth_comments").MustFloat64(0.2)
 	RadarMap.RecordBeginTime = sec.Key("record_beigin_time").MustString("2021-11-05")
+	RadarMap.IgnoreMirrorRepo = sec.Key("ignore_mirror_repo").MustBool(true)
 
 }
 

routers/repo/repo_statistic.go

@@ -49,9 +49,11 @@ func RepoStatisticDaily(date string) {
 	var minRepoRadar models.RepoStatistic
 	var maxRepoRadar models.RepoStatistic
 
-	for i, repo := range repos {
+	isInitMinMaxRadar := false
+
+	for _, repo := range repos {
 		log.Info("start statistic: %s", getDistinctProjectName(repo))
-		var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth int64
+		var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth, numCommits int64
 		repoGitStat, err := models.GetRepoKPIStats(repo)
 		if err != nil {
 			log.Error("GetRepoKPIStats failed: %s", getDistinctProjectName(repo))
@@ -63,6 +65,8 @@ func RepoStatisticDaily(date string) {
 			numCommitsGrowth = repoGitStat.CommitsAdded
 			numCommitLinesGrowth = repoGitStat.CommitLinesModified
 			numContributorsGrowth = repoGitStat.ContributorsAdded
+			numCommits = repoGitStat.Commits
+
 		}
 
 		var issueFixedRate float32
@@ -98,15 +102,15 @@ func RepoStatisticDaily(date string) {
 		}
 
 		repoStat := models.RepoStatistic{
-			RepoID:     repo.ID,
-			Date:       date,
-			Name:       repo.Name,
-			IsPrivate:  repo.IsPrivate,
-			OwnerName:  repo.OwnerName,
-			NumWatches: int64(repo.NumWatches),
-			NumStars:   int64(repo.NumStars),
-			NumForks:   int64(repo.NumForks),
-
+			RepoID:                repo.ID,
+			Date:                  date,
+			Name:                  repo.Name,
+			IsPrivate:             repo.IsPrivate,
+			IsMirror:              repo.IsMirror,
+			OwnerName:             repo.OwnerName,
+			NumWatches:            int64(repo.NumWatches),
+			NumStars:              int64(repo.NumStars),
+			NumForks:              int64(repo.NumForks),
 			NumDownloads:          repo.CloneCnt,
 			NumComments:           numComments,
 			NumVisits:             int64(numVisits),
@@ -117,7 +121,7 @@ func RepoStatisticDaily(date string) {
 			DatasetSize:           datasetSize,
 			NumModels:             0,
 			NumWikiViews:          numWikiViews,
-			NumCommits:            repo.NumCommit,
+			NumCommits:            numCommits,
 			NumIssues:             int64(repo.NumIssues),
 			NumPulls:              int64(repo.NumPulls),
 			IssueFixedRate:        issueFixedRate,
@@ -167,6 +171,7 @@ func RepoStatisticDaily(date string) {
 		tempRepoStat := models.RepoStatistic{
 			RepoID:        repoStat.RepoID,
 			Date:          repoStat.Date,
+			IsMirror:      repoStat.IsMirror,
 			Impact:        normalization.GetImpactInitValue(repoStat.NumWatches, repoStat.NumStars, repoStat.NumForks, repoStat.NumDownloads, repoStat.NumComments, repoStat.NumVisits),
 			Completeness:  normalization.GetCompleteInitValue(repoStat.NumClosedIssues, repoStat.NumVersions, repoStat.NumDevMonths, repoStat.DatasetSize, repoStat.NumModels, repoStat.NumWikiViews),
 			Liveness:      normalization.GetLivenessInitValue(repoStat.NumCommits, repoStat.NumIssues, repoStat.NumPulls, repoStat.NumVisits),
@@ -177,57 +182,64 @@ func RepoStatisticDaily(date string) {
 
 		reposRadar = append(reposRadar, &tempRepoStat)
 
-		if i == 0 {
-			minRepoRadar = tempRepoStat
-			maxRepoRadar = tempRepoStat
-		} else {
+		if !isInitMinMaxRadar {
 
-			if tempRepoStat.Impact < minRepoRadar.Impact {
-				minRepoRadar.Impact = tempRepoStat.Impact
+			if !setting.RadarMap.IgnoreMirrorRepo || (setting.RadarMap.IgnoreMirrorRepo && !tempRepoStat.IsMirror) {
+				minRepoRadar = tempRepoStat
+				maxRepoRadar = tempRepoStat
+				isInitMinMaxRadar = true
 			}
 
-			if tempRepoStat.Impact > maxRepoRadar.Impact {
-				maxRepoRadar.Impact = tempRepoStat.Impact
-			}
+		} else {
+			if !setting.RadarMap.IgnoreMirrorRepo || (setting.RadarMap.IgnoreMirrorRepo && !tempRepoStat.IsMirror) {
+				if tempRepoStat.Impact < minRepoRadar.Impact {
+					minRepoRadar.Impact = tempRepoStat.Impact
+				}
 
-			if tempRepoStat.Completeness < minRepoRadar.Completeness {
-				minRepoRadar.Completeness = tempRepoStat.Completeness
-			}
+				if tempRepoStat.Impact > maxRepoRadar.Impact {
+					maxRepoRadar.Impact = tempRepoStat.Impact
+				}
 
-			if tempRepoStat.Completeness > maxRepoRadar.Completeness {
-				maxRepoRadar.Completeness = tempRepoStat.Completeness
-			}
+				if tempRepoStat.Completeness < minRepoRadar.Completeness {
+					minRepoRadar.Completeness = tempRepoStat.Completeness
+				}
 
-			if tempRepoStat.Liveness < minRepoRadar.Completeness {
-				minRepoRadar.Liveness = tempRepoStat.Liveness
-			}
+				if tempRepoStat.Completeness > maxRepoRadar.Completeness {
+					maxRepoRadar.Completeness = tempRepoStat.Completeness
+				}
 
-			if tempRepoStat.Liveness > maxRepoRadar.Liveness {
-				maxRepoRadar.Liveness = tempRepoStat.Liveness
-			}
+				if tempRepoStat.Liveness < minRepoRadar.Completeness {
+					minRepoRadar.Liveness = tempRepoStat.Liveness
+				}
 
-			if tempRepoStat.ProjectHealth < minRepoRadar.ProjectHealth {
-				minRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
-			}
+				if tempRepoStat.Liveness > maxRepoRadar.Liveness {
+					maxRepoRadar.Liveness = tempRepoStat.Liveness
+				}
 
-			if tempRepoStat.ProjectHealth > maxRepoRadar.ProjectHealth {
-				maxRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
-			}
+				if tempRepoStat.ProjectHealth < minRepoRadar.ProjectHealth {
+					minRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
+				}
 
-			if tempRepoStat.TeamHealth < minRepoRadar.TeamHealth {
-				minRepoRadar.TeamHealth = tempRepoStat.TeamHealth
-			}
+				if tempRepoStat.ProjectHealth > maxRepoRadar.ProjectHealth {
+					maxRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
+				}
 
-			if tempRepoStat.TeamHealth > maxRepoRadar.TeamHealth {
-				maxRepoRadar.TeamHealth = tempRepoStat.TeamHealth
-			}
+				if tempRepoStat.TeamHealth < minRepoRadar.TeamHealth {
+					minRepoRadar.TeamHealth = tempRepoStat.TeamHealth
+				}
 
-			if tempRepoStat.Growth < minRepoRadar.Growth {
-				minRepoRadar.Growth = tempRepoStat.Growth
-			}
+				if tempRepoStat.TeamHealth > maxRepoRadar.TeamHealth {
+					maxRepoRadar.TeamHealth = tempRepoStat.TeamHealth
+				}
+
+				if tempRepoStat.Growth < minRepoRadar.Growth {
+					minRepoRadar.Growth = tempRepoStat.Growth
+				}
+
+				if tempRepoStat.Growth > maxRepoRadar.Growth {
+					maxRepoRadar.Growth = tempRepoStat.Growth
+				}
 
-			if tempRepoStat.Growth > maxRepoRadar.Growth {
-				maxRepoRadar.Growth = tempRepoStat.Growth
 			}
 
 		}
@@ -238,13 +250,23 @@ func RepoStatisticDaily(date string) {
 	//radar  map
 	log.Info("begin statistic radar")
 	for _, radarInit := range reposRadar {
-		radarInit.Impact = normalization.Normalization(radarInit.Impact, minRepoRadar.Impact, maxRepoRadar.Impact)
-		radarInit.Completeness = normalization.Normalization(radarInit.Completeness, minRepoRadar.Completeness, maxRepoRadar.Completeness)
-		radarInit.Liveness = normalization.Normalization(radarInit.Liveness, minRepoRadar.Liveness, maxRepoRadar.Liveness)
-		radarInit.ProjectHealth = normalization.Normalization(radarInit.ProjectHealth, minRepoRadar.ProjectHealth, maxRepoRadar.ProjectHealth)
-		radarInit.TeamHealth = normalization.Normalization(radarInit.TeamHealth, minRepoRadar.TeamHealth, maxRepoRadar.TeamHealth)
-		radarInit.Growth = normalization.Normalization(radarInit.Growth, minRepoRadar.Growth, maxRepoRadar.Growth)
-		radarInit.RadarTotal = normalization.GetRadarValue(radarInit.Impact, radarInit.Completeness, radarInit.Liveness, radarInit.ProjectHealth, radarInit.TeamHealth, radarInit.Growth)
+		if radarInit.IsMirror && setting.RadarMap.IgnoreMirrorRepo {
+			radarInit.Impact = 0
+			radarInit.Completeness = 0
+			radarInit.Liveness = 0
+			radarInit.ProjectHealth = 0
+			radarInit.TeamHealth = 0
+			radarInit.Growth = 0
+			radarInit.RadarTotal = 0
+		} else {
+			radarInit.Impact = normalization.Normalization(radarInit.Impact, minRepoRadar.Impact, maxRepoRadar.Impact)
+			radarInit.Completeness = normalization.Normalization(radarInit.Completeness, minRepoRadar.Completeness, maxRepoRadar.Completeness)
+			radarInit.Liveness = normalization.Normalization(radarInit.Liveness, minRepoRadar.Liveness, maxRepoRadar.Liveness)
+			radarInit.ProjectHealth = normalization.Normalization(radarInit.ProjectHealth, minRepoRadar.ProjectHealth, maxRepoRadar.ProjectHealth)
+			radarInit.TeamHealth = normalization.Normalization(radarInit.TeamHealth, minRepoRadar.TeamHealth, maxRepoRadar.TeamHealth)
+			radarInit.Growth = normalization.Normalization(radarInit.Growth, minRepoRadar.Growth, maxRepoRadar.Growth)
+			radarInit.RadarTotal = normalization.GetRadarValue(radarInit.Impact, radarInit.Completeness, radarInit.Liveness, radarInit.ProjectHealth, radarInit.TeamHealth, radarInit.Growth)
+		}
 		models.UpdateRepoStat(radarInit)
 	}