package repo import ( "encoding/csv" "fmt" "io/ioutil" "net/http" "net/url" "os" "path" "strconv" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" ) const DEFAULT_PAGE_SIZE = 10 const DATE_FORMAT = "2006-01-02" type ProjectsPeriodData struct { RecordBeginTime string `json:"recordBeginTime"` LastUpdatedTime string `json:"lastUpdatedTime"` PageSize int `json:"pageSize"` TotalPage int `json:"totalPage"` TotalCount int64 `json:"totalCount"` PageRecords []*models.RepoStatistic `json:"pageRecords"` } type UserInfo struct { User string `json:"user"` Mode int `json:"mode"` PR int64 `json:"pr"` Commit int `json:"commit"` RelAvatarLink string `json:"relAvatarLink"` Email string `json:"email"` } type ProjectLatestData struct { RecordBeginTime string `json:"recordBeginTime"` LastUpdatedTime string `json:"lastUpdatedTime"` CreatTime string `json:"creatTime"` OpenI float64 `json:"openi"` Comment int64 `json:"comment"` View int64 `json:"view"` Download int64 `json:"download"` IssueClosedRatio float32 `json:"issueClosedRatio"` Impact float64 `json:"impact"` Completeness float64 `json:"completeness"` Liveness float64 `json:"liveness"` ProjectHealth float64 `json:"projectHealth"` TeamHealth float64 `json:"teamHealth"` Growth float64 `json:"growth"` Description string `json:"description"` Top10 []UserInfo `json:"top10"` } func RestoreForkNumber(ctx *context.Context) { repos, err := models.GetAllRepositories() if err != nil { log.Error("GetAllRepositories failed: %v", err.Error()) return } for _, repo := range repos { models.RestoreRepoStatFork(int64(repo.NumForks), repo.ID) } ctx.JSON(http.StatusOK, struct{}{}) } func GetAllProjectsPeriodStatistics(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) if err != nil { log.Error("Parameter is wrong", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } q := ctx.QueryTrim("q") page := ctx.QueryInt("page") if page <= 0 { page = 1 } pageSize := ctx.QueryInt("pagesize") if pageSize <= 0 { pageSize = DEFAULT_PAGE_SIZE } orderBy := getOrderBy(ctx) latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime() if err != nil { log.Error("Can not query the last updated time.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) return } countSql := generateCountSql(beginTime, endTime, latestDate, q) total, err := models.CountRepoStatByRawSql(countSql) if err != nil { log.Error("Can not query total count.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) return } sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) projectsPeriodData := ProjectsPeriodData{ RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), PageSize: pageSize, TotalPage: getTotalPage(total, pageSize), TotalCount: total, LastUpdatedTime: latestUpdatedTime, PageRecords: models.GetRepoStatisticByRawSql(sql), } ctx.JSON(http.StatusOK, projectsPeriodData) } func generateSqlByType(ctx *context.Context, beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { sql := "" if ctx.QueryTrim("type") == "all" { sql = generateTypeAllSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize) } else { sql = generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize) } return sql } func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) if err != nil { log.Error("Parameter is wrong", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } q := ctx.QueryTrim("q") page := ctx.QueryInt("page") if page <= 0 { page = 1 } pageSize := 1000 orderBy := getOrderBy(ctx) _, latestDate, err := models.GetRepoStatLastUpdatedTime() if err != nil { log.Error("Can not query the last updated time.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) return } countSql := generateCountSql(beginTime, endTime, latestDate, q) total, err := models.CountRepoStatByRawSql(countSql) if err != nil { log.Error("Can not query total count.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) return } fileName, frontName := getFileName(ctx, beginTime, endTime) if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil { ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error()) } totalPage := getTotalPage(total, pageSize) f, e := os.Create(fileName) defer f.Close() if e != nil { log.Warn("Failed to create file", e) } writer := csv.NewWriter(f) writer.Write(allProjectsPeroidHeader(ctx)) for i := 0; i <= totalPage; i++ { pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) for _, record := range pageRecords { e = writer.Write(allProjectsPeroidValues(record, ctx)) if e != nil { log.Warn("Failed to write record", e) } } writer.Flush() } ctx.ServeFile(fileName, url.QueryEscape(frontName)) } func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) (string, string) { baseName := setting.RadarMap.Path + "/项目分析_" if ctx.QueryTrim("q") != "" { baseName = baseName + ctx.QueryTrim("q") + "_" } if ctx.QueryTrim("type") == "all" { baseName = baseName + "所有" } else { baseName = baseName + beginTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(DATE_FORMAT) } frontName := baseName + ".csv" localName := baseName + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv" return localName, path.Base(frontName) } func ClearUnusedStatisticsFile() { fileInfos, err := ioutil.ReadDir(setting.RadarMap.Path) if err != nil { log.Warn("can not read dir: "+setting.RadarMap.Path, err) return } for _, fileInfo := range fileInfos { if !fileInfo.IsDir() && fileInfo.ModTime().Before(time.Now().AddDate(0, 0, -1)) { os.Remove(path.Join(setting.RadarMap.Path, fileInfo.Name())) } } } func allProjectsPeroidHeader(ctx *context.Context) []string { return []string{ctx.Tr("admin.repos.id"), ctx.Tr("admin.repos.projectName"), ctx.Tr("admin.repos.isPrivate"), ctx.Tr("admin.repos.openi"), ctx.Tr("admin.repos.visit"), ctx.Tr("admin.repos.download"), ctx.Tr("admin.repos.pr"), ctx.Tr("admin.repos.commit"), ctx.Tr("admin.repos.watches"), ctx.Tr("admin.repos.stars"), ctx.Tr("admin.repos.forks"), ctx.Tr("admin.repos.issues"), ctx.Tr("admin.repos.closedIssues"), ctx.Tr("admin.repos.contributor")} } func allProjectsPeroidValues(rs *models.RepoStatistic, ctx *context.Context) []string { return []string{strconv.FormatInt(rs.RepoID, 10), constructDistinctName(rs), getIsPrivateDisplay(rs.IsPrivate, ctx), strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), strconv.FormatInt(rs.NumVisits, 10), strconv.FormatInt(rs.NumDownloads, 10), strconv.FormatInt(rs.NumPulls, 10), strconv.FormatInt(rs.NumCommits, 10), strconv.FormatInt(rs.NumWatches, 10), strconv.FormatInt(rs.NumStars, 10), strconv.FormatInt(rs.NumForks, 10), strconv.FormatInt(rs.NumIssues, 10), strconv.FormatInt(rs.NumClosedIssues, 10), strconv.FormatInt(rs.NumContributor, 10), } } func constructDistinctName(rs *models.RepoStatistic) string { if rs.OwnerName == "" { return rs.Name } return rs.OwnerName + "/" + rs.Name } func getIsPrivateDisplay(private bool, ctx *context.Context) string { if private { return ctx.Tr("admin.repos.yes") } else { return ctx.Tr("admin.repos.no") } } func GetProjectLatestStatistics(ctx *context.Context) { repoId := ctx.Params(":id") recordBeginTime, err := getRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime(repoId) repoIdInt, _ := strconv.ParseInt(repoId, 10, 64) repoStat, err := models.GetRepoStatisticByDateAndRepoId(latestDate, repoIdInt) if err != nil { log.Error("Can not get the repo statistics "+repoId, err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.get_repo_stat_error")) return } repository, err := models.GetRepositoryByID(repoIdInt) if err != nil { log.Error("Can not get the repo info "+repoId, err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.get_repo_info_error")) return } projectLatestData := ProjectLatestData{ RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), CreatTime: time.Unix(int64(repository.CreatedUnix), 0).Format(DATE_FORMAT), LastUpdatedTime: latestUpdatedTime, OpenI: repoStat.RadarTotal, Comment: repoStat.NumComments, View: repoStat.NumVisits, Download: repoStat.NumDownloads, IssueClosedRatio: repoStat.IssueFixedRate, Impact: repoStat.Impact, Completeness: repoStat.Completeness, Liveness: repoStat.Liveness, ProjectHealth: repoStat.ProjectHealth, TeamHealth: repoStat.TeamHealth, Growth: repoStat.Growth, Description: repository.Description, } contributors, err := models.GetTop10Contributor(repository.RepoPath()) if err != nil { log.Error("can not get contributors", err) } users := make([]UserInfo, 0) for _, contributor := range contributors { mode := repository.GetCollaboratorMode(contributor.UserId) if mode == -1 { if contributor.IsAdmin { mode = int(models.AccessModeAdmin) } if contributor.UserId == repository.OwnerID { mode = int(models.AccessModeOwner) } } pr := models.GetPullCountByUserAndRepoId(repoIdInt, contributor.UserId) userInfo := UserInfo{ User: contributor.Committer, Commit: contributor.CommitCnt, Mode: mode, PR: pr, RelAvatarLink: contributor.RelAvatarLink, } users = append(users, userInfo) } projectLatestData.Top10 = users ctx.JSON(http.StatusOK, projectLatestData) } func GetProjectPeriodStatistics(ctx *context.Context) { repoId := ctx.Params(":id") recordBeginTime, err := getRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } repoIdInt, _ := strconv.ParseInt(repoId, 10, 64) if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) isOpenI := ctx.QueryBool("openi") var repositorys []*models.RepoStatistic if isOpenI { repositorys = models.GetRepoStatisticByRawSql(generateRadarSql(beginTime, endTime, repoIdInt)) } else { repositorys = models.GetRepoStatisticByRawSql(generateTargetSql(beginTime, endTime, repoIdInt)) } ctx.JSON(http.StatusOK, repositorys) } func generateRadarSql(beginTime time.Time, endTime time.Time, repoId int64) string { sql := "SELECT date, impact, completeness, liveness, project_health, team_health, growth, radar_total FROM repo_statistic" + " where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) return sql } func generateTargetSql(beginTime time.Time, endTime time.Time, repoId int64) string { sql := "SELECT date, num_visits,num_downloads,num_commits FROM repo_statistic" + " where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) return sql } func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, q string) string { countSql := "SELECT count(*) FROM " + "(SELECT repo_id FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { countSql = countSql + " and B.name like '%" + q + "%'" } return countSql } func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { sql := "SELECT A.repo_id,name,owner_name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + "(SELECT repo_id,sum(num_visits) as num_visits " + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + "(SELECT repo_id,name,owner_name,is_private,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { sql = sql + " and name like '%" + q + "%'" } sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) return sql } func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { sql := "SELECT A.repo_id,name,owner_name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + "(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + "(SELECT repo_id,name,owner_name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { sql = sql + " and B.name like '%" + q + "%'" } sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) return sql } func getOrderBy(ctx *context.Context) string { orderBy := "" switch ctx.Query("sort") { case "openi": orderBy = "B.radar_total" case "view": orderBy = "A.num_visits" case "download": orderBy = "A.num_downloads" case "pr": orderBy = "A.num_pulls" case "commit": orderBy = "A.num_commits" case "watch": orderBy = "A.num_watches" case "star": orderBy = "A.num_stars" case "fork": orderBy = "A.num_forks" case "issue": orderBy = "A.num_issues" case "issue_closed": orderBy = "A.num_closed_issues" case "contributor": orderBy = "A.num_contributor" default: orderBy = "B.radar_total" } return orderBy } func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) { queryType := ctx.QueryTrim("type") now := time.Now() recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") var beginTime time.Time var endTime time.Time var err error if queryType != "" { if queryType == "all" { beginTime = recordBeginTimeTemp endTime = now } else if queryType == "yesterday" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) } else if queryType == "current_week" { beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now } else if queryType == "current_month" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) } else if queryType == "monthly" { endTime = now beginTime = now.AddDate(0, -1, 1) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) } else if queryType == "current_year" { endTime = now beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) } else if queryType == "last_month" { lastMonthTime := now.AddDate(0, -1, 0) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) } else { return now, now, fmt.Errorf("The value of type parameter is wrong.") } } else { if beginTimeStr == "" || endTimeStr == "" { //如果查询类型和开始时间结束时间都未设置,按queryType=all处理 beginTime = recordBeginTimeTemp endTime = now } else { beginTime, err = time.Parse("2006-01-02", beginTimeStr) if err != nil { return now, now, err } endTime, err = time.Parse("2006-01-02", endTimeStr) if err != nil { return now, now, err } beginTime = beginTime.AddDate(0, 0, 1) endTime = endTime.AddDate(0, 0, 1) } } if beginTime.Before(recordBeginTimeTemp) { beginTime = recordBeginTimeTemp } return beginTime, endTime, nil } func getRecordBeginTime() (time.Time, error) { return time.Parse(DATE_FORMAT, setting.RadarMap.RecordBeginTime) } func getTotalPage(total int64, pageSize int) int { another := 0 if int(total)%pageSize != 0 { another = 1 } return int(total)/pageSize + another }