aiforge/routers/repo/user_data_analysis.go

package repo

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/services/mailer"
	"github.com/360EntSecGroup-Skylar/excelize/v2"
)

func DownloadAllUserData(ctx *context.Context) {
	re, count := models.QueryUserStaticDataAll(pageOpts)
	log.Info("return count=" + fmt.Sprint(count))
	//writer exec file.
	xlsx := excelize.NewFile()
	sheetName := ctx.Tr("user.static.sheetname")
	index := xlsx.NewSheet(sheetName)
	xlsx.DeleteSheet("Sheet1")
	dataHeader := map[string]string{
		"A1": ctx.Tr("user.static.id"),
		"B1": ctx.Tr("user.static.name"),
		"C1": ctx.Tr("user.static.codemergecount"),
		"D1": ctx.Tr("user.static.commitcount"),
		"E1": ctx.Tr("user.static.issuecount"),
		"F1": ctx.Tr("user.static.commentcount"),
		"G1": ctx.Tr("user.static.focusrepocount"),
		"H1": ctx.Tr("user.static.starrepocount"),
		"I1": ctx.Tr("user.static.logincount"),
		"J1": ctx.Tr("user.static.watchedcount"),
		"K1": ctx.Tr("user.static.commitcodesize"),
		"L1": ctx.Tr("user.static.solveissuecount"),
		"M1": ctx.Tr("user.static.encyclopediascount"),
		"N1": ctx.Tr("user.static.createrepocount"),
		"O1": ctx.Tr("user.static.openiindex"),
		"P1": ctx.Tr("user.static.registdate"),
		"Q1": ctx.Tr("user.static.countdate"),
	}
	for k, v := range dataHeader {
		//设置单元格的值
		xlsx.SetCellValue(sheetName, k, v)
	}

	for i, userRecord := range re {
		rows := fmt.Sprint(i + 2)

		xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID)
		xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name)
		xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount)
		xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount)
		xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount)
		xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount)
		xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount)
		xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount)
		xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount)
		xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount)
		xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize)
		xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount)
		xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount)
		xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount)
		xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))

		formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
		xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3])

		formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05")
		xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3])
	}

	//设置默认打开的表单
	xlsx.SetActiveSheet(index)
	var filename string

	filename = sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx"

	ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
	if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
		log.Info("writer exel error." + err.Error())
	}
}

func QueryUserStaticDataPage(ctx *context.Context) {
	startDate := ctx.Query("startDate")
	endDate := ctx.Query("endDate")
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := ctx.QueryInt("pageSize")
	if pageSize <= 0 {
		pageSize = setting.UI.IssuePagingNum
	}
	userName := ctx.Query("userName")
	IsReturnFile := ctx.QueryBool("IsReturnFile")
	log.Info("startDate=" + startDate + "  endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page))
	var startTime time.Time
	var endTime time.Time
	var isAll bool
	if startDate == "all" {
		isAll = true
		startTime = time.Now()
		endTime = time.Now()
	} else {
		startTime, _ = time.ParseInLocation("2006-01-02", startDate, time.Local)
		settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime)
		if startTime.Unix() < settingStartTime.Unix() {
			startTime = settingStartTime
			startDate = settingStartTime.Format("2006-01-02")
		}
		endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local)
		endTime = endTime.AddDate(0, 0, 1)
		isAll = false
		log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
	}

	if IsReturnFile {
		page = -1
		pageSize = -1
	}

	pageOpts := &models.UserBusinessAnalysisQueryOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: pageSize,
		},
		UserName:  userName,
		StartTime: startTime.Unix(),
		EndTime:   endTime.Unix(),
		IsAll:     isAll,
	}

	if IsReturnFile {
		re, count := models.QueryUserStaticDataAll(pageOpts)
		log.Info("return count=" + fmt.Sprint(count))
		//writer exec file.
		xlsx := excelize.NewFile()
		sheetName := ctx.Tr("user.static.sheetname")
		index := xlsx.NewSheet(sheetName)
		xlsx.DeleteSheet("Sheet1")
		dataHeader := map[string]string{
			"A1": ctx.Tr("user.static.id"),
			"B1": ctx.Tr("user.static.name"),
			"C1": ctx.Tr("user.static.codemergecount"),
			"D1": ctx.Tr("user.static.commitcount"),
			"E1": ctx.Tr("user.static.issuecount"),
			"F1": ctx.Tr("user.static.commentcount"),
			"G1": ctx.Tr("user.static.focusrepocount"),
			"H1": ctx.Tr("user.static.starrepocount"),
			"I1": ctx.Tr("user.static.logincount"),
			"J1": ctx.Tr("user.static.watchedcount"),
			"K1": ctx.Tr("user.static.commitcodesize"),
			"L1": ctx.Tr("user.static.solveissuecount"),
			"M1": ctx.Tr("user.static.encyclopediascount"),
			"N1": ctx.Tr("user.static.createrepocount"),
			"O1": ctx.Tr("user.static.openiindex"),
			"P1": ctx.Tr("user.static.registdate"),
			"Q1": ctx.Tr("user.static.countdate"),
		}
		for k, v := range dataHeader {
			//设置单元格的值
			xlsx.SetCellValue(sheetName, k, v)
		}

		for i, userRecord := range re {
			rows := fmt.Sprint(i + 2)

			xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID)
			xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name)
			xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount)
			xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount)
			xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount)
			xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount)
			xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount)
			xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount)
			xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount)
			xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount)
			xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize)
			xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount)
			xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount)
			xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount)
			xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))

			formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
			xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3])

			formatTime = userRecord.DataDate
			xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01")
		}

		//设置默认打开的表单
		xlsx.SetActiveSheet(index)
		var filename string

		nowTime := time.Now()
		nowZeroTime := time.Date(nowTime.Year(), nowTime.Month(), nowTime.Day(), 0, 0, 0, 0, nowTime.Location())
		if endTime.Unix() >= nowZeroTime.Unix() {
			endDate = nowZeroTime.AddDate(0, 0, -1).Format("2006-01-02")
		}

		if isAll {
			filename = sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx"
		} else {
			filename = sheetName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx"
		}

		if len(userName) > 0 {
			filename = sheetName + "_" + userName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx"
		}

		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
		if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
			log.Info("writer exel error." + err.Error())
		}

	} else {
		mapInterface := make(map[string]interface{})
		re, count := models.QueryUserStaticDataPage(pageOpts)
		mapInterface["data"] = re
		mapInterface["count"] = count
		ctx.JSON(http.StatusOK, mapInterface)
	}
}

func TimingCountDataByDateAndReCount(date string, isReCount bool) {

	if date == "refreshAll" {
		models.RefreshUserStaticAllTabel()
		return
	}

	t, _ := time.Parse("2006-01-02", date)
	startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())

	endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location())

	//query wiki data
	log.Info("start to time count data")
	wikiMap := make(map[string]int)
	warnEmailMessage := "用户统计信息入库失败，请尽快定位。"
	repoList, err := models.GetAllRepositories()
	if err != nil {
		log.Error("query repo error." + err.Error())
		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
		return
	}
	log.Info("start to query wiki data")
	for _, repoRecord := range repoList {
		wikiPath := models.WikiPath(repoRecord.OwnerName, repoRecord.Name)
		time, err := git.GetLatestCommitTime(wikiPath)
		if err == nil {
			log.Info("last commit time:" + time.Format("2006-01-02 15:04:05") + " wikiPath=" + wikiPath)
			if time.After(startTime) {
				wikiRepo, _, err := FindWikiRepoCommitByWikiPath(wikiPath)
				if err != nil {
					log.Error("wiki not exist. wikiPath=" + wikiPath)
				} else {
					log.Info("wiki exist, wikiPath=" + wikiPath)
					list, err := wikiRepo.GetCommitByPathAndDays(wikiPath, 1)
					if err != nil {
						log.Info("err,err=v%", err)
					} else {
						for logEntry := list.Front(); logEntry != nil; logEntry = logEntry.Next() {
							commit := logEntry.Value.(*git.Commit)
							log.Info("commit msg=" + commit.CommitMessage + " time=" + commit.Committer.When.Format("2006-01-02 15:04:05") + " user=" + commit.Committer.Name)
							if _, ok := wikiMap[commit.Committer.Name]; !ok {
								wikiMap[commit.Committer.Name] = 1
							} else {
								wikiMap[commit.Committer.Name] += 1
							}
						}
					}

				}
			}
		}
	}
	//other user info data
	err = models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount)
	if err != nil {
		log.Error("count user info error." + err.Error())
		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
	}
}

func TimingCountDataByDate(date string) {
	TimingCountDataByDateAndReCount(date, true)
}

func TimingCountData() {
	log.Info("start to time count data")
	currentTimeNow := time.Now()
	log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05"))
	startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02")
	TimingCountDataByDateAndReCount(startTime, false)
}