aiforge/models/user_business_analysis.go

package models

import (
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"time"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/timeutil"
	"xorm.io/builder"
	"xorm.io/xorm"
)

type UserBusinessAnalysisAll struct {
	ID int64 `xorm:"pk"`

	CountDate int64 `xorm:"pk"`

	//action :ActionMergePullRequest                         // 11
	CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"`

	//action :ActionCommitRepo                               // 5
	CommitCount int `xorm:"NOT NULL DEFAULT 0"`

	//action :ActionCreateIssue                             // 10
	IssueCount int `xorm:"NOT NULL DEFAULT 0"`

	//comment table current date
	CommentCount int `xorm:"NOT NULL DEFAULT 0"`

	//watch table current date
	FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"`

	//star table current date
	StarRepoCount int `xorm:"NOT NULL DEFAULT 0"`

	//follow table
	WatchedCount int `xorm:"NOT NULL DEFAULT 0"`

	// user table
	GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"`

	//
	CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"`

	//attachement table
	CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"`

	//0
	CommitModelCount int `xorm:"NOT NULL DEFAULT 0"`

	//issue, issueassignees
	SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"`

	//baike
	EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"`

	//user
	RegistDate timeutil.TimeStamp `xorm:"NOT NULL"`

	//repo
	CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"`

	//login count, from elk
	LoginCount int `xorm:"NOT NULL DEFAULT 0"`

	//openi index
	OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"`

	//user
	Email string `xorm:"NOT NULL"`

	//user
	Name string `xorm:"NOT NULL"`

	DataDate string `xorm:"NULL"`
}

type UserBusinessAnalysis struct {
	ID int64 `xorm:"pk"`

	CountDate int64 `xorm:"pk"`

	//action :ActionMergePullRequest                         // 11
	CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"`

	//action :ActionCommitRepo                               // 5
	CommitCount int `xorm:"NOT NULL DEFAULT 0"`

	//action :ActionCreateIssue                             // 10
	IssueCount int `xorm:"NOT NULL DEFAULT 0"`

	//comment table current date
	CommentCount int `xorm:"NOT NULL DEFAULT 0"`

	//watch table current date
	FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"`

	//star table current date
	StarRepoCount int `xorm:"NOT NULL DEFAULT 0"`

	//follow table
	WatchedCount int `xorm:"NOT NULL DEFAULT 0"`

	// user table
	GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"`

	//
	CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"`

	//attachement table
	CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"`

	//0
	CommitModelCount int `xorm:"NOT NULL DEFAULT 0"`

	//issue, issueassignees
	SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"`

	//baike
	EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"`

	//user
	RegistDate timeutil.TimeStamp `xorm:"NOT NULL"`

	//repo
	CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"`

	//login count, from elk
	LoginCount int `xorm:"NOT NULL DEFAULT 0"`

	//openi index
	OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"`

	//user
	Email string `xorm:"NOT NULL"`

	//user
	Name string `xorm:"NOT NULL"`

	DataDate string `xorm:"NULL"`
}

type UserBusinessAnalysisQueryOptions struct {
	ListOptions
	UserName  string
	SortType  string
	StartTime int64
	EndTime   int64
	IsAll     bool
}

type UserBusinessAnalysisList []*UserBusinessAnalysis

func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] }
func (ulist UserBusinessAnalysisList) Len() int      { return len(ulist) }
func (ulist UserBusinessAnalysisList) Less(i, j int) bool {
	return ulist[i].ID > ulist[j].ID
}

type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll

func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] }
func (ulist UserBusinessAnalysisAllList) Len() int      { return len(ulist) }
func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool {
	return ulist[i].ID > ulist[j].ID
}

func getLastCountDate() int64 {
	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()
	statictisSess.Limit(1, 0)
	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
	if err := statictisSess.Table("user_business_analysis").OrderBy("count_date desc").Limit(1, 0).
		Find(&userBusinessAnalysisList); err == nil {
		for _, userRecord := range userBusinessAnalysisList {
			return userRecord.CountDate - 10000
		}
	} else {
		log.Info("query error." + err.Error())
	}
	currentTimeNow := time.Now()
	pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
	return pageStartTime.Unix()
}

func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) {
	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll))

	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()

	allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll))
	if err != nil {
		log.Info("query error." + err.Error())
		return nil, 0
	}
	log.Info("query return total:" + fmt.Sprint(allCount))
	pageSize := 1000
	totalPage := int(allCount) / pageSize
	userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{}
	for i := 0; i <= int(totalPage); i++ {
		userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0)
		if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize).
			Find(&userBusinessAnalysisAllList); err != nil {
			return nil, 0
		}
		log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList)))
		for _, userRecord := range userBusinessAnalysisAllList {
			userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord)
		}
	}

	sort.Sort(userBusinessAnalysisReturnList)
	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
	return userBusinessAnalysisReturnList, allCount
}

func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) {

	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll))
	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()

	currentTimeNow := time.Now()
	pageStartTime := getLastCountDate()
	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix()

	var cond = builder.NewCond()
	if len(opts.UserName) > 0 {
		cond = cond.And(
			builder.Like{"name", opts.UserName},
		)
	}
	cond = cond.And(
		builder.Gte{"count_date": pageStartTime},
	)
	cond = cond.And(
		builder.Lte{"count_date": pageEndTime},
	)

	count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis))
	if err != nil {
		log.Info("query error." + err.Error())
		return nil, 0
	}

	if opts.Page >= 0 && opts.PageSize > 0 {
		var start int
		if opts.Page == 0 {
			start = 0
		} else {
			start = (opts.Page - 1) * opts.PageSize
		}
		statictisSess.Limit(opts.PageSize, start)
	}

	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
	if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc").
		Find(&userBusinessAnalysisList); err != nil {
		return nil, 0
	}

	resultMap := make(map[int64]*UserBusinessAnalysis)

	if len(userBusinessAnalysisList) > 0 {
		var newAndCond = builder.NewCond()
		var newOrCond = builder.NewCond()
		for _, userRecord := range userBusinessAnalysisList {
			newOrCond = newOrCond.Or(
				builder.Eq{"id": userRecord.ID},
			)
		}
		newAndCond = newAndCond.And(
			newOrCond,
		)
		if !opts.IsAll {
			newAndCond = newAndCond.And(
				builder.Gte{"count_date": opts.StartTime},
			)
			newAndCond = newAndCond.And(
				builder.Lte{"count_date": opts.EndTime},
			)
		}

		allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis))
		if err != nil {
			log.Info("query error." + err.Error())
			return nil, 0
		}

		pageSize := 1000
		totalPage := int(allCount) / pageSize

		for i := 0; i <= int(totalPage); i++ {
			userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0)
			if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize).
				Find(&userBusinessAnalysisList); err != nil {
				return nil, 0
			}
			log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
			for _, userRecord := range userBusinessAnalysisList {
				if _, ok := resultMap[userRecord.ID]; !ok {
					resultMap[userRecord.ID] = userRecord
				} else {
					resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
					resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
					resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
					resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
					resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
					resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
					resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
					resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
					resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
					resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
					resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
					resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
					resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
					resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
				}
			}
		}
	}

	userBusinessAnalysisReturnList := UserBusinessAnalysisList{}
	for _, v := range resultMap {
		userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v)
	}
	sort.Sort(userBusinessAnalysisReturnList)
	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
	return userBusinessAnalysisReturnList, count
}

func RefreshUserStaticAllTabel() {

	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()

	statictisSess.Exec("delete from user_business_analysis_all")

	currentTimeNow := time.Now()
	pageStartTime := getLastCountDate()
	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix()

	var cond = builder.NewCond()
	cond = cond.And(
		builder.Gte{"count_date": pageStartTime},
	)
	cond = cond.And(
		builder.Lte{"count_date": pageEndTime},
	)
	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
	if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc").
		Find(&userBusinessAnalysisList); err != nil {
		return
	}
	for _, userRecord := range userBusinessAnalysisList {
		log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID))
		allData := getAllData(userRecord.ID, statictisSess)
		allData.ID = userRecord.ID
		allData.CountDate = 0
		allData.DataDate = userRecord.DataDate
		allData.Email = userRecord.Email
		allData.OpenIIndex = userRecord.OpenIIndex
		allData.GiteaAgeMonth = userRecord.GiteaAgeMonth
		allData.Name = userRecord.Name
		allData.RegistDate = userRecord.RegistDate

		_, err := statictisSess.Insert(&allData)
		if err != nil {
			log.Info("insert all data failed." + err.Error())
		}
	}
}

func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error {

	log.Info("start to count other user info data")
	sess := x.NewSession()
	defer sess.Close()
	sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true")
	userList := make([]*User, 0)
	sess.Find(&userList)

	currentTimeNow := time.Now()
	log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05"))

	//yesterday := currentTimeNow.AddDate(0, 0, -1)
	//startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location())
	start_unix := startTime.Unix()
	log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05"))

	//endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
	end_unix := endTime.Unix()
	CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location())
	if isReCount {
		CountDate = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 1, 0, 0, currentTimeNow.Location())
	}

	DataDate := startTime.Format("2006-01-02")
	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
	CommitCountMap := queryAction(start_unix, end_unix, 5)
	IssueCountMap := queryAction(start_unix, end_unix, 6)

	CommentCountMap := queryComment(start_unix, end_unix)
	FocusRepoCountMap := queryWatch(start_unix, end_unix)
	StarRepoCountMap := queryStar(start_unix, end_unix)
	WatchedCountMap := queryFollow(start_unix, end_unix)

	CommitCodeSizeMap, err := GetAllUserKPIStats()
	if err != nil {
		log.Info("query commit code errr.")
	} else {
		log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap)))
	}
	CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix)
	SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
	CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix)
	LoginCountMap := queryLoginCount(start_unix, end_unix)
	OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)

	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()

	for i, userRecord := range userList {
		var dateRecord UserBusinessAnalysis
		dateRecord.ID = userRecord.ID
		log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name)
		dateRecord.CountDate = CountDate.Unix()

		statictisSess.Delete(&dateRecord)

		dateRecord.Email = userRecord.Email
		dateRecord.RegistDate = userRecord.CreatedUnix
		dateRecord.Name = userRecord.Name
		dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime())
		dateRecord.DataDate = DataDate
		if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok {
			dateRecord.CodeMergeCount = 0
		} else {
			dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID]
		}

		if _, ok := CommitCountMap[dateRecord.ID]; !ok {
			dateRecord.CommitCount = 0
		} else {
			dateRecord.CommitCount = CommitCountMap[dateRecord.ID]
		}

		if _, ok := IssueCountMap[dateRecord.ID]; !ok {
			dateRecord.IssueCount = 0
		} else {
			dateRecord.IssueCount = IssueCountMap[dateRecord.ID]
		}

		if _, ok := CommentCountMap[dateRecord.ID]; !ok {
			dateRecord.CommentCount = 0
		} else {
			dateRecord.CommentCount = CommentCountMap[dateRecord.ID]
		}

		if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok {
			dateRecord.FocusRepoCount = 0
		} else {
			dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID]
		}

		if _, ok := StarRepoCountMap[dateRecord.ID]; !ok {
			dateRecord.StarRepoCount = 0
		} else {
			dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID]
		}

		if _, ok := WatchedCountMap[dateRecord.ID]; !ok {
			dateRecord.WatchedCount = 0
		} else {
			dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID]
		}

		if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok {
			dateRecord.CommitCodeSize = 0
		} else {
			dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines)
		}

		if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok {
			dateRecord.CommitDatasetSize = 0
		} else {
			dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID]
		}

		if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok {
			dateRecord.SolveIssueCount = 0
		} else {
			dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID]
		}

		if _, ok := wikiCountMap[dateRecord.Name]; !ok {
			dateRecord.EncyclopediasCount = 0
		} else {
			dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name]
		}

		if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok {
			dateRecord.CreateRepoCount = 0
		} else {
			dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID]
		}

		if _, ok := LoginCountMap[dateRecord.ID]; !ok {
			dateRecord.LoginCount = 0
		} else {
			dateRecord.LoginCount = LoginCountMap[dateRecord.ID]
		}

		if _, ok := OpenIIndexMap[dateRecord.ID]; !ok {
			dateRecord.OpenIIndex = 0
		} else {
			dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID]
		}

		dateRecord.CommitModelCount = 0

		_, err = statictisSess.Insert(&dateRecord)
		if err != nil {
			log.Info("insert daterecord failed." + err.Error())
			return err
		}

		if isExistUserInAllTable(dateRecord.ID, statictisSess) {
			updateCurrentData(dateRecord.ID, statictisSess, dateRecord)
		} else {
			log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID))
			allData := getAllData(dateRecord.ID, statictisSess)
			allData.ID = dateRecord.ID
			allData.CountDate = 0
			allData.DataDate = dateRecord.DataDate
			allData.Email = dateRecord.Email
			allData.OpenIIndex = dateRecord.OpenIIndex
			allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth
			allData.Name = dateRecord.Name
			allData.RegistDate = dateRecord.RegistDate

			_, err = statictisSess.Insert(&allData)
			if err != nil {
				log.Info("insert all data failed." + err.Error())
				return err
			}
		}

	}
	return nil
}

func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) {

	_, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) +
		",commit_count+=" + fmt.Sprint(currentData.CommitCount) +
		",issue_count+=" + fmt.Sprint(currentData.IssueCount) +
		",comment_count+=" + fmt.Sprint(currentData.CommentCount) +
		",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) +
		",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) +
		",watched_count+=" + fmt.Sprint(currentData.WatchedCount) +
		",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) +
		",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) +
		",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) +
		",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) +
		",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) +
		",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) +
		",login_count+=" + fmt.Sprint(currentData.LoginCount) +
		" where id=" + fmt.Sprint(userId))

	if err != nil {
		log.Info("update table failed." + err.Error())
	}

}

func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool {

	allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll))
	if err != nil {
		return false
	}
	return allCount > 0
}

func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll {
	var dateRecord UserBusinessAnalysisAll

	rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id")
	if err == nil {
		for i, row := range rows {
			log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"]))
			dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"]))
			dateRecord.CommitCount = getInt(string(row["commit_count"]))
			dateRecord.IssueCount = getInt(string(row["issue_count"]))
			dateRecord.CommentCount = getInt(string(row["comment_count"]))
			dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"]))
			dateRecord.StarRepoCount = getInt(string(row["star_repo_count"]))
			dateRecord.WatchedCount = getInt(string(row["watched_count"]))
			dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"]))
			dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"]))
			dateRecord.CommitModelCount = getInt(string(row["commit_model_count"]))
			dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"]))
			dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"]))
			dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"]))
			dateRecord.LoginCount = getInt(string(row["login_count"]))
		}
	}
	return dateRecord
}

func getInt(str string) int {
	re, err := strconv.ParseInt(str, 10, 32)
	if err != nil {
		return 0
	}
	return int(re)
}

func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
	CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false)
}

func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
	//select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id
	sess := x.NewSession()
	defer sess.Close()
	sess.Select("issue_assignees.*").Table("issue_assignees").
		Join("inner", "issue", "issue.id=issue_assignees.issue_id").
		Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix))
	issueAssigneesList := make([]*IssueAssignees, 0)
	sess.Find(&issueAssigneesList)
	resultMap := make(map[int64]int)
	log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList)))
	for _, issueAssigneesRecord := range issueAssigneesList {
		if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok {
			resultMap[issueAssigneesRecord.AssigneeID] = 1
		} else {
			resultMap[issueAssigneesRecord.AssigneeID] += 1
		}
	}
	return resultMap

}

func queryPullRequest(start_unix int64, end_unix int64) map[int64]int {
	sess := x.NewSession()
	defer sess.Close()

	sess.Select("issue.*").Table("issue").
		Join("inner", "pull_request", "issue.id=pull_request.issue_id").
		Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix))

	issueList := make([]*Issue, 0)
	sess.Find(&issueList)
	resultMap := make(map[int64]int)
	log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList)))
	for _, issueRecord := range issueList {
		if _, ok := resultMap[issueRecord.PosterID]; !ok {
			resultMap[issueRecord.PosterID] = 1
		} else {
			resultMap[issueRecord.PosterID] += 1
		}
	}
	return resultMap
}

func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int {
	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	actionList := make([]*Action, 0)
	sess.Find(&actionList)
	resultMap := make(map[int64]int)
	log.Info("query action size=" + fmt.Sprint(len(actionList)))
	for _, actionRecord := range actionList {
		if _, ok := resultMap[actionRecord.UserID]; !ok {
			resultMap[actionRecord.UserID] = 1
		} else {
			resultMap[actionRecord.UserID] += 1
		}
	}
	return resultMap
}

func queryComment(start_unix int64, end_unix int64) map[int64]int {

	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	commentList := make([]*Comment, 0)
	sess.Find(&commentList)
	resultMap := make(map[int64]int)
	log.Info("query Comment size=" + fmt.Sprint(len(commentList)))
	for _, commentRecord := range commentList {
		if _, ok := resultMap[commentRecord.PosterID]; !ok {
			resultMap[commentRecord.PosterID] = 1
		} else {
			resultMap[commentRecord.PosterID] += 1
		}
	}
	return resultMap
}

func queryWatch(start_unix int64, end_unix int64) map[int64]int {

	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	watchList := make([]*Watch, 0)
	sess.Find(&watchList)
	resultMap := make(map[int64]int)
	log.Info("query Watch size=" + fmt.Sprint(len(watchList)))
	for _, watchRecord := range watchList {
		if _, ok := resultMap[watchRecord.UserID]; !ok {
			resultMap[watchRecord.UserID] = 1
		} else {
			resultMap[watchRecord.UserID] += 1
		}
	}
	return resultMap

}

func queryStar(start_unix int64, end_unix int64) map[int64]int {

	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	starList := make([]*Star, 0)
	sess.Find(&starList)
	resultMap := make(map[int64]int)
	log.Info("query Star size=" + fmt.Sprint(len(starList)))
	for _, starRecord := range starList {
		if _, ok := resultMap[starRecord.UID]; !ok {
			resultMap[starRecord.UID] = 1
		} else {
			resultMap[starRecord.UID] += 1
		}
	}
	return resultMap

}

func queryFollow(start_unix int64, end_unix int64) map[int64]int {

	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	followList := make([]*Follow, 0)
	sess.Find(&followList)
	resultMap := make(map[int64]int)
	log.Info("query Follow size=" + fmt.Sprint(len(followList)))
	for _, followRecord := range followList {
		if _, ok := resultMap[followRecord.FollowID]; !ok {
			resultMap[followRecord.FollowID] = 1
		} else {
			resultMap[followRecord.FollowID] += 1
		}
	}
	return resultMap
}

func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {
	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	attachmentList := make([]*Attachment, 0)
	sess.Find(&attachmentList)
	resultMap := make(map[int64]int)
	log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList)))
	for _, attachRecord := range attachmentList {
		if _, ok := resultMap[attachRecord.UploaderID]; !ok {
			resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB
		} else {
			resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB
		}
	}
	return resultMap

}

func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int {
	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	repoList := make([]*Repository, 0)
	sess.Find(&repoList)
	resultMap := make(map[int64]int)
	log.Info("query Repository size=" + fmt.Sprint(len(repoList)))
	for _, repoRecord := range repoList {
		if _, ok := resultMap[repoRecord.OwnerID]; !ok {
			resultMap[repoRecord.OwnerID] = 1
		} else {
			resultMap[repoRecord.OwnerID] += 1
		}
	}
	return resultMap
}

func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 {
	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()
	statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	repoStatisticList := make([]*RepoStatistic, 0)
	statictisSess.Find(&repoStatisticList)
	repoOpenIIndexMap := make(map[int64]float64)
	log.Info("query repo_statistic size=" + fmt.Sprint(len(repoStatisticList)))
	for _, repoRecord := range repoStatisticList {
		if _, ok := repoOpenIIndexMap[repoRecord.RepoID]; !ok {
			repoOpenIIndexMap[repoRecord.RepoID] = repoRecord.RadarTotal
		}
	}

	sess := x.NewSession()
	defer sess.Close()
	sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false")
	repoList := make([]*Repository, 0)
	sess.Find(&repoList)

	userMap := make(map[int64]float64)

	log.Info("query Repository size=" + fmt.Sprint(len(repoList)))
	for _, repoRecord := range repoList {
		if _, ok := userMap[repoRecord.OwnerID]; !ok {
			if _, ok := repoOpenIIndexMap[repoRecord.ID]; ok {
				userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID]
			}
		}
	}

	//query collaboration
	sess.Select("repo_id,user_id,mode").Table("collaboration")
	collaborationList := make([]*Collaboration, 0)
	sess.Find(&collaborationList)

	log.Info("query collaborationList size=" + fmt.Sprint(len(collaborationList)))

	for _, collaborationRecord := range collaborationList {
		if _, ok := userMap[collaborationRecord.UserID]; !ok {
			if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok {
				userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID]
			}
		} else {
			if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok {
				userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID]
			}
		}
	}

	userMapJson, _ := json.Marshal(userMap)
	log.Info("userMapJson=" + string(userMapJson))

	return userMap
}

func queryLoginCount(start_unix int64, end_unix int64) map[int64]int {
	statictisSess := xStatistic.NewSession()
	defer statictisSess.Close()
	statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
	userLoginLogList := make([]*UserLoginLog, 0)
	statictisSess.Find(&userLoginLogList)
	resultMap := make(map[int64]int)
	log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList)))
	for _, loginRecord := range userLoginLogList {
		if _, ok := resultMap[loginRecord.UId]; !ok {
			resultMap[loginRecord.UId] = 1
		} else {
			resultMap[loginRecord.UId] += 1
		}
	}
	return resultMap
}

func subMonth(t1, t2 time.Time) (month int) {
	y1 := t1.Year()
	y2 := t2.Year()
	m1 := int(t1.Month())
	m2 := int(t2.Month())
	d1 := t1.Day()
	d2 := t2.Day()

	yearInterval := y1 - y2
	// 如果 d1的 月-日 小于 d2的 月-日 那么 yearInterval-- 这样就得到了相差的年数
	if m1 < m2 || m1 == m2 && d1 < d2 {
		yearInterval--
	}
	// 获取月数差值
	monthInterval := (m1 + 12) - m2
	if d1 < d2 {
		monthInterval--
	}
	monthInterval %= 12
	month = yearInterval*12 + monthInterval
	if month == 0 {
		month = 1
	}
	return month
}