diff --git a/models/models.go b/models/models.go index 0cdcee4fd..696d0949b 100755 --- a/models/models.go +++ b/models/models.go @@ -139,6 +139,7 @@ func init() { new(RepoStatistic), new(SummaryStatistic), new(UserBusinessAnalysis), + new(UserLoginLog), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index f6cbf0331..9cb7e4a09 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -1,13 +1,115 @@ package models -import "code.gitea.io/gitea/modules/git" +import ( + "fmt" + "strings" + "time" + + "code.gitea.io/gitea/modules/git" +) func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) { wikiPath := "" if repo.HasWiki() { wikiPath = repo.WikiPath() } - return git.GetRepoKPIStats(repo.RepoPath(), wikiPath) + return getRepoKPIStats(repo.RepoPath(), wikiPath) +} + +func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error) { + stats := &git.RepoKPIStats{} + + contributors, err := git.GetContributors(repoPath) + if err != nil { + return nil, err + } + timeUntil := time.Now() + fourMonthAgo := timeUntil.AddDate(0, -4, 0) + recentlyContributors, err := git.GetContributorsDetail(repoPath, fourMonthAgo) + newContributersDict := make(map[string]struct{}) + if err != nil { + return nil, err + } + + if contributors != nil { + contributorDistinctDict := make(map[string]int, 0) + keyContributorsDict := make(map[string]struct{}, 0) + + for _, contributor := range contributors { + if strings.Compare(contributor.Email, "") == 0 { + continue + } + + user, err := GetUserByActivateEmail(contributor.Email) + if err == nil { + value, ok := contributorDistinctDict[user.Email] + if !ok { + contributorDistinctDict[user.Email] = contributor.CommitCnt + } else { + contributorDistinctDict[user.Email] = value + contributor.CommitCnt + } + setKeyContributerDict(contributorDistinctDict, user.Email, keyContributorsDict) + + } else { + value, ok := contributorDistinctDict[contributor.Email] + if !ok { + contributorDistinctDict[contributor.Email] = contributor.CommitCnt + } else { + contributorDistinctDict[contributor.Email] = value + contributor.CommitCnt + } + setKeyContributerDict(contributorDistinctDict, contributor.Email, keyContributorsDict) + } + + } + + if recentlyContributors != nil { + for _, recentlyContributor := range recentlyContributors { + + user, err := GetUserByActivateEmail(recentlyContributor.Email) + var ok bool + if err == nil { + _, ok = contributorDistinctDict[user.Email] + } else { + _, ok = contributorDistinctDict[recentlyContributor.Email] + } + + if !ok { + stats.ContributorsAdded++ + newContributersDict[recentlyContributor.Email] = struct{}{} + } + + } + } + + stats.Contributors = int64(len(contributorDistinctDict)) + stats.KeyContributors = int64(len(keyContributorsDict)) + + } + + err = git.SetDevelopAge(repoPath, stats) + if err != nil { + return nil, fmt.Errorf("FillFromGit: %v", err) + } + err = git.SetRepoKPIStats(repoPath, fourMonthAgo, stats, newContributersDict) + + if err != nil { + return nil, fmt.Errorf("FillFromGit: %v", err) + } + + git.SetWikiPages(wikiPath, stats) + return stats, nil + +} + +func setKeyContributerDict(contributorDistinctDict map[string]int, email string, keyContributorsDict map[string]struct{}) { + if contributorDistinctDict[email] >= 3 { + _, ok := keyContributorsDict[email] + if !ok { + keyContributorsDict[email] = struct{}{} + + } + + } } func GetAllUserKPIStats() (map[string]*git.UserKPIStats, error) { diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 8b7691677..bb6726a2c 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -114,7 +114,7 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis return userBusinessAnalysisReturnList } -func CountData(wikiCountMap map[string]int) { +func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() @@ -125,12 +125,12 @@ func CountData(wikiCountMap map[string]int) { currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - yesterday := currentTimeNow.AddDate(0, 0, -1) - startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) + //yesterday := currentTimeNow.AddDate(0, 0, -1) + //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) @@ -153,12 +153,19 @@ func CountData(wikiCountMap map[string]int) { CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() for i, userRecord := range userList { var dateRecord UserBusinessAnalysis dateRecord.ID = userRecord.ID log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + dateRecord.Email = userRecord.Email dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name @@ -235,10 +242,14 @@ func CountData(wikiCountMap map[string]int) { dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] } + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } + dateRecord.CommitModelCount = 0 - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() statictisSess.Insert(&dateRecord) } @@ -421,6 +432,24 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + return resultMap +} + func subMonth(t1, t2 time.Time) (month int) { y1 := t1.Year() y2 := t2.Year() diff --git a/models/user_login_log.go b/models/user_login_log.go new file mode 100644 index 000000000..4a499d527 --- /dev/null +++ b/models/user_login_log.go @@ -0,0 +1,34 @@ +package models + +import ( + "net/http" + + "code.gitea.io/gitea/modules/timeutil" +) + +type UserLoginLog struct { + ID int64 `xorm:"pk autoincr"` + UId int64 `xorm:"NOT NULL"` + IpAddr string `xorm:"default NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` +} + +func SaveLoginInfoToDb(r *http.Request, u *User) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + var dateRecord UserLoginLog + + dateRecord.UId = u.ID + dateRecord.IpAddr = getIP(r) + + statictisSess.Insert(&dateRecord) +} + +func getIP(r *http.Request) string { + forwarded := r.Header.Get("X-FORWARDED-FOR") + if forwarded != "" { + return forwarded + } + return r.RemoteAddr +} diff --git a/modules/git/repo_stats_custom.go b/modules/git/repo_stats_custom.go index f7556d5c2..5d99bd8af 100644 --- a/modules/git/repo_stats_custom.go +++ b/modules/git/repo_stats_custom.go @@ -35,58 +35,7 @@ type UserKPITypeStats struct { isNewContributor bool //是否是4个月内的新增贡献者 } -func GetRepoKPIStats(repoPath string, wikiPath string) (*RepoKPIStats, error) { - stats := &RepoKPIStats{} - - contributors, err := GetContributors(repoPath) - if err != nil { - return nil, err - } - timeUntil := time.Now() - fourMonthAgo := timeUntil.AddDate(0, -4, 0) - recentlyContributors, err := getContributors(repoPath, fourMonthAgo) - newContributersDict := make(map[string]struct{}) - if err != nil { - return nil, err - } - - if contributors != nil { - stats.Contributors = int64(len(contributors)) - for _, contributor := range contributors { - if contributor.CommitCnt >= 3 { - stats.KeyContributors++ - } - - if recentlyContributors != nil { - for _, recentlyContributor := range recentlyContributors { - if recentlyContributor.Email == contributor.Email && recentlyContributor.CommitCnt == contributor.CommitCnt { - stats.ContributorsAdded++ - newContributersDict[recentlyContributor.Email] = struct{}{} - } - - } - } - - } - - } - - err = setDevelopAge(repoPath, stats) - if err != nil { - return nil, fmt.Errorf("FillFromGit: %v", err) - } - err = setRepoKPIStats(repoPath, fourMonthAgo, stats, newContributersDict) - - if err != nil { - return nil, fmt.Errorf("FillFromGit: %v", err) - } - - setWikiPages(wikiPath, stats) - return stats, nil - -} - -func setDevelopAge(repoPath string, stats *RepoKPIStats) error { +func SetDevelopAge(repoPath string, stats *RepoKPIStats) error { args := []string{"log", "--no-merges", "--branches=*", "--format=%cd", "--date=short"} stdout, err := NewCommand(args...).RunInDirBytes(repoPath) if err != nil { @@ -173,7 +122,7 @@ func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { } -func setRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { +func SetRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { since := fromTime.Format(time.RFC3339) args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} @@ -259,7 +208,7 @@ func setRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, n } -func getContributors(repoPath string, fromTime time.Time) ([]Contributor, error) { +func GetContributorsDetail(repoPath string, fromTime time.Time) ([]Contributor, error) { since := fromTime.Format(time.RFC3339) cmd := NewCommand("shortlog", "-sne", "--all", fmt.Sprintf("--since='%s'", since)) stdout, err := cmd.RunInDir(repoPath) @@ -289,7 +238,7 @@ func getContributors(repoPath string, fromTime time.Time) ([]Contributor, error) return nil, nil } -func setWikiPages(wikiPath string, stats *RepoKPIStats) { +func SetWikiPages(wikiPath string, stats *RepoKPIStats) { wikiPages := 0 if wikiPath == "" { diff --git a/modules/normalization/normalization.go b/modules/normalization/normalization.go index f651cb5f1..c2b545f90 100644 --- a/modules/normalization/normalization.go +++ b/modules/normalization/normalization.go @@ -34,7 +34,7 @@ func GetImpactInitValue(watch int64, star int64, fork int64, download int64, com setting.RadarMap.ImpactFork*float64(fork) + setting.RadarMap.ImpactCodeDownload*float64(download)*0.001 + setting.RadarMap.ImpactComments*float64(comments) + - setting.RadarMap.ImpactBrowser*float64(browser) + setting.RadarMap.ImpactBrowser*float64(browser)*0.001 } diff --git a/public/self/labelTaskPage.js b/public/self/labelTaskPage.js index 03acd6df1..a68dfaadc 100644 --- a/public/self/labelTaskPage.js +++ b/public/self/labelTaskPage.js @@ -41,7 +41,7 @@ function setDataSetTask(){ //dislpayUser(); getLabelPropertyTask(); displayLabelPropertyTask(); - + dataset_sele_Change(""); $(".ui.dataset.modal").modal("show"); } @@ -132,6 +132,7 @@ function setPredictTask(){ get_model_list(); displayModelTask(); + dataset_auto_sele_Change(""); $(".ui.predict.modal").modal("show"); } @@ -197,13 +198,13 @@ function sele_export_Change(sele){ function dataset_sele_Change(sele){ var dataset_listName = $('#dataset_list option:selected').text(); console.log("select dataset_list =" + dataset_listName); - $("#datasetlabeltaskname").attr({value:dataset_listName+"-人工标注"}); + $("#datasetlabeltaskname").val(dataset_listName+"-人工标注"); } function dataset_auto_sele_Change(sele){ var dataset_listName = $('#dataset_list_auto option:selected').text(); console.log("select dataset_list_auto =" + dataset_listName); - $("#autolabeltaskname").attr({value:dataset_listName+"-自动标注"}); + $("#autolabeltaskname").val(dataset_listName+"-自动标注"); } diff --git a/routers/private/tool.go b/routers/private/tool.go index a7a7bee9d..b93f17090 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -41,4 +41,6 @@ func UpdateAllRepoCommitCnt(ctx *macaron.Context) { func RepoStatisticManually(ctx *macaron.Context) { date := ctx.Query("date") repo.RepoStatisticDaily(date) + repo.SummaryStatisticDaily(date) + repo.TimingCountDataByDate(date) } diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 3c54f982e..b6661433e 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -278,7 +278,21 @@ func GetAttachment(ctx *context.Context) { ctx.ServerError("Update", err) return } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + if dataSet != nil { + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + } else { + fr, err := storage.Attachments.Open(attach.RelativePath()) + if err != nil { + ctx.ServerError("Open", err) + return + } + defer fr.Close() + if err = ServeData(ctx, attach.Name, fr); err != nil { + ctx.ServerError("ServeData", err) + return + } + } + } else { fr, err := storage.Attachments.Open(attach.RelativePath()) if err != nil { diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 3260780ac..7dc7af321 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -8,16 +8,17 @@ import ( "code.gitea.io/gitea/modules/log" ) -func TimingCountData() { +func TimingCountDataByDate(date string) { + + t, _ := time.Parse("2006-01-02", date) + startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + + endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location()) + //query wiki data log.Info("start to time count data") wikiMap := make(map[string]int) - currentTimeNow := time.Now() - log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - - yesterday := currentTimeNow.AddDate(0, 0, -1) - repoList, err := models.GetAllRepositories() if err != nil { log.Error("query repo error.") @@ -29,7 +30,7 @@ func TimingCountData() { time, err := git.GetLatestCommitTime(wikiPath) if err == nil { log.Info("last commit time:" + time.Format("2006-01-02 15:04:05") + " wikiPath=" + wikiPath) - if time.After(yesterday) { + if time.After(startTime) { wikiRepo, _, err := FindWikiRepoCommitByWikiPath(wikiPath) if err != nil { log.Error("wiki not exist. wikiPath=" + wikiPath) @@ -55,5 +56,16 @@ func TimingCountData() { } } //other user info data - models.CountData(wikiMap) + models.CounDataByDate(wikiMap, startTime, endTime) + +} + +func TimingCountData() { + + log.Info("start to time count data") + currentTimeNow := time.Now() + log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) + startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02") + + TimingCountDataByDate(startTime) } diff --git a/routers/user/auth.go b/routers/user/auth.go index 13e338565..44c5ad97d 100755 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -215,6 +215,7 @@ func SignInPost(ctx *context.Context, form auth.SignInForm) { } return } + models.SaveLoginInfoToDb(ctx.Req.Request, u) // If this user is enrolled in 2FA, we can't sign the user in just yet. // Instead, redirect them to the 2FA authentication page. _, err = models.GetTwoFactorByUID(u.ID) @@ -226,7 +227,6 @@ func SignInPost(ctx *context.Context, form auth.SignInForm) { } return } - // User needs to use 2FA, save data and redirect to 2FA page. if err := ctx.Session.Set("twofaUid", u.ID); err != nil { ctx.ServerError("UserSignIn: Unable to set twofaUid in session", err) @@ -240,7 +240,6 @@ func SignInPost(ctx *context.Context, form auth.SignInForm) { ctx.ServerError("UserSignIn: Unable to save session", err) return } - regs, err := models.GetU2FRegistrationsByUID(u.ID) if err == nil && len(regs) > 0 { ctx.Redirect(setting.AppSubURL + "/user/u2f") @@ -1168,8 +1167,8 @@ func SignUpPost(ctx *context.Context, cpt *captcha.Captcha, form auth.RegisterFo log.Trace("Account created: %s", u.Name, ctx.Data["MsgID"]) err := models.AddEmailAddress(&models.EmailAddress{ - UID: u.ID, - Email: form.Email, + UID: u.ID, + Email: form.Email, IsActivated: !setting.Service.RegisterEmailConfirm, }) @@ -1267,7 +1266,7 @@ func Activate(ctx *context.Context) { } email, err := models.GetEmailAddressByIDAndEmail(user.ID, user.Email) - if err != nil || email == nil{ + if err != nil || email == nil { log.Error("GetEmailAddressByIDAndEmail failed", ctx.Data["MsgID"]) } else { if err := email.Activate(); err != nil {