diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index fb639459f..9e9bf7f3a 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -82,6 +82,7 @@ type UserBusinessAnalysisQueryOptions struct { SortType string StartTime int64 EndTime int64 + IsAll bool } type UserBusinessAnalysisList []*UserBusinessAnalysis @@ -142,15 +143,33 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis return userBusinessAnalysisReturnList } +func getLastCountDate() int64 { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Limit(1, 0) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").OrderBy("count_date desc").Limit(1, 0). + Find(&userBusinessAnalysisList); err == nil { + for _, userRecord := range userBusinessAnalysisList { + return userRecord.CountDate - 10000 + } + } else { + log.Info("query error." + err.Error()) + } + currentTimeNow := time.Now() + pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + return pageStartTime.Unix() +} + func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { - log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime)) + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) statictisSess := xStatistic.NewSession() defer statictisSess.Close() currentTimeNow := time.Now() - pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + pageStartTime := getLastCountDate() + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() var cond = builder.NewCond() if len(opts.UserName) > 0 { @@ -159,10 +178,10 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus ) } cond = cond.And( - builder.Gte{"count_date": pageStartTime.Unix()}, + builder.Gte{"count_date": pageStartTime}, ) cond = cond.And( - builder.Lte{"count_date": pageEndTime.Unix()}, + builder.Lte{"count_date": pageEndTime}, ) count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) @@ -190,7 +209,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap := make(map[int64]*UserBusinessAnalysis) - if opts.Page >= 0 && opts.PageSize > 0 { + if opts.Page >= 0 && opts.PageSize > 0 && len(userBusinessAnalysisList) > 0 { var newAndCond = builder.NewCond() var newOrCond = builder.NewCond() for _, userRecord := range userBusinessAnalysisList { @@ -201,12 +220,14 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus newAndCond = newAndCond.And( newOrCond, ) - newAndCond = newAndCond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - newAndCond = newAndCond.And( - builder.Lte{"count_date": opts.EndTime}, - ) + if !opts.IsAll { + newAndCond = newAndCond.And( + builder.Gte{"count_date": opts.StartTime}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": opts.EndTime}, + ) + } userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) if err := statictisSess.Table("user_business_analysis").Where(newAndCond). @@ -246,7 +267,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) { +func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { log.Info("start to count other user info data") sess := x.NewSession() @@ -394,9 +415,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.CommitModelCount = 0 - statictisSess.Insert(&dateRecord) + _, err = statictisSess.Insert(&dateRecord) + if err != nil { + log.Info("insert daterecord failed." + err.Error()) + return err + } } - + return nil } func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { diff --git a/modules/git/repo_compare.go b/modules/git/repo_compare.go index 5bc7f9ca5..8082a8400 100644 --- a/modules/git/repo_compare.go +++ b/modules/git/repo_compare.go @@ -32,7 +32,7 @@ func (repo *Repository) GetMergeBase(tmpRemote string, base, head string) (strin if tmpRemote != "origin" { tmpBaseName := "refs/remotes/" + tmpRemote + "/tmp_" + base // Fetch commit into a temporary branch in order to be able to handle commits and tags - _, err := NewCommand("fetch", tmpRemote, base+":"+tmpBaseName).RunInDir(repo.Path) + _, err := NewCommand("fetch", tmpRemote, base+":"+tmpBaseName,"--no-tags").RunInDir(repo.Path) if err == nil { base = tmpBaseName } diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index 8d043eebd..2cc9e34be 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -366,6 +366,16 @@ sendjob: return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) } log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + BootFileErrorMsg := "Invalid OBS path '" + createJobParams.Config.BootFileUrl + "'." + DataSetErrorMsg := "Invalid OBS path '" + createJobParams.Config.DataUrl + "'." + if temp.ErrorMsg == BootFileErrorMsg { + log.Error("启动文件错误!createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("启动文件错误!") + } + if temp.ErrorMsg == DataSetErrorMsg { + log.Error("数据集错误!createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("数据集错误!") + } return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) } @@ -411,7 +421,16 @@ sendjob: log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) } - log.Error("createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + BootFileErrorMsg := "Invalid OBS path '" + createJobVersionParams.Config.BootFileUrl + "'." + DataSetErrorMsg := "Invalid OBS path '" + createJobVersionParams.Config.DataUrl + "'." + if temp.ErrorMsg == BootFileErrorMsg { + log.Error("启动文件错误!createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("启动文件错误!") + } + if temp.ErrorMsg == DataSetErrorMsg { + log.Error("数据集错误!createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("数据集错误!") + } return &result, fmt.Errorf("createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9f31612b6..c533eb924 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -547,8 +547,9 @@ var ( GrowthCommit float64 GrowthComments float64 RecordBeginTime string - Path string }{} + + Warn_Notify_Mails []string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1292,6 +1293,9 @@ func NewContext() { ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") SetRadarMapConfig() + + sec = Cfg.Section("warn_mail") + Warn_Notify_Mails = strings.Split(sec.Key("mails").MustString(""), ",") } func SetRadarMapConfig() { @@ -1329,7 +1333,6 @@ func SetRadarMapConfig() { RadarMap.GrowthCommit = sec.Key("growth_commit").MustFloat64(0.2) RadarMap.GrowthComments = sec.Key("growth_comments").MustFloat64(0.2) RadarMap.RecordBeginTime = sec.Key("record_beigin_time").MustString("2021-11-05") - RadarMap.Path = sec.Key("PATH").MustString("data/projectborad") } diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 7a5b47099..b91f85024 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -29,20 +29,20 @@ type FileInfo struct { } //check if has the object -//todo:修改查询方式 func ObsHasObject(path string) (bool, error) { hasObject := false - output, err := ObsCli.ListObjects(&obs.ListObjectsInput{Bucket: setting.Bucket}) - if err != nil { - log.Error("ListObjects failed:%v", err) - return hasObject, err - } - for _, obj := range output.Contents { - //obj.Key:attachment/0/1/019fd24e-4ef7-41cc-9f85-4a7b8504d958 - if path == obj.Key { - hasObject = true - break + input := &obs.GetObjectMetadataInput{} + input.Bucket = setting.Bucket + input.Key = path + _, err := ObsCli.GetObjectMetadata(input) + if err == nil { + hasObject = true + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("GetObjectMetadata failed(%d): %s", obsError.StatusCode, obsError.Message) + } else { + log.Error("%v", err.Error()) } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 598e5c7ca..ebe717cbc 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -420,7 +420,7 @@ static.createrepocount=Create Repo Count static.openiindex=OpenI Index static.registdate=Regist Date static.countdate=Count Date - +static.all=All [settings] profile = Profile @@ -813,10 +813,9 @@ total_count_get_error=Can not get the total page. last_update_time_error=Can not get the last updated time. get_repo_stat_error=Can not get the statistics of the repository. get_repo_info_error=Can not get the information of the repository. -modelarts.status=Status -modelarts.createtime=CreateTime -modelarts.version_nums = Version Nums -modelarts.computing_resources=compute Resources +generate_statistic_file_error=Fail to generate file. +repo_stat_inspect=ProjectAnalysis +all=All modelarts.notebook=Debug Task modelarts.train_job=Train Task modelarts.train_job.new_debug= New Debug Task diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 26d9c82d6..441d5ee0f 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -423,6 +423,7 @@ static.createrepocount=创建项目数 static.openiindex=OpenI指数 static.registdate=用户注册时间 static.countdate=系统统计时间 +static.all=所有 [settings] profile=个人信息 account=账号 diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 041162039..519077af2 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -4,8 +4,6 @@ import ( "fmt" "net/http" "net/url" - "os" - "path" "strconv" "time" @@ -175,15 +173,12 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) - if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil { - ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error()) - } - totalPage := getTotalPage(total, pageSize) f := excelize.NewFile() index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") for k, v := range allProjectsPeroidHeader(ctx) { f.SetCellValue(projectAnalysis, k, v) @@ -204,7 +199,6 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { } f.SetActiveSheet(index) - f.DeleteSheet("Sheet1") ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) ctx.Resp.Header().Set("Content-Type", "application/octet-stream") @@ -214,7 +208,7 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { } func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, projectAnalysis string) string { - baseName := setting.RadarMap.Path + "/" + projectAnalysis + "_" + baseName := projectAnalysis + "_" if ctx.QueryTrim("q") != "" { baseName = baseName + ctx.QueryTrim("q") + "_" @@ -225,21 +219,21 @@ func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, p baseName = baseName + beginTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) } frontName := baseName + ".xlsx" - return path.Base(frontName) + return frontName } func allProjectsPeroidHeader(ctx *context.Context) map[string]string { - return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("admin.repos.isPrivate"), "D1": ctx.Tr("admin.repos.openi"), "E1": ctx.Tr("admin.repos.visit"), "F1": ctx.Tr("admin.repos.download"), "G1": ctx.Tr("admin.repos.pr"), "H1": ctx.Tr("admin.repos.commit"), - "I1": ctx.Tr("admin.repos.watches"), "J1": ctx.Tr("admin.repos.stars"), "K1": ctx.Tr("admin.repos.forks"), "L1": ctx.Tr("admin.repos.issues"), "M1": ctx.Tr("admin.repos.closedIssues"), "N1": ctx.Tr("admin.repos.contributor")} + return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"), + "J1": ctx.Tr("admin.repos.watches"), "K1": ctx.Tr("admin.repos.stars"), "L1": ctx.Tr("admin.repos.forks"), "M1": ctx.Tr("admin.repos.issues"), "N1": ctx.Tr("admin.repos.closedIssues"), "O1": ctx.Tr("admin.repos.contributor")} } func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string { - return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): constructDistinctName(rs), getCellName("C", row): getIsPrivateDisplay(rs.IsPrivate, ctx), getCellName("D", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), - getCellName("E", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("F", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("G", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("H", row): strconv.FormatInt(rs.NumCommits, 10), - getCellName("I", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("J", row): strconv.FormatInt(rs.NumStars, 10), getCellName("K", row): strconv.FormatInt(rs.NumForks, 10), getCellName("L", row): strconv.FormatInt(rs.NumIssues, 10), - getCellName("M", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("N", row): strconv.FormatInt(rs.NumContributor, 10), + return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.Name, getCellName("C", row): rs.OwnerName, getCellName("D", row): getIsPrivateDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), + getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10), + getCellName("J", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("K", row): strconv.FormatInt(rs.NumStars, 10), getCellName("L", row): strconv.FormatInt(rs.NumForks, 10), getCellName("M", row): strconv.FormatInt(rs.NumIssues, 10), + getCellName("N", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("O", row): strconv.FormatInt(rs.NumContributor, 10), } } @@ -247,13 +241,6 @@ func getCellName(col string, row int) string { return col + strconv.Itoa(row) } -func constructDistinctName(rs *models.RepoStatistic) string { - if rs.OwnerName == "" { - return rs.Name - } - return rs.OwnerName + "/" + rs.Name -} - func getIsPrivateDisplay(private bool, ctx *context.Context) string { if private { return ctx.Tr("admin.repos.yes") @@ -327,6 +314,7 @@ func GetProjectLatestStatistics(ctx *context.Context) { Mode: mode, PR: pr, RelAvatarLink: contributor.RelAvatarLink, + Email: contributor.Email, } users = append(users, userInfo) @@ -508,12 +496,12 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, } else { - beginTime, err = time.Parse("2006-01-02", beginTimeStr) + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) if err != nil { return now, now, err } - endTime, err = time.Parse("2006-01-02", endTimeStr) + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) if err != nil { return now, now, err } @@ -533,8 +521,7 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, } func getRecordBeginTime() (time.Time, error) { - - return time.Parse(DATE_FORMAT, setting.RadarMap.RecordBeginTime) + return time.ParseInLocation(DATE_FORMAT, setting.RadarMap.RecordBeginTime, time.Local) } func getTotalPage(total int64, pageSize int) int { diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 2c082a289..8443d6488 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -360,6 +360,7 @@ func GetPresignedPutObjectURL(ctx *context.Context) { // AddAttachment response for add attachment record func AddAttachment(ctx *context.Context) { typeCloudBrain := ctx.QueryInt("type") + fileName := ctx.Query("file_name") err := checkTypeCloudBrain(typeCloudBrain) if err != nil { ctx.ServerError("checkTypeCloudBrain failed", err) @@ -375,7 +376,7 @@ func AddAttachment(ctx *context.Context) { return } } else { - has, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(uuid) + "/" + uuid) + has, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(uuid) + "/" + fileName) if err != nil { ctx.ServerError("ObsHasObject", err) return @@ -391,7 +392,7 @@ func AddAttachment(ctx *context.Context) { UUID: uuid, UploaderID: ctx.User.ID, IsPrivate: true, - Name: ctx.Query("file_name"), + Name: fileName, Size: ctx.QueryInt64("size"), DatasetID: ctx.QueryInt64("dataset_id"), Type: typeCloudBrain, @@ -479,6 +480,7 @@ func UpdateAttachmentDecompressState(ctx *context.Context) { func GetSuccessChunks(ctx *context.Context) { fileMD5 := ctx.Query("md5") typeCloudBrain := ctx.QueryInt("type") + fileName := ctx.Query("file_name") var chunks string err := checkTypeCloudBrain(typeCloudBrain) @@ -510,7 +512,7 @@ func GetSuccessChunks(ctx *context.Context) { return } } else { - isExist, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(fileChunk.UUID) + "/" + fileChunk.UUID) + isExist, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(fileChunk.UUID) + "/" + fileName) if err != nil { ctx.ServerError("ObsHasObject failed", err) return diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 08a4f6c11..c08c557b7 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1006,6 +1006,8 @@ func TrainJobShow(ctx *context.Context) { paramTemp = paramTemp + param } VersionListTasks[i].Parameters = paramTemp[:len(paramTemp)-2] + } else { + VersionListTasks[i].Parameters = "" } } diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 59cb5e4c7..066b29772 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -3,6 +3,8 @@ package repo import ( "time" + "code.gitea.io/gitea/services/mailer" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/normalization" @@ -26,14 +28,17 @@ func RepoStatisticDaily(date string) { log.Info("%s", date) log.Info("begin Repo Statistic") t, _ := time.Parse("2006-01-02", date) + warnEmailMessage := "项目统计信息入库失败,请尽快定位。" if err := models.DeleteRepoStatDaily(date); err != nil { log.Error("DeleteRepoStatDaily failed: %v", err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) return } repos, err := models.GetAllRepositories() if err != nil { log.Error("GetAllRepositories failed: %v", err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) return } @@ -47,7 +52,7 @@ func RepoStatisticDaily(date string) { var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth int64 repoGitStat, err := models.GetRepoKPIStats(repo) if err != nil { - log.Error("GetRepoKPIStats failed: %s", repo.Name) + log.Error("GetRepoKPIStats failed: %s", getDistinctProjectName(repo)) } else { numDevMonths = repoGitStat.DevelopAge numKeyContributor = repoGitStat.KeyContributors @@ -68,26 +73,26 @@ func RepoStatisticDaily(date string) { var numVersions int64 numVersions, err = models.GetReleaseCountByRepoID(repo.ID, models.FindReleasesOptions{}) if err != nil { - log.Error("GetReleaseCountByRepoID failed(%s): %v", repo.Name, err) + log.Error("GetReleaseCountByRepoID failed(%s): %v", getDistinctProjectName(repo), err) } var datasetSize int64 datasetSize, err = getDatasetSize(repo) if err != nil { - log.Error("getDatasetSize failed(%s): %v", repo.Name, err) + log.Error("getDatasetSize failed(%s): %v", getDistinctProjectName(repo), err) } var numComments int64 numComments, err = models.GetCommentCountByRepoID(repo.ID) if err != nil { - log.Error("GetCommentCountByRepoID failed(%s): %v", repo.Name, err) + log.Error("GetCommentCountByRepoID failed(%s): %v", getDistinctProjectName(repo), err) } beginTime, endTime := getStatTime(date) var numVisits int numVisits, err = repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) if err != nil { - log.Error("AppointProjectView failed(%s): %v", repo.Name, err) + log.Error("AppointProjectView failed(%s): %v", getDistinctProjectName(repo), err) } repoStat := models.RepoStatistic{ @@ -151,8 +156,9 @@ func RepoStatisticDaily(date string) { } if _, err = models.InsertRepoStat(&repoStat); err != nil { - log.Error("InsertRepoStat failed(%s): %v", repo.Name, err) - log.Error("failed statistic: %s", repo.Name) + log.Error("InsertRepoStat failed(%s): %v", getDistinctProjectName(repo), err) + log.Error("failed statistic: %s", getDistinctProjectName(repo)) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) continue } @@ -244,6 +250,10 @@ func RepoStatisticDaily(date string) { } +func getDistinctProjectName(repo *models.Repository) string { + return repo.OwnerName + "/" + repo.Name +} + func getDatasetSize(repo *models.Repository) (int64, error) { dataset, err := models.GetDatasetByRepo(repo) if err != nil { diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 134896177..5c701eadb 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer" "github.com/360EntSecGroup-Skylar/excelize/v2" ) @@ -39,12 +40,26 @@ func QueryUserStaticDataPage(ctx *context.Context) { } userName := ctx.Query("userName") IsReturnFile := ctx.QueryBool("IsReturnFile") - log.Info("startDate=" + startDate + " endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page)) - startTime, _ := time.Parse("2006-01-02", startDate) - endTime, _ := time.Parse("2006-01-02", endDate) - endTime = endTime.AddDate(0, 0, 1) - log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) + var startTime time.Time + var endTime time.Time + var isAll bool + if startDate == "all" { + isAll = true + startTime = time.Now() + endTime = time.Now() + } else { + startTime, _ = time.Parse("2006-01-02", startDate) + settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime) + if startTime.Unix() < settingStartTime.Unix() { + startTime = settingStartTime + startDate = settingStartTime.Format("2006-01-02") + } + endTime, _ = time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) + isAll = false + log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) + } if IsReturnFile { page = -1 @@ -59,6 +74,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { UserName: userName, StartTime: startTime.Unix(), EndTime: endTime.Unix(), + IsAll: isAll, } mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) @@ -67,8 +83,10 @@ func QueryUserStaticDataPage(ctx *context.Context) { if IsReturnFile { //writer exec file. xlsx := excelize.NewFile() + xlsx.DeleteSheet("Sheet1") sheetName := ctx.Tr("user.static.sheetname") index := xlsx.NewSheet(sheetName) + dataHeader := map[string]string{ "A1": ctx.Tr("user.static.id"), "B1": ctx.Tr("user.static.name"), @@ -110,14 +128,27 @@ func QueryUserStaticDataPage(ctx *context.Context) { xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount) xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) - xlsx.SetCellValue(sheetName, "O"+rows, userRecord.OpenIIndex) + xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) xlsx.SetCellValue(sheetName, "P"+rows, userRecord.RegistDate.Format("2006-01-02")) xlsx.SetCellValue(sheetName, "Q"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02")) } //设置默认打开的表单 xlsx.SetActiveSheet(index) - filename := sheetName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" + var filename string + + nowTime := time.Now() + nowZeroTime := time.Date(nowTime.Year(), nowTime.Month(), nowTime.Day(), 0, 0, 0, 0, nowTime.Location()) + if endTime.Unix() >= nowZeroTime.Unix() { + endDate = nowZeroTime.AddDate(0, 0, -1).Format("2006-01-02") + } + + if isAll { + filename = sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx" + } else { + filename = sheetName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" + } + if len(userName) > 0 { filename = sheetName + "_" + userName + "_" + strings.ReplaceAll(startDate, "-", "") + "_" + strings.ReplaceAll(endDate, "-", "") + ".xlsx" } @@ -142,10 +173,11 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { //query wiki data log.Info("start to time count data") wikiMap := make(map[string]int) - + warnEmailMessage := "用户统计信息入库失败,请尽快定位。" repoList, err := models.GetAllRepositories() if err != nil { - log.Error("query repo error.") + log.Error("query repo error." + err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) return } log.Info("start to query wiki data") @@ -180,7 +212,11 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { } } //other user info data - models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount) + err = models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount) + if err != nil { + log.Error("count user info error." + err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) + } } func TimingCountDataByDate(date string) { diff --git a/routers/repo/view.go b/routers/repo/view.go index 13dcf22aa..a5f2e1477 100755 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -824,24 +824,21 @@ func renderCode(ctx *context.Context) { */ baseGitRepo, err := git.OpenRepository(ctx.Repo.Repository.BaseRepo.RepoPath()) defer baseGitRepo.Close() + var compareInfo *git.CompareInfo if err != nil { log.Error("error open baseRepo:%s", ctx.Repo.Repository.BaseRepo.RepoPath()) ctx.Data["FetchUpstreamCnt"] = -1 // minus value indicates error } else { if _, error := baseGitRepo.GetBranch(ctx.Repo.BranchName); error == nil { //base repo has the same branch, then compare between current repo branch and base repo's branch - compareUrl := ctx.Repo.BranchName + "..." + ctx.Repo.Repository.BaseRepo.OwnerName + "/" + ctx.Repo.Repository.BaseRepo.Name + ":" + ctx.Repo.BranchName - ctx.SetParams("*", compareUrl) + compareInfo, err = baseGitRepo.GetCompareInfo(ctx.Repo.Repository.RepoPath(), ctx.Repo.BranchName, ctx.Repo.BranchName) ctx.Data["UpstreamSameBranchName"] = true } else { //else, compare between current repo branch and base repo's default branch - compareUrl := ctx.Repo.BranchName + "..." + ctx.Repo.Repository.BaseRepo.OwnerName + "/" + ctx.Repo.Repository.BaseRepo.Name + ":" + ctx.Repo.Repository.BaseRepo.DefaultBranch - ctx.SetParams("*", compareUrl) + compareInfo, err = baseGitRepo.GetCompareInfo(ctx.Repo.Repository.RepoPath(), ctx.Repo.BranchName, ctx.Repo.Repository.BaseRepo.DefaultBranch) ctx.Data["UpstreamSameBranchName"] = false } - _, _, headGitRepo, compareInfo, _, _ := ParseCompareInfo(ctx) - defer headGitRepo.Close() - if compareInfo != nil { + if err==nil && compareInfo != nil { if compareInfo.Commits != nil { log.Info("compareInfoCommits数量:%d", compareInfo.Commits.Len()) ctx.Data["FetchUpstreamCnt"] = compareInfo.Commits.Len() diff --git a/services/mailer/mail.go b/services/mailer/mail.go index e9395f130..ef7447be2 100755 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -115,6 +115,22 @@ func SendActivateEmailMail(locale Locale, u *models.User, email *models.EmailAdd SendAsync(msg) } +func SendWarnNotifyMail(emails []string, message string) { + if setting.MailService == nil { + log.Warn("SendWarnNotifyMail is being invoked but mail service hasn't been initialized") + return + } + if len(emails) == 0 { + log.Warn("SendWarnNotifyMail is being invoked but do not have email to send") + return + } + msg := NewMessage(emails, message, message) + msg.Info = fmt.Sprintf(message) + + SendAsync(msg) + +} + // SendRegisterNotifyMail triggers a notify e-mail by admin created a account. func SendRegisterNotifyMail(locale Locale, u *models.User) { if setting.MailService == nil { diff --git a/web_src/js/components/DataAnalysis.vue b/web_src/js/components/DataAnalysis.vue index 4c2657751..931f10e36 100755 --- a/web_src/js/components/DataAnalysis.vue +++ b/web_src/js/components/DataAnalysis.vue @@ -7,10 +7,8 @@ 概览 -