From 4eb86a5d9cd367f3dabfac6ed51320a8864185a7 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 22 Apr 2022 14:22:47 +0800 Subject: [PATCH 01/62] =?UTF-8?q?fix-1937=E5=90=8E=E7=AB=AF=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/repo.go | 5 + models/summary_statistic.go | 34 ++++ options/locale/locale_en-US.ini | 11 ++ options/locale/locale_zh-CN.ini | 12 ++ routers/api/v1/api.go | 3 + routers/api/v1/repo/repo_dashbord.go | 313 +++++++++++++++++++++++++++++++++ routers/repo/repo_summary_statistic.go | 7 + 7 files changed, 385 insertions(+) diff --git a/models/repo.go b/models/repo.go index 25bfb4a74..db2694617 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1554,6 +1554,11 @@ func GetAllMirrorRepositoriesCount() (int64, error) { return x.Where("is_mirror = ?", true).Count(repo) } +func GetAllOrgRepositoriesCount() (int64, error) { + repo := new(Repository) + return x.Table("repository").Join("INNER", []string{"\"user\"", "u"}, "repository.owner_id = u.id and u.type=1").Count(repo) +} + func GetAllForkRepositoriesCount() (int64, error) { repo := new(Repository) return x.Where("is_fork = ?", true).Count(repo) diff --git a/models/summary_statistic.go b/models/summary_statistic.go index e5cf54b75..7787aa292 100644 --- a/models/summary_statistic.go +++ b/models/summary_statistic.go @@ -2,6 +2,8 @@ package models import ( "fmt" + "strconv" + "time" "code.gitea.io/gitea/modules/timeutil" ) @@ -45,6 +47,7 @@ type SummaryStatistic struct { NumRepoFork int64 `xorm:"NOT NULL DEFAULT 0"` NumRepoMirror int64 `xorm:"NOT NULL DEFAULT 0"` NumRepoSelf int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepoOrg int64 `xorm:"NOT NULL DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } @@ -69,6 +72,37 @@ func DeleteSummaryStatisticDaily(date string) error { return nil } +func GetLatest2SummaryStatistic() ([]*SummaryStatistic, error) { + summaryStatistics := make([]*SummaryStatistic, 0) + err := xStatistic.Desc("created_unix").Limit(2).Find(&summaryStatistics) + return summaryStatistics, err +} + +func GetSummaryStatisticByTimeCount(beginTime time.Time, endTime time.Time) (int64, error) { + summaryStatistics := new(SummaryStatistic) + total, err := xStatistic.Asc("created_unix").Where("created_unix>=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)).Count(summaryStatistics) + return total, err +} + +func GetSummaryStatisticByDateCount(dates []string) (int64, error) { + summaryStatistics := new(SummaryStatistic) + total, err := xStatistic.Asc("created_unix").In("date", dates).Count(summaryStatistics) + return total, err +} + +func GetSummaryStatisticByTime(beginTime time.Time, endTime time.Time, page int, pageSize int) ([]*SummaryStatistic, error) { + summaryStatistics := make([]*SummaryStatistic, 0) + err := xStatistic.Asc("created_unix").Limit(pageSize+1, (page-1)*pageSize).Where("created_unix>=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)).Find(&summaryStatistics) + + return summaryStatistics, err +} + +func GetSummaryStatisticByDates(dates []string, page int, pageSize int) ([]*SummaryStatistic, error) { + summaryStatistics := make([]*SummaryStatistic, 0) + err := xStatistic.Asc("created_unix").In("date", dates).Limit(pageSize+1, (page-1)*pageSize).Find(&summaryStatistics) + return summaryStatistics, err +} + func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) { return xStatistic.Insert(summaryStatistic) } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f53bea7b1..6d1ac24c7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -937,6 +937,15 @@ model_manager = Model model_noright=No right model_rename=Duplicate model name, please modify model name. +date=Date +repo_add=Project Increment +repo_total=Project Total +repo_public_add=Public Project Increment +repo_private_add=Private Project Increment +repo_fork_add=Fork Project Increment +repo_mirror_add=Mirror Project Increment +repo_self_add=Custom Project Increment + debug=Debug debug_again=Restart stop=Stop @@ -1001,7 +1010,9 @@ get_repo_stat_error=Can not get the statistics of the repository. get_repo_info_error=Can not get the information of the repository. generate_statistic_file_error=Failed to generate file. repo_stat_inspect=ProjectAnalysis +repo_stat_develop=ProjectGrowthAnalysis all=All +current_year=Current_Year computing.all = All computing.Introduction=Introduction diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index c82347d5e..2232c36a1 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -938,6 +938,16 @@ model_manager = 模型 model_noright=无权限操作 model_rename=模型名称重复，请修改模型名称 + +date=日期 +repo_add=新增项目 +repo_total=累计项目 +repo_public_add=新增公开项目 +repo_private_add=新增私有项目 +repo_fork_add=新增派生项目 +repo_mirror_add=新增镜像项目 +repo_self_add=新增自建项目 + debug=调试 debug_again=再次调试 stop=停止 @@ -1009,7 +1019,9 @@ get_repo_stat_error=查询当前仓库的统计信息失败。 get_repo_info_error=查询当前仓库信息失败。 generate_statistic_file_error=生成文件失败。 repo_stat_inspect=项目分析 +repo_stat_develop=项目增长趋势 all=所有 +current_year=今年 computing.all=全部 computing.Introduction=简介 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 2b070a4b8..3cc1a6693 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -535,6 +535,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/restoreFork", repo.RestoreForkNumber) m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) + m.Get("/summary", repo.GetLatestProjectsSummaryData) + m.Get("/summary/period", repo.GetProjectsSummaryData) + m.Get("/summary/download", repo.GetProjectsSummaryDataFile) m.Group("/project", func() { m.Get("", repo.GetAllProjectsPeriodStatistics) m.Get("/numVisit", repo.ProjectNumVisit) diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 2c42f8a16..7ec315fd9 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -20,8 +20,10 @@ import ( const DEFAULT_PAGE_SIZE = 10 const DATE_FORMAT = "2006-01-02" +const MONTH_FORMAT = "2006-01" const EXCEL_DATE_FORMAT = "20060102" const CREATE_TIME_FORMAT = "2006/01/02 15:04:05" +const UPDATE_TIME_FORMAT = "2006-01-02 15:04:05" type ProjectsPeriodData struct { RecordBeginTime string `json:"recordBeginTime"` @@ -60,6 +62,40 @@ type ProjectLatestData struct { Top10 []UserInfo `json:"top10"` } +type ProjectSummaryBaseData struct { + NumReposAdd int64 `json:"numReposAdd"` + NumRepoPublicAdd int64 `json:"numRepoPublicAdd"` + NumRepoPrivateAdd int64 `json:"numRepoPrivateAdd"` + NumRepoForkAdd int64 `json:"numRepoForkAdd"` + NumRepoMirrorAdd int64 `json:"numRepoMirrorAdd"` + NumRepoSelfAdd int64 `json:"numRepoSelfAdd"` + NumRepos int64 `json:"numRepos"` + CreatTime string `json:"creatTime"` +} + +type ProjectSummaryData struct { + ProjectSummaryBaseData + NumRepoPublic int64 `json:"numRepoPublic"` + NumRepoPrivate int64 `json:"numRepoPrivate"` + NumRepoFork int64 `json:"numRepoFork"` + NumRepoMirror int64 `json:"numRepoMirror"` + NumRepoSelf int64 `json:"numRepoSelf"` + + NumRepoOrgAdd int64 `json:"numRepoOrgAdd"` + NumRepoNotOrgAdd int64 `json:"numRepoNotOrgAdd"` + + NumRepoOrg int64 `json:"numRepoOrg"` + NumRepoNotOrg int64 `json:"numRepoNotOrg"` +} + +type ProjectSummaryPeriodData struct { + RecordBeginTime string `json:"recordBeginTime"` + PageSize int `json:"pageSize"` + TotalPage int `json:"totalPage"` + TotalCount int64 `json:"totalCount"` + PageRecords []*ProjectSummaryBaseData `json:"pageRecords"` +} + func RestoreForkNumber(ctx *context.Context) { repos, err := models.GetAllRepositories() if err != nil { @@ -73,6 +109,144 @@ func RestoreForkNumber(ctx *context.Context) { ctx.JSON(http.StatusOK, struct{}{}) } +func GetLatestProjectsSummaryData(ctx *context.Context) { + stat, err := models.GetLatest2SummaryStatistic() + data := ProjectSummaryData{} + if err == nil && len(stat) > 0 { + data.NumRepos = stat[0].NumRepos + data.NumRepoOrg = stat[0].NumRepoOrg + data.NumRepoNotOrg = stat[0].NumRepos - stat[0].NumRepoOrg + data.NumRepoFork = stat[0].NumRepoFork + data.NumRepoMirror = stat[0].NumRepoMirror + data.NumRepoSelf = stat[0].NumRepoSelf + data.NumRepoPrivate = stat[0].NumRepoPrivate + data.NumRepoPublic = stat[0].NumRepoPublic + data.CreatTime = stat[0].CreatedUnix.Format(UPDATE_TIME_FORMAT) + if len(stat) == 2 { + data.NumReposAdd = stat[0].NumRepos - stat[1].NumRepos + data.NumRepoOrgAdd = stat[0].NumRepoOrg - stat[1].NumRepoOrg + data.NumRepoNotOrgAdd = (stat[0].NumRepos - stat[0].NumRepoOrg) - (stat[1].NumRepos - stat[1].NumRepoOrg) + data.NumRepoForkAdd = stat[0].NumRepoFork - stat[1].NumRepoFork + data.NumRepoMirrorAdd = stat[0].NumRepoMirror - stat[1].NumRepoMirror + data.NumRepoSelfAdd = stat[0].NumRepoSelf - stat[1].NumRepoSelf + data.NumRepoPrivateAdd = stat[0].NumRepoPrivate - stat[1].NumRepoPrivate + data.NumRepoPublicAdd = stat[0].NumRepoPublic - stat[1].NumRepoPublic + } + } + ctx.JSON(200, data) +} + +func GetProjectsSummaryData(ctx *context.Context) { + + var datas = make([]*ProjectSummaryBaseData, 0) + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + + beginTime = beginTime.AddDate(0, 0, -1) + + queryType := ctx.QueryTrim("type") + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pagesize") + if pageSize <= 0 { + pageSize = DEFAULT_PAGE_SIZE + } + var count int64 + + if queryType == "all" || queryType == "current_year" { + dates := getEndOfMonthDates(beginTime, endTime) + count, _ = models.GetSummaryStatisticByDateCount(dates) + stats, err := models.GetSummaryStatisticByDates(dates, page, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + + for i, v := range stats { + if i == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[i-1]) + createTime, _ := time.Parse(DATE_FORMAT, v.Date) + data.CreatTime = createTime.Format(MONTH_FORMAT) + datas = append(datas, &data) + } + } + + } else { + count, _ = models.GetSummaryStatisticByTimeCount(beginTime, endTime) + stats, err := models.GetSummaryStatisticByTime(beginTime, endTime, page, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + + for i, v := range stats { + if i == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[i-1]) + data.CreatTime = v.Date + datas = append(datas, &data) + } + } + + } + projectSummaryPeriodData := ProjectSummaryPeriodData{ + TotalCount: count - 1, + TotalPage: getTotalPage(count-1, pageSize), + RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), + PageSize: pageSize, + PageRecords: datas, + } + + ctx.JSON(200, projectSummaryPeriodData) + +} + +func setStatisticsData(data *ProjectSummaryBaseData, v *models.SummaryStatistic, stats *models.SummaryStatistic) { + data.NumReposAdd = v.NumRepos - stats.NumRepos + data.NumRepoPublicAdd = v.NumRepoPublic - stats.NumRepoPublic + data.NumRepoPrivateAdd = v.NumRepoPrivate - stats.NumRepoPrivate + data.NumRepoMirrorAdd = v.NumRepoMirror - stats.NumRepoMirror + data.NumRepoForkAdd = v.NumRepoFork - stats.NumRepoFork + data.NumRepoSelfAdd = v.NumRepoSelf - stats.NumRepoSelf + + data.NumRepos = v.NumRepos +} + +func getEndOfMonthDates(beginTime time.Time, endTime time.Time) []string { + var dates = []string{} + date := endOfMonth(beginTime.AddDate(0, -1, 0)) + dates = append(dates, date.Format(DATE_FORMAT)) + + tempDate := endOfMonth(beginTime) + + for { + if tempDate.Before(endTime) { + dates = append(dates, tempDate.Format(DATE_FORMAT)) + tempDate = endOfMonth(tempDate.AddDate(0, 0, 1)) + } else { + break + } + } + + return dates +} + +func endOfMonth(date time.Time) time.Time { + return date.AddDate(0, 1, -date.Day()) +} + func GetAllProjectsPeriodStatistics(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() @@ -210,6 +384,118 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { } +func GetProjectsSummaryDataFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + beginTime = beginTime.AddDate(0, 0, -1) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 100 + + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + var projectAnalysis = ctx.Tr("repo.repo_stat_develop") + fileName := getSummaryFileName(ctx, beginTime, endTime, projectAnalysis) + + f := excelize.NewFile() + + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") + + for k, v := range allProjectsPeriodSummaryHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + + var total int64 + queryType := ctx.QueryTrim("type") + row := 2 + + if queryType == "all" || queryType == "current_year" { + dates := getEndOfMonthDates(beginTime, endTime) + total, _ = models.GetSummaryStatisticByDateCount(dates) + totalPage := getTotalPage(total, pageSize) + + for i := 0; i < totalPage; i++ { + + stats, err := models.GetSummaryStatisticByDates(dates, i+1, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + for j, v := range stats { + if j == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[j-1]) + createTime, _ := time.Parse(DATE_FORMAT, v.Date) + data.CreatTime = createTime.Format(MONTH_FORMAT) + + for k, v := range allProjectsPeriodSummaryValues(row, &data, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + } + + } + + } + + } else { + total, _ = models.GetSummaryStatisticByTimeCount(beginTime, endTime) + totalPage := getTotalPage(total, pageSize) + + for i := 0; i < totalPage; i++ { + + stats, err := models.GetSummaryStatisticByTime(beginTime, endTime, i+1, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + for j, v := range stats { + if j == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[j-1]) + data.CreatTime = v.Date + + for k, v := range allProjectsPeriodSummaryValues(row, &data, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + } + + } + + } + } + + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) + +} + func ServeAllProjectsOpenIStatisticsFile(ctx *context.Context) { page := ctx.QueryInt("page") @@ -290,6 +576,20 @@ func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, p return frontName } +func getSummaryFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, projectAnalysis string) string { + baseName := projectAnalysis + "_" + + if ctx.QueryTrim("type") == "all" { + baseName = baseName + ctx.Tr("repo.all") + } else if ctx.QueryTrim("type") == "current_year" { + baseName = baseName + ctx.Tr("repo.current_year") + } else { + baseName = baseName + beginTime.Format(EXCEL_DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) + } + frontName := baseName + ".xlsx" + return frontName +} + func allProjectsPeroidHeader(ctx *context.Context) map[string]string { return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"), @@ -297,6 +597,19 @@ func allProjectsPeroidHeader(ctx *context.Context) map[string]string { } +func allProjectsPeriodSummaryHeader(ctx *context.Context) map[string]string { + + return map[string]string{"A1": ctx.Tr("repo.date"), "B1": ctx.Tr("repo.repo_add"), "C1": ctx.Tr("repo.repo_total"), "D1": ctx.Tr("repo.repo_public_add"), "E1": ctx.Tr("repo.repo_private_add"), "F1": ctx.Tr("repo.repo_self_add"), "G1": ctx.Tr("repo.repo_fork_add"), "H1": ctx.Tr("repo.repo_mirror_add")} + +} + +func allProjectsPeriodSummaryValues(row int, rs *ProjectSummaryBaseData, ctx *context.Context) map[string]string { + + return map[string]string{getCellName("A", row): rs.CreatTime, getCellName("B", row): strconv.FormatInt(rs.NumReposAdd, 10), getCellName("C", row): strconv.FormatInt(rs.NumRepos, 10), getCellName("D", row): strconv.FormatInt(rs.NumRepoPublicAdd, 10), getCellName("E", row): strconv.FormatInt(rs.NumRepoPrivateAdd, 10), + getCellName("F", row): strconv.FormatInt(rs.NumRepoSelfAdd, 10), getCellName("G", row): strconv.FormatInt(rs.NumRepoForkAdd, 10), getCellName("H", row): strconv.FormatInt(rs.NumRepoMirrorAdd, 10), + } +} + func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.DisplayName(), getCellName("C", row): rs.OwnerName, getCellName("D", row): getBoolDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10), diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go index 3af31737c..65ba2cf0b 100644 --- a/routers/repo/repo_summary_statistic.go +++ b/routers/repo/repo_summary_statistic.go @@ -60,6 +60,12 @@ func SummaryStatisticDaily(date string) { } selfRepositoryNumber := repositoryNumer - mirrorRepositoryNumber - forkRepositoryNumber + organizationRepoNumber, err := models.GetAllOrgRepositoriesCount() + if err != nil { + log.Error("can not get org repository number", err) + organizationRepoNumber = 0 + } + //repository size repositorySize, err := models.GetAllRepositoriesSize() if err != nil { @@ -99,6 +105,7 @@ func SummaryStatisticDaily(date string) { NumRepoPrivate: privateRepositoryNumer, NumRepoPublic: publicRepositoryNumer, NumRepoSelf: selfRepositoryNumber, + NumRepoOrg: organizationRepoNumber, NumRepoBigModel: topicsCount[0], NumRepoAI: topicsCount[1], NumRepoVision: topicsCount[2], From 5cecfa5159029e19f841bc4507c6a5da1a5d30e5 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 28 Apr 2022 15:37:28 +0800 Subject: [PATCH 02/62] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82=E7=94=A8=E6=88=B7=E7=BB=9F=E8=AE=A1=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/dbsql/repo_foreigntable_for_es.sql | 10 ++- models/user_business_analysis.go | 144 ++++++++++++++++++++++++++++-- models/user_business_struct.go | 17 ++-- options/locale/locale_zh-CN.ini | 13 +++ routers/repo/user_data_analysis.go | 82 +++++++++++++++-- 5 files changed, 247 insertions(+), 19 deletions(-) diff --git a/models/dbsql/repo_foreigntable_for_es.sql b/models/dbsql/repo_foreigntable_for_es.sql index 7e06fd99e..e927eb7c2 100644 --- a/models/dbsql/repo_foreigntable_for_es.sql +++ b/models/dbsql/repo_foreigntable_for_es.sql @@ -523,17 +523,21 @@ DROP TRIGGER IF EXISTS es_udpate_repository_lang on public.language_stat; CREATE OR REPLACE FUNCTION public.udpate_repository_lang() RETURNS trigger AS $def$ + DECLARE + privateValue bigint; BEGIN if (TG_OP = 'UPDATE') then - update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; + select into privateValue updated_unix from public.repository where id=NEW.repo_id; + update public.repository_es SET updated_unix=privateValue,lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; elsif (TG_OP = 'INSERT') then - update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; + select into privateValue updated_unix from public.repository where id=NEW.repo_id; + update public.repository_es SET updated_unix=privateValue,lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; elsif (TG_OP = 'DELETE') then if exists(select 1 from public.repository where id=OLD.repo_id) then update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=OLD.repo_id) where id=OLD.repo_id; end if; end if; - return null; + return NEW; END; $def$ LANGUAGE plpgsql; diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 2d7592baf..7e9e877c2 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" "code.gitea.io/gitea/modules/log" @@ -103,6 +104,8 @@ type UserBusinessAnalysisAll struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + HasActivity int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysis struct { @@ -190,6 +193,8 @@ type UserBusinessAnalysis struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + HasActivity int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisQueryOptions struct { @@ -227,15 +232,93 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } -func QueryMetrics(start int64, end int64) ([]*UserMetrics, int64) { +func QueryMetricsPage(start int64, end int64, page int, pageSize int) ([]UserMetrics, int64) { + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + cond := "count_date >" + fmt.Sprint(start) + " and count_date<" + fmt.Sprint(end) + allCount, err := statictisSess.Where(cond).Count(new(UserMetrics)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + userMetricsList := make([]UserMetrics, 0) + if err := statictisSess.Table(new(UserMetrics)).Where(cond).Limit(pageSize, page*pageSize).OrderBy("count_date desc"). + Find(&userMetricsList); err != nil { + return nil, 0 + } + return userMetricsList, allCount + +} + +func QueryMetrics(start int64, end int64) ([]UserMetrics, int) { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - userMetricsList := make([]*UserMetrics, 0) + userMetricsList := make([]UserMetrics, 0) if err := statictisSess.Table(new(UserMetrics)).Where("count_date >" + fmt.Sprint(start) + " and count_date<" + fmt.Sprint(end)).OrderBy("count_date desc"). Find(&userMetricsList); err != nil { return nil, 0 } - return userMetricsList, int64(len(userMetricsList)) + return userMetricsList, len(userMetricsList) +} + +func QueryMetricsForAll() []UserMetrics { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + userMetricsList := make([]UserMetrics, 0) + if err := statictisSess.Table(new(UserMetrics)).OrderBy("count_date desc"). + Find(&userMetricsList); err != nil { + return nil + } + return makeResultForMonth(userMetricsList, len(userMetricsList)) +} + +func QueryMetricsForYear() []UserMetrics { + currentTimeNow := time.Now() + currentYearEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + currentYearStartTime := time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location()) + allUserInfo, count := QueryMetrics(currentYearStartTime.Unix(), currentYearEndTime.Unix()) + + return makeResultForMonth(allUserInfo, count) +} + +func makeResultForMonth(allUserInfo []UserMetrics, count int) []UserMetrics { + monthMap := make(map[string]UserMetrics) + if count > 0 { + for _, userMetrics := range allUserInfo { + dateTime := time.Unix(userMetrics.CountDate, 0) + month := fmt.Sprint(dateTime.Year()) + "-" + fmt.Sprint(int(dateTime.Month())) + if _, ok := monthMap[month]; !ok { + var monthUserMetrics UserMetrics + monthUserMetrics.DisplayDate = month + monthUserMetrics.ActivateRegistUser = userMetrics.ActivateRegistUser + monthUserMetrics.NotActivateRegistUser = userMetrics.NotActivateRegistUser + monthUserMetrics.TotalUser = userMetrics.TotalUser + monthUserMetrics.TotalActivateRegistUser = userMetrics.TotalActivateRegistUser + monthUserMetrics.TotalHasActivityUser = userMetrics.TotalHasActivityUser + monthUserMetrics.HasActivityUser = userMetrics.HasActivityUser + monthUserMetrics.DaysForMonth = 1 + monthMap[month] = monthUserMetrics + } else { + value := monthMap[month] + value.ActivateRegistUser += userMetrics.ActivateRegistUser + value.NotActivateRegistUser += userMetrics.NotActivateRegistUser + value.TotalUser += userMetrics.TotalUser + value.TotalActivateRegistUser += userMetrics.TotalActivateRegistUser + value.TotalHasActivityUser += userMetrics.TotalHasActivityUser + value.HasActivityUser += userMetrics.HasActivityUser + value.DaysForMonth += 1 + } + } + } + result := make([]UserMetrics, 0) + for _, value := range monthMap { + result = append(result, value) + } + sort.Slice(result, func(i, j int) bool { + return strings.Compare(result[i].DisplayDate, result[j].DisplayDate) > 0 + }) + return result } func QueryRankList(key string, tableName string, limit int) ([]*UserBusinessAnalysisAll, int64) { @@ -540,6 +623,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS if minUserIndex > dateRecordAll.UserIndexPrimitive { minUserIndex = dateRecordAll.UserIndexPrimitive } + dateRecordBatch = append(dateRecordBatch, dateRecordAll) if len(dateRecordBatch) >= BATCH_INSERT_SIZE { insertTable(dateRecordBatch, tableName, statictisSess) @@ -695,7 +779,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("query user error. return.") return err } - + userNewAddActivity := make(map[int64]map[int64]int64) ParaWeight := getParaWeight() userMetrics := make(map[string]int) var indexTotal int64 @@ -767,6 +851,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.UserIndexPrimitive = getUserIndex(dateRecord, ParaWeight) setUserMetrics(userMetrics, userRecord, start_unix, end_unix, dateRecord) + if getUserActivate(dateRecord) > 0 { + addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID) + } _, err = statictisSess.Insert(&dateRecord) if err != nil { log.Info("insert daterecord failed." + err.Error()) @@ -792,11 +879,58 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, useMetrics.NotActivateRegistUser = getMapKeyStringValue("NotActivateRegistUser", userMetrics) useMetrics.TotalActivateRegistUser = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) useMetrics.TotalHasActivityUser = getMapKeyStringValue("TotalHasActivityUser", userMetrics) - statictisSess.Insert(&useMetrics) + count, err = sess.Count(new(User)) + if err != nil { + log.Info("query user error. return.") + } + useMetrics.TotalUser = int(count) + useMetrics.ActivateIIndex = float64(useMetrics.ActivateRegistUser) / float64(useMetrics.ActivateRegistUser+useMetrics.NotActivateRegistUser) + statictisSess.Insert(&useMetrics) + //update new user activity + updateNewUserAcitivity(userNewAddActivity, statictisSess) return nil } +func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) { + for key, value := range currentUserActivity { + useMetrics := &UserMetrics{CountDate: key} + has, err := statictisSess.Get(useMetrics) + if err == nil && has { + userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",") + for _, userIdStr := range userIdArrays { + userIdInt, err := strconv.ParseInt(userIdStr, 10, 64) + if err == nil { + value[userIdInt] = userIdInt + } + } + userIdArray := "" + for _, tmpValue := range value { + userIdArray += fmt.Sprint(tmpValue) + "," + } + useMetrics.HasActivityUser = len(value) + if len(userIdArray) > 0 { + useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] + + } + statictisSess.Update(useMetrics) + } + } +} + +func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64) { + CountDateTime := time.Date(registDate.Year(), registDate.AsTime().Month(), registDate.AsTime().Day(), 0, 1, 0, 0, registDate.AsTime().Location()) + CountDate := CountDateTime.Unix() + if _, ok := currentUserActivity[CountDate]; !ok { + userIdMap := make(map[int64]int64, 0) + userIdMap[userId] = userId + currentUserActivity[CountDate] = userIdMap + } else { + currentUserActivity[CountDate][userId] = userId + } + +} + func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, end_time int64, dateRecord UserBusinessAnalysis) { //ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` //NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 86aecd545..5560cb19a 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -400,10 +400,15 @@ type UserAnalysisPara struct { } type UserMetrics struct { - CountDate int64 `xorm:"pk"` - ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` - TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - TotalHasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + CountDate int64 `xorm:"pk"` + ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + ActivateIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + TotalUser int `xorm:"NOT NULL DEFAULT 0"` + TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + TotalHasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + DisplayDate string `xorm:"-"` + DaysForMonth int `xorm:"NOT NULL DEFAULT 0"` + HasActivityUserJson string `xorm:"NULL"` } diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 474b96c3f..da70e984d 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -530,6 +530,19 @@ static.public.user_business_analysis_last30_day=近30天 static.public.user_business_analysis_last_month=上月 static.public.user_business_analysis_yesterday=昨天 static.public.user_business_analysis_all=所有 + +metrics.sheetname=用户趋势分析 +metrics.date=日期 +metrics.newregistuser=新增注册用户 +metrics.newregistandactiveuser=新增已激活 +metrics.hasactivateuser=新增有贡献活动 +metrics.newregistnotactiveuser=新增未激活 +metrics.averageuser=平均新增用户 +metrics.newuseractiveindex=新增用户激活率 +metrics.totalregistuser=累计注册用户 +metrics.totalregistuser=累计已激活 +metrics.totalregistuser=累计有贡献活动 + [settings] profile=个人信息 account=账号 diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2280e8288..240613566 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -19,6 +19,53 @@ const ( PAGE_SIZE = 2000 ) +func getUserMetricsExcelHeader(ctx *context.Context) map[string]string { + excelHeader := make([]string, 0) + excelHeader = append(excelHeader, ctx.Tr("user.static.id")) + excelHeader = append(excelHeader, ctx.Tr("user.static.name")) + excelHeader = append(excelHeader, ctx.Tr("user.static.UserIndex")) + excelHeader = append(excelHeader, ctx.Tr("user.static.UserIndexPrimitive")) + excelHeader = append(excelHeader, ctx.Tr("user.static.codemergecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commitcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.issuecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commentcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.focusrepocount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.starrepocount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.logincount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.watchedcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commitcodesize")) + excelHeader = append(excelHeader, ctx.Tr("user.static.solveissuecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.encyclopediascount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.createrepocount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.openiindex")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainTaskNum")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) + + excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectedDataset")) + excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendDataset")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectImage")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CollectedImage")) + excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendImage")) + + excelHeader = append(excelHeader, ctx.Tr("user.static.registdate")) + excelHeader = append(excelHeader, ctx.Tr("user.static.countdate")) + + excelHeaderMap := make(map[string]string, 0) + var i byte + i = 0 + for _, value := range excelHeader { + excelColumn := getColumn(i) + fmt.Sprint(1) + log.Info("excelColumn=" + excelColumn) + excelHeaderMap[excelColumn] = value + i++ + } + return excelHeaderMap +} + func getExcelHeader(ctx *context.Context) map[string]string { excelHeader := make([]string, 0) excelHeader = append(excelHeader, ctx.Tr("user.static.id")) @@ -205,11 +252,36 @@ func QueryMetrics(ctx *context.Context) { endDate := ctx.Query("endDate") startTime, _ := time.ParseInLocation("2006-01-02", startDate, time.Local) endTime, _ := time.ParseInLocation("2006-01-02", endDate, time.Local) - result, count := models.QueryMetrics(startTime.Unix(), endTime.Unix()) - mapInterface := make(map[string]interface{}) - mapInterface["data"] = result - mapInterface["count"] = count - ctx.JSON(http.StatusOK, mapInterface) + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + IsReturnFile := ctx.QueryBool("IsReturnFile") + + if IsReturnFile { + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.metrics.sheetname") + //index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + dataHeader := getUserMetricsExcelHeader(ctx) + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + } else { + result, count := models.QueryMetricsPage(startTime.Unix(), endTime.Unix(), page, pageSize) + mapInterface := make(map[string]interface{}) + mapInterface["data"] = result + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) + } + } func QueryRankingList(ctx *context.Context) { From 8342c48013a1e9f5958142e9090c7c344f3a471c Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 28 Apr 2022 16:02:59 +0800 Subject: [PATCH 03/62] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 8 +++++++- models/user_business_struct.go | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 7e9e877c2..72465daa7 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -885,7 +885,11 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("query user error. return.") } useMetrics.TotalUser = int(count) - useMetrics.ActivateIIndex = float64(useMetrics.ActivateRegistUser) / float64(useMetrics.ActivateRegistUser+useMetrics.NotActivateRegistUser) + if useMetrics.ActivateRegistUser+useMetrics.NotActivateRegistUser == 0 { + useMetrics.ActivateIndex = 0 + } else { + useMetrics.ActivateIndex = float64(useMetrics.ActivateRegistUser) / float64(useMetrics.ActivateRegistUser+useMetrics.NotActivateRegistUser) + } statictisSess.Insert(&useMetrics) //update new user activity updateNewUserAcitivity(userNewAddActivity, statictisSess) @@ -893,7 +897,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) { + log.Info("go here len=" + fmt.Sprint(len(currentUserActivity))) for key, value := range currentUserActivity { + log.Info("key=" + fmt.Sprint(key) + " ") useMetrics := &UserMetrics{CountDate: key} has, err := statictisSess.Get(useMetrics) if err == nil && has { diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 5560cb19a..8f4c94636 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -403,12 +403,12 @@ type UserMetrics struct { CountDate int64 `xorm:"pk"` ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - ActivateIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + ActivateIndex float64 `xorm:"NOT NULL DEFAULT 0"` HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` TotalUser int `xorm:"NOT NULL DEFAULT 0"` TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` TotalHasActivityUser int `xorm:"NOT NULL DEFAULT 0"` DisplayDate string `xorm:"-"` DaysForMonth int `xorm:"NOT NULL DEFAULT 0"` - HasActivityUserJson string `xorm:"NULL"` + HasActivityUserJson string `xorm:"text NULL"` } From 6eb83a3da7d5d53e3e4aa82c23e08ad454e36e1a Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 28 Apr 2022 16:30:40 +0800 Subject: [PATCH 04/62] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 72465daa7..b33625e06 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -901,8 +901,9 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, stati for key, value := range currentUserActivity { log.Info("key=" + fmt.Sprint(key) + " ") useMetrics := &UserMetrics{CountDate: key} - has, err := statictisSess.Get(useMetrics) - if err == nil && has { + _, err := statictisSess.Get(useMetrics) + log.Info("json=" + useMetrics.HasActivityUserJson + " totalUser=" + fmt.Sprint(useMetrics.TotalUser) + " len(value)=" + fmt.Sprint(len(value))) + if err == nil { userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",") for _, userIdStr := range userIdArrays { userIdInt, err := strconv.ParseInt(userIdStr, 10, 64) @@ -914,10 +915,11 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, stati for _, tmpValue := range value { userIdArray += fmt.Sprint(tmpValue) + "," } + useMetrics.HasActivityUser = len(value) if len(userIdArray) > 0 { useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] - + log.Info("useMetrics.HasActivityUserJson=" + useMetrics.HasActivityUserJson) } statictisSess.Update(useMetrics) } From 3fedbeec1d5b6a688178b34a7edcfa780fd94339 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 28 Apr 2022 16:45:38 +0800 Subject: [PATCH 05/62] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index b33625e06..93bc640c0 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -901,9 +901,9 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, stati for key, value := range currentUserActivity { log.Info("key=" + fmt.Sprint(key) + " ") useMetrics := &UserMetrics{CountDate: key} - _, err := statictisSess.Get(useMetrics) + has, err := statictisSess.Get(useMetrics) log.Info("json=" + useMetrics.HasActivityUserJson + " totalUser=" + fmt.Sprint(useMetrics.TotalUser) + " len(value)=" + fmt.Sprint(len(value))) - if err == nil { + if err == nil && has { userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",") for _, userIdStr := range userIdArrays { userIdInt, err := strconv.ParseInt(userIdStr, 10, 64) @@ -921,7 +921,9 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, stati useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] log.Info("useMetrics.HasActivityUserJson=" + useMetrics.HasActivityUserJson) } - statictisSess.Update(useMetrics) + updateSql := "update public.user_metrics set has_activity_user_json=" + useMetrics.HasActivityUserJson + " where count_date=" + fmt.Sprint(key) + statictisSess.Exec(updateSql) + //statictisSess.Update(useMetrics) } } } From b6def70bb4e78a38fc3e309ca81459a54d0e4bae Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 28 Apr 2022 17:01:38 +0800 Subject: [PATCH 06/62] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 93bc640c0..311c3ae43 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -897,12 +897,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) { - log.Info("go here len=" + fmt.Sprint(len(currentUserActivity))) for key, value := range currentUserActivity { - log.Info("key=" + fmt.Sprint(key) + " ") useMetrics := &UserMetrics{CountDate: key} has, err := statictisSess.Get(useMetrics) - log.Info("json=" + useMetrics.HasActivityUserJson + " totalUser=" + fmt.Sprint(useMetrics.TotalUser) + " len(value)=" + fmt.Sprint(len(value))) if err == nil && has { userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",") for _, userIdStr := range userIdArrays { @@ -915,15 +912,12 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, stati for _, tmpValue := range value { userIdArray += fmt.Sprint(tmpValue) + "," } - useMetrics.HasActivityUser = len(value) if len(userIdArray) > 0 { useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] - log.Info("useMetrics.HasActivityUserJson=" + useMetrics.HasActivityUserJson) } - updateSql := "update public.user_metrics set has_activity_user_json=" + useMetrics.HasActivityUserJson + " where count_date=" + fmt.Sprint(key) + updateSql := "update public.user_metrics set has_activity_user_json=" + useMetrics.HasActivityUserJson + ",has_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + " where count_date=" + fmt.Sprint(key) statictisSess.Exec(updateSql) - //statictisSess.Update(useMetrics) } } } From fb5513fdfdbf93ccf1751fc5456d94e41d46f3f1 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 29 Apr 2022 09:56:25 +0800 Subject: [PATCH 07/62] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 3 ++- models/user_business_struct.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 311c3ae43..d5ac13fbb 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -876,6 +876,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, useMetrics.ActivateRegistUser = getMapKeyStringValue("ActivateRegistUser", userMetrics) useMetrics.HasActivityUser = getMapKeyStringValue("HasActivityUser", userMetrics) + useMetrics.RegistActivityUser = 0 useMetrics.NotActivateRegistUser = getMapKeyStringValue("NotActivateRegistUser", userMetrics) useMetrics.TotalActivateRegistUser = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) useMetrics.TotalHasActivityUser = getMapKeyStringValue("TotalHasActivityUser", userMetrics) @@ -916,7 +917,7 @@ func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, stati if len(userIdArray) > 0 { useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] } - updateSql := "update public.user_metrics set has_activity_user_json=" + useMetrics.HasActivityUserJson + ",has_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + " where count_date=" + fmt.Sprint(key) + updateSql := "update public.user_metrics set has_activity_user_json=" + useMetrics.HasActivityUserJson + ",regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + " where count_date=" + fmt.Sprint(key) statictisSess.Exec(updateSql) } } diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 8f4c94636..ad4ff0bdb 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -404,6 +404,7 @@ type UserMetrics struct { ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` ActivateIndex float64 `xorm:"NOT NULL DEFAULT 0"` + RegistActivityUser int `xorm:"NOT NULL DEFAULT 0"` HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` TotalUser int `xorm:"NOT NULL DEFAULT 0"` TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` From d3252d42188d9d8f016b05c0bd2dc140ebff7d78 Mon Sep 17 00:00:00 2001 From: wangjr Date: Fri, 29 Apr 2022 17:21:12 +0800 Subject: [PATCH 08/62] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=95=B0=E5=AD=97?= =?UTF-8?q?=E7=9C=8B=E6=9D=BF=E6=96=B0=E5=A2=9E=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web_src/js/components/DataAnalysis.vue | 151 ++-- web_src/js/components/DataAnalysis1.vue | 158 ++++ web_src/js/components/Overview.vue | 5 + web_src/js/components/ProAnalysis.vue | 2 +- web_src/js/components/ProAnalysis1.vue | 1336 +++++++++++++++++++++++++++++++ web_src/js/components/ProTend.vue | 1000 +++++++++++++++++++++++ web_src/js/components/UserAnalysis.vue | 2 +- web_src/js/components/UserAnalysis1.vue | 520 ++++++++++++ web_src/js/index.js | 4 +- web_src/js/router/index.js | 45 ++ web_src/js/router/main_.js | 17 + 11 files changed, 3187 insertions(+), 53 deletions(-) create mode 100644 web_src/js/components/DataAnalysis1.vue create mode 100644 web_src/js/components/Overview.vue create mode 100644 web_src/js/components/ProAnalysis1.vue create mode 100644 web_src/js/components/ProTend.vue create mode 100644 web_src/js/components/UserAnalysis1.vue create mode 100644 web_src/js/router/index.js create mode 100644 web_src/js/router/main_.js diff --git a/web_src/js/components/DataAnalysis.vue b/web_src/js/components/DataAnalysis.vue index 186a216c2..68af96239 100755 --- a/web_src/js/components/DataAnalysis.vue +++ b/web_src/js/components/DataAnalysis.vue @@ -1,4 +1,4 @@ -