From 2531431009fda56338b9f30ed3d09c5fbdcfd3d2 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 14:19:06 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E6=94=B9=E7=94=A8=E6=88=B7=E8=BF=90=E8=90=A5?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E7=9B=B8=E5=85=B3Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 33 +++++++++++++++++++-- models/models.go | 4 ++- models/user_business_analysis.go | 64 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 0b9a0d552..6dbbf00f2 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -1,6 +1,8 @@ package models import ( + "fmt" + "code.gitea.io/gitea/modules/log" "xorm.io/xorm" ) @@ -10,11 +12,18 @@ type CustomMigration struct { Migrate func(*xorm.Engine) error } +type CustomMigrationStatic struct { + Description string + Migrate func(*xorm.Engine, *xorm.Engine) error +} + var customMigrations = []CustomMigration{ {"Custom v1 Topic struct change to support chinese", syncTopicStruct}, } -var customMigrationsStatic = []CustomMigration{} +var customMigrationsStatic = []CustomMigrationStatic{ + {"Delete zuzhi user history data ", deleteNotDisplayUser}, +} func MigrateCustom(x *xorm.Engine) { @@ -29,10 +38,10 @@ func MigrateCustom(x *xorm.Engine) { } -func MigrateCustomStatic(x *xorm.Engine) { +func MigrateCustomStatic(x *xorm.Engine, static *xorm.Engine) { for _, m := range customMigrationsStatic { log.Info("Migration: %s", m.Description) - if err := m.Migrate(x); err != nil { + if err := m.Migrate(x, static); err != nil { log.Error("Migration: %v", err) @@ -47,3 +56,21 @@ func syncTopicStruct(x *xorm.Engine) error { _, err := x.Exec(query) return err } + +func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { + + querySQL := "select id,name from public.user where type=1" + rows, err := x.Query(querySQL) + if err != nil { + log.Info("select db failed,err:", err) + return err + } + + for i, userRow := range rows { + log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["name"])) + deleteSql := "delete from user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" + static.Exec(deleteSql) + } + + return nil +} diff --git a/models/models.go b/models/models.go index 86a091bca..6631a9124 100755 --- a/models/models.go +++ b/models/models.go @@ -133,6 +133,8 @@ func init() { new(FileChunk), new(BlockChain), new(RecommendOrg), + new(AiModelManage), + new(TrainjobConfigDetail), ) tablesStatistic = append(tablesStatistic, @@ -230,7 +232,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil { return fmt.Errorf("newEngine statistic failed: %v", err) } - MigrateCustomStatic(xStatistic) + MigrateCustomStatic(x, xStatistic) HasEngine = true diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a4a155ba7..ac51c2816 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1,6 +1,7 @@ package models import ( + "encoding/json" "fmt" "time" @@ -267,6 +268,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) + OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -361,6 +363,12 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti dateRecord.LoginCount = LoginCountMap[dateRecord.ID] } + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = int(OpenIIndexMap[dateRecord.ID] * 100) + } + dateRecord.CommitModelCount = 0 statictisSess.Insert(&dateRecord) @@ -545,6 +553,62 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoStatisticList := make([]*RepoStatistic, 0) + statictisSess.Find(&repoStatisticList) + repoOpenIIndexMap := make(map[int64]float64) + log.Info("query repo_statistic size=" + fmt.Sprint(len(repoStatisticList))) + for _, repoRecord := range repoStatisticList { + if _, ok := repoOpenIIndexMap[repoRecord.RepoID]; !ok { + repoOpenIIndexMap[repoRecord.RepoID] = repoRecord.RadarTotal + } + } + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false") + repoList := make([]*Repository, 0) + sess.Find(&repoList) + + userMap := make(map[int64]float64) + + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := userMap[repoRecord.OwnerID]; !ok { + if _, ok := repoOpenIIndexMap[repoRecord.ID]; ok { + userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID] + } + } + } + + //query collaboration + sess.Select("repo_id,user_id,mode").Table("collaboration") + collaborationList := make([]*Collaboration, 0) + sess.Find(&collaborationList) + + log.Info("query collaborationList size=" + fmt.Sprint(len(collaborationList))) + + for _, collaborationRecord := range collaborationList { + if _, ok := userMap[collaborationRecord.UserID]; !ok { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { + userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID] + } + } else { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { + userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID] + } + } + } + + userMapJson, _ := json.Marshal(userMap) + log.Info("userMapJson=" + string(userMapJson)) + + return userMap +} + func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() From 998e48d75ea6f0e41d57999d3860f7163ad55e15 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 14:23:03 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E8=BF=90=E8=90=A5=E7=BB=9F=E8=AE=A1=E7=9B=B8=E5=85=B3=E4=BB=A3?= =?UTF-8?q?=E7=A0=81Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/models.go b/models/models.go index 6631a9124..4d39e2f14 100755 --- a/models/models.go +++ b/models/models.go @@ -133,8 +133,6 @@ func init() { new(FileChunk), new(BlockChain), new(RecommendOrg), - new(AiModelManage), - new(TrainjobConfigDetail), ) tablesStatistic = append(tablesStatistic, From 93e51c25d0a05723c8dbe283c7d761d9c04122a0 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 14:34:22 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E8=BF=90=E8=90=A5=E7=BB=9F=E8=AE=A1=E7=9B=B8=E5=85=B3=E4=BB=A3?= =?UTF-8?q?=E7=A0=81Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 9 +++++++++ models/user_business_analysis.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 6dbbf00f2..ec0e5f75e 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -22,6 +22,7 @@ var customMigrations = []CustomMigration{ } var customMigrationsStatic = []CustomMigrationStatic{ + {"Alter user static table field type ", alterUserStaticTable}, {"Delete zuzhi user history data ", deleteNotDisplayUser}, } @@ -57,6 +58,14 @@ func syncTopicStruct(x *xorm.Engine) error { return err } +func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error { + alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision" + + _, err := static.Exec(alterSql) + return err + +} + func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { querySQL := "select id,name from public.user where type=1" diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ac51c2816..f30631ae1 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -65,7 +65,7 @@ type UserBusinessAnalysis struct { LoginCount int `xorm:"NOT NULL DEFAULT 0"` //openi index - OpenIIndex int `xorm:"NOT NULL DEFAULT 0"` + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` //user Email string `xorm:"NOT NULL"` @@ -366,7 +366,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { dateRecord.OpenIIndex = 0 } else { - dateRecord.OpenIIndex = int(OpenIIndexMap[dateRecord.ID] * 100) + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] } dateRecord.CommitModelCount = 0