From 546226167e7db67476a1de5a3ec4e9a5a5877a27 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 7 Dec 2021 14:50:22 +0800 Subject: [PATCH 01/31] download model --- routers/repo/cloudbrain.go | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a3af588fb..0689741d0 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -251,13 +251,9 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { downloadCode(repo, codePath) uploadCodeToMinio(codePath + "/", jobName, "/code/") - modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath - err = os.MkdirAll(modelPath, os.ModePerm) - if err != nil { - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) - return - } + modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" + mkModelPath(modelPath) + uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath + "/") benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { @@ -283,7 +279,9 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } - err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, getMinioPath(jobName, cloudbrain.ModelMountPath + "/"), + getMinioPath(jobName, cloudbrain.BenchMarkMountPath + "/"), getMinioPath(jobName, cloudbrain.Snn4imagenetMountPath + "/"), + getMinioPath(jobName, cloudbrain.BrainScoreMountPath + "/"), jobType, gpuQueue, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) @@ -607,7 +605,7 @@ func getImages(ctx *context.Context, imageType string) { func getModelDirs(jobName string, parentDir string) (string, error) { var req string - modelActualPath := setting.JobPath + jobName + "/model/" + modelActualPath := getMinioPath(jobName, cloudbrain.ModelMountPath + "/") if parentDir == "" { req = "baseDir=" + modelActualPath } else { @@ -617,6 +615,10 @@ func getModelDirs(jobName string, parentDir string) (string, error) { return getDirs(req) } +func getMinioPath(jobName, suffixPath string) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + jobName + suffixPath +} + func CloudBrainDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") @@ -772,6 +774,32 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { return nil } +func mkModelPath(modelPath string) error { + err := os.MkdirAll(modelPath, os.ModePerm) + if err != nil { + log.Error("MkdirAll(%s) failed:%v", modelPath, err) + return err + } + + fileName := modelPath + "README" + f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + log.Error("OpenFile failed", err.Error()) + return err + } + + defer f.Close() + + _, err = f.WriteString("You can put the model file into this directory and download it by the web page.") + if err != nil { + log.Error("WriteString failed", err.Error()) + return err + } + + return nil +} + + func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil { From fa87516be1d3a4dd1736e403f336cbd34fce6fe5 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 17:51:06 +0800 Subject: [PATCH 02/31] =?UTF-8?q?=E6=8F=90=E4=BE=9B=E6=8E=A8=E8=8D=90?= =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E5=8F=8A=E7=BB=84=E7=BB=87=E7=9A=84=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/setting/setting.go | 6 ++++++ routers/home.go | 34 ++++++++++++++++++++++++++++++++++ routers/repo/http.go | 2 +- routers/routes/routes.go | 2 ++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0ab1bbc88..ce9738f08 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,6 +433,9 @@ var ( AuthUser string AuthPassword string + //home page + RecommentRepoAddr string + //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -1225,6 +1228,9 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") + sec = Cfg.Section("homepage") + RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") + sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") diff --git a/routers/home.go b/routers/home.go index d2c93c771..eb76088c0 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,6 +7,8 @@ package routers import ( "bytes" + "fmt" + "io/ioutil" "net/http" "strings" @@ -511,3 +513,35 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } +func RecommendOrgFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "organizations" + recommendFromPromote(ctx, url) +} + +func recommendFromPromote(ctx *context.Context, url string) { + resp, err := http.Get(url) + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + + allLineStr := string(bytes) + lines := strings.Split(allLineStr, "\n") + for i, line := range lines { + log.Info("i=" + fmt.Sprint(i) + " line=" + line) + } + ctx.JSON(http.StatusOK, lines) +} + +func RecommendRepoFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "projects" + recommendFromPromote(ctx, url) +} diff --git a/routers/repo/http.go b/routers/repo/http.go index ad2abf567..87406a2c3 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -257,7 +257,6 @@ func HTTP(ctx *context.Context) { models.EnvPusherID + fmt.Sprintf("=%d", authUser.ID), models.EnvIsDeployKey + "=false", } - if !authUser.KeepEmailPrivate { environ = append(environ, models.EnvPusherEmail+"="+authUser.Email) } @@ -559,6 +558,7 @@ func serviceRPC(h serviceHandler, service string) { if service == "receive-pack" { cmd.Env = append(os.Environ(), h.environ...) } + cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 680b0601c..6ea751531 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) + m.Get("/recommend/org", routers.RecommendOrgFromPromote) + m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") From c8899105b99ea4339ac5fa119c7011a7be7b42f6 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 7 Dec 2021 18:01:26 +0800 Subject: [PATCH 03/31] opt --- modules/setting/setting.go | 18 ++++++++++++------ routers/repo/cloudbrain.go | 21 +++++++++++---------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0ab1bbc88..f7b78235b 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -451,18 +451,21 @@ var ( //benchmark config IsBenchmarkEnabled bool - BenchmarkCode string + BenchmarkOwner string + BenchmarkName string BenchmarkServerHost string BenchmarkCategory string //snn4imagenet config IsSnn4imagenetEnabled bool - Snn4imagenetCode string + Snn4imagenetOwner string + Snn4imagenetName string Snn4imagenetServerHost string //snn4imagenet config IsBrainScoreEnabled bool - BrainScoreCode string + BrainScoreOwner string + BrainScoreName string BrainScoreServerHost string //blockchain config @@ -1238,18 +1241,21 @@ func NewContext() { sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("") + BenchmarkOwner = sec.Key("OWNER").MustString("") + BenchmarkName = sec.Key("NAME").MustString("") BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("") + Snn4imagenetOwner = sec.Key("OWNER").MustString("") + Snn4imagenetName = sec.Key("NAME").MustString("") Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("") + BrainScoreOwner = sec.Key("OWNER").MustString("") + BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 65a475e5b..5c2f5b5b7 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "os" - "os/exec" "regexp" "sort" "strconv" @@ -263,19 +262,19 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuType = gpuInfo.Value } } - downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BrainScoreName, benchmarkPath, form.BenchmarkCategory, gpuType) uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "") uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "") uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } @@ -700,19 +699,21 @@ func downloadCode(repo *models.Repository, codePath string) error { return nil } -func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { +func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) return err } - command := "git clone " + gitPath + " " + codePath - cmd := exec.Command("/bin/bash", "-c", command) - _, err = cmd.Output() - + repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName) if err != nil { - log.Error("exec.Command(%s) failed:%v", command, err) + log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error()) + return err + } + + if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } From 7fbddc95e0a845241762c631c83c463fdbf66801 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 8 Dec 2021 09:29:29 +0800 Subject: [PATCH 04/31] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=95=8C=E9=9D=A2?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E4=B9=9F=E5=AF=B9=E7=9C=9F=E5=AE=9E=E7=9A=84?= =?UTF-8?q?git=20clone=E6=AC=A1=E6=95=B0=E5=8A=A01?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 437521d5a..a182e9087 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -532,6 +532,7 @@ func Download(ctx *context.Context) { } ctx.Repo.Repository.IncreaseCloneCnt() + ctx.Repo.Repository.IncreaseGitCloneCnt() ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } From 3c416724560310d752464965e4e1ac004aa434f3 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 8 Dec 2021 10:00:37 +0800 Subject: [PATCH 05/31] #687 fix bug --- models/user.go | 2 +- options/locale/locale_zh-CN.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/user.go b/models/user.go index ec96420dc..b362472e8 100755 --- a/models/user.go +++ b/models/user.go @@ -929,7 +929,7 @@ var ( "template", "user", "vendor", - "dashbord", + "dashboard", "operation", "blockchain", "avatar", diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index e26e07903..15e41b7ea 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -848,7 +848,7 @@ modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 modelarts.train_job.parameter_setting_info=参数信息 modelarts.train_job.fast_parameter_setting=一键式参数配置 -modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 +modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 modelarts.train_job.fast_parameter_setting_config_link=这里 modelarts.train_job.frames=常用框架 modelarts.train_job.algorithm_origin=算法来源 @@ -1965,7 +1965,7 @@ team_unit_desc=允许访问项目单元 team_unit_disabled=(已禁用) form.name_reserved=组织名称 '%s' 是被保留的。 -form.name_pattern_not_allowed=项目名称中不允许使用 "%s"。 +form.name_pattern_not_allowed=组织名称中不允许使用 "%s"。 form.create_org_not_allowed=此账号禁止创建组织 settings=组织设置 From 57781347b365c0dc88d234e021465ea2419fcb2b Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 10:30:33 +0800 Subject: [PATCH 06/31] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B8=B8=E5=AE=A2?= =?UTF-8?q?=E6=9F=A5=E7=9C=8B=E4=BA=91=E8=84=91=E5=88=97=E8=A1=A8=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 299133290..cbc9580f9 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -53,6 +53,9 @@ func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { } func CanCreateOrDebugJob(ctx *context.Context) bool { + if !ctx.IsSigned { + return false + } return ctx.Repo.CanWrite(models.UnitTypeCloudBrain) } From 1bf092295f01e200c1aaf1b9c769ac0a6c6f195c Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 8 Dec 2021 10:33:52 +0800 Subject: [PATCH 07/31] #999 fix bug --- models/dataset.go | 4 ++++ routers/user/profile.go | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/models/dataset.go b/models/dataset.go index 402a548ef..2b3de752b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -93,6 +93,7 @@ type SearchDatasetOptions struct { IncludePublic bool ListOptions SearchOrderBy + IsOwner bool } func CreateDataset(dataset *Dataset) (err error) { @@ -150,6 +151,9 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) + if !opts.IsOwner { + cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) + } } return cond diff --git a/routers/user/profile.go b/routers/user/profile.go index 0e29b6117..9da2bfa22 100755 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -210,10 +210,15 @@ func Profile(ctx *context.Context) { total = int(count) case "datasets": + var isOwner = false + if ctx.User != nil && ctx.User.ID == ctxUser.ID { + isOwner = true + } datasetSearchOptions := &models.SearchDatasetOptions{ Keyword: keyword, OwnerID: ctxUser.ID, SearchOrderBy: orderBy, + IsOwner: isOwner, ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.ExplorePagingNum, From 4d938521cde74cda45e63fb2e2c2ee934e807cef Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 10:47:08 +0800 Subject: [PATCH 08/31] =?UTF-8?q?=E5=85=88=E7=94=A8=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E5=88=A4=E6=96=AD=E6=9D=83=E9=99=90=EF=BC=8C?= =?UTF-8?q?=E7=84=B6=E5=90=8E=E5=9C=A8=E7=94=A8=E8=BA=AB=E4=BB=BD=E5=88=A4?= =?UTF-8?q?=E6=96=AD=EF=BC=8C=E6=8F=90=E9=AB=98=E6=95=88=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index cbc9580f9..8a5e11997 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -2,6 +2,7 @@ package cloudbrain import ( "errors" + "strconv" "code.gitea.io/gitea/modules/setting" @@ -30,10 +31,13 @@ var ( ) func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { - + log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner())) + log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin())) if err != nil { + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() } else { + log.Info("is job creator:" + strconv.FormatBool(ctx.User.ID == job.UserID)) return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID } @@ -44,11 +48,16 @@ func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { return false } + if !ctx.IsSigned { + return false + } return isAdminOrOwnerOrJobCreater(ctx, job, nil) } func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { - + if !ctx.IsSigned { + return false + } return isAdminOrOwnerOrJobCreater(ctx, job, nil) } From ace4ec5ca6b8a3ff1e4667e92916ead67e31f6ad Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 10:59:31 +0800 Subject: [PATCH 09/31] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 8a5e11997..96dfce615 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -31,6 +31,9 @@ var ( ) func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { + if !ctx.IsSigned { + return false + } log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner())) log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin())) if err != nil { @@ -48,16 +51,12 @@ func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { return false } - if !ctx.IsSigned { - return false - } + return isAdminOrOwnerOrJobCreater(ctx, job, nil) } func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { - if !ctx.IsSigned { - return false - } + return isAdminOrOwnerOrJobCreater(ctx, job, nil) } @@ -74,7 +73,9 @@ func CanModifyJob(ctx *context.Context, job *models.Cloudbrain) bool { } func isAdminOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { - + if !ctx.IsSigned { + return false + } if err != nil { return ctx.IsUserSiteAdmin() } else { @@ -125,7 +126,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, log.Error("no such resourceSpecId(%d)", resourceSpecId, ctx.Data["MsgID"]) return errors.New("no such resourceSpec") } - + jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -210,8 +211,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, JobName: jobName, SubTaskName: SubTaskName, JobType: jobType, - Type: models.TypeCloudBrainOne, - Uuid: uuid, + Type: models.TypeCloudBrainOne, + Uuid: uuid, }) if err != nil { From 19f9541eb74b4b52701d694ed03229fa4fc1f60a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 11:25:33 +0800 Subject: [PATCH 10/31] =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=9D=83=E9=99=90?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 11 +---------- routers/repo/cloudbrain.go | 2 +- routers/repo/modelarts.go | 4 ++-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 96dfce615..0f1c700d2 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -46,16 +46,7 @@ func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, er } -func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { - - if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { - return false - } - - return isAdminOrOwnerOrJobCreater(ctx, job, nil) -} - -func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { +func CanDeleteJob(ctx *context.Context, job *models.Cloudbrain) bool { return isAdminOrOwnerOrJobCreater(ctx, job, nil) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a3af588fb..ae7163d77 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -79,7 +79,7 @@ func CloudBrainIndex(ctx *context.Context) { ciTasks[i].CanDebug = false } - ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f0b807d96..483efbc20 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -74,7 +74,7 @@ func NotebookIndex(ctx *context.Context) { } else { ciTasks[i].CanDebug = false } - ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) @@ -306,7 +306,7 @@ func TrainJobIndex(ctx *context.Context) { } for i, task := range tasks { - tasks[i].CanDel = cloudbrain.CanDeleteTrainJob(ctx, &task.Cloudbrain) + tasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) } From 6333e3a4e1a03da16b88ec0bc9c51d16a9c4c5d5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 8 Dec 2021 11:35:15 +0800 Subject: [PATCH 11/31] fix issue --- options/locale/locale_en-US.ini | 2 ++ options/locale/locale_zh-CN.ini | 2 +- templates/repo/header.tmpl | 9 +++++++-- templates/repo/settings/options.tmpl | 17 ++++++++--------- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3ab3a00c9..48a010b73 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -778,6 +778,8 @@ datasets = Datasets datasets.desc = Enable Dataset cloudbrain_helper=Use GPU/NPU resources to open notebooks, model training tasks, etc. +model_manager = Model + debug=Debug stop=Stop delete=Delete diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 98d581fb7..b1c339080 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -782,7 +782,7 @@ datasets=数据集 datasets.desc=数据集功能 cloudbrain_helper=使用GPU/NPU资源，开启Notebook、模型训练任务等 -model_manager = 模型管理 +model_manager = 模型 model_noright=无权限操作 debug=调试 diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index b84f4771f..f5376b07c 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -134,7 +134,8 @@ {{if .Permission.CanRead $.UnitTypeDatasets}} - {{svg "octicon-inbox" 16}} {{.i18n.Tr "datasets"}} + + {{.i18n.Tr "datasets"}} {{end}} {{if .Permission.CanRead $.UnitTypeModelManage}} @@ -145,7 +146,11 @@ {{end}} {{if .Permission.CanRead $.UnitTypeCloudBrain}} - {{svg "octicon-server" 16}} {{.i18n.Tr "repo.cloudbrain"}} + + + {{.i18n.Tr "repo.cloudbrain"}} + + {{end}} + + {{range .Orgs}} +
    + + {{if (or .Visibility.IsPublic (and ($.SignedUser) (or .Visibility.IsLimited (and (.IsUserPartOfOrg $.SignedUserID) .Visibility.IsPrivate) ($.IsAdmin))))}} +
  • + +
    • +
  • + {{.Name}} +
    • + +
  • + + + {{.NumMembers}} +
    • +
  • + + {{.NumRepos}} +
    • + {{end}} + +
+ {{end}} {{end}} {{if and .IsSigned (ne .SignedUserName .Owner.Name)}} @@ -79,13 +109,6 @@ {{end}} -
-
    -
  • -
        -
        • -
      -
      @@ -155,3 +178,22 @@
      {{template "base/footer" .}} + + \ No newline at end of file From ab16e888d269704de98def56db03957f346b2c88 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 09:07:24 +0800 Subject: [PATCH 19/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 43 ++++++++++++++++++++--------------------- modules/setting/setting.go | 6 ------ routers/home.go | 42 ---------------------------------------- routers/repo/ai_model_manage.go | 29 +++++++++++++++++++++------ routers/routes/routes.go | 2 -- 5 files changed, 44 insertions(+), 78 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index af96444ac..ace51002d 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -11,30 +11,30 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string `xorm:"NULL"` + ID string `xorm:"pk"` + Name string `xorm:"INDEX NOT NULL"` + Version string `xorm:"NOT NULL"` + VersionCount int `xorm:"NOT NULL DEFAULT 0"` + New int `xorm:"NOT NULL"` + Type int `xorm:"NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Description string `xorm:"varchar(2000)"` + Label string `xorm:"varchar(1000)"` + Path string `xorm:"varchar(400) NOT NULL"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0"` + Engine int64 `xorm:"NOT NULL DEFAULT 0"` + Status int `xorm:"NOT NULL DEFAULT 0"` + Accuracy string `xorm:"varchar(1000)"` + AttachmentId string `xorm:"NULL"` + RepoId int64 `xorm:"INDEX NULL"` + CodeBranch string `xorm:"varchar(400) NULL"` + CodeCommitID string `xorm:"NULL"` + UserId int64 `xorm:"NOT NULL"` + UserName string UserRelAvatarLink string `xorm:"NULL"` TrainTaskInfo string `xorm:"text NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` IsCanOper bool } @@ -197,7 +197,6 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { Find(&aiModelManages); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } - sess.Close() return aiModelManages, count, nil } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2d70e47b1..dd51623c1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,9 +433,6 @@ var ( AuthUser string AuthPassword string - //home page - RecommentRepoAddr string - //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -1232,9 +1229,6 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") - sec = Cfg.Section("homepage") - RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") - sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") diff --git a/routers/home.go b/routers/home.go index 7cc353ed8..d2c93c771 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,8 +7,6 @@ package routers import ( "bytes" - "fmt" - "io/ioutil" "net/http" "strings" @@ -513,43 +511,3 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } -func RecommendOrgFromPromote(ctx *context.Context) { - url := setting.RecommentRepoAddr + "organizations" - recommendFromPromote(ctx, url) -} - -func recommendFromPromote(ctx *context.Context, url string) { - resp, err := http.Get(url) - if err != nil { - log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return - } - bytes, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return - } - - allLineStr := string(bytes) - lines := strings.Split(allLineStr, "\n") - result := make([]string, len(lines)) - for i, line := range lines { - - tmpIndex := strings.Index(line, ".") - log.Info("i=" + fmt.Sprint(i) + " line=" + line + " tmpIndex=" + fmt.Sprint(tmpIndex)) - if tmpIndex == -1 { - result[i] = strings.Trim(line, " ") - } else { - result[i] = strings.Trim(line[tmpIndex+1:], " ") - } - } - ctx.JSON(http.StatusOK, result) -} - -func RecommendRepoFromPromote(ctx *context.Context) { - url := setting.RecommentRepoAddr + "projects" - recommendFromPromote(ctx, url) -} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index c6ec6c6ae..d3afe61a4 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -28,7 +28,6 @@ const ( func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, ctx *context.Context) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) - //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { log.Info("query task error." + err.Error()) return err @@ -71,7 +70,6 @@ func saveModelByParameters(jobId string, versionName string, name string, versio log.Info("accuracyJson=" + string(accuracyJson)) aiTaskJson, _ := json.Marshal(aiTask) - //taskConfigInfo,err := models.GetCloudbrainByJobIDAndVersionName(jobId,aiTask.VersionName) model := &models.AiModelManage{ ID: id, Version: version, @@ -86,7 +84,6 @@ func saveModelByParameters(jobId string, versionName string, name string, versio AttachmentId: aiTask.Uuid, RepoId: aiTask.RepoID, UserId: ctx.User.ID, - UserName: ctx.User.Name, UserRelAvatarLink: ctx.User.RelAvatarLink(), CodeBranch: aiTask.BranchName, CodeCommitID: aiTask.CommitID, @@ -399,11 +396,22 @@ func ShowOneVersionOtherModel(ctx *context.Context) { repoId := ctx.Repo.Repository.ID name := ctx.Query("name") aimodels := models.QueryModelByName(name, repoId) - for _, model := range aimodels { + + userIds := make([]int64, len(aimodels)) + for i, model := range aimodels { log.Info("model=" + model.Name) log.Info("model.UserId=" + fmt.Sprint(model.UserId)) model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + + userNames, err := models.GetUserNamesByIDs(userIds) + if err == nil { + for i, model := range aimodels { + model.UserName = userNames[i] + } } + if len(aimodels) > 0 { ctx.JSON(200, aimodels[1:]) } else { @@ -431,7 +439,7 @@ func isOper(ctx *context.Context, modelUserId int64) bool { if ctx.User == nil { return false } - if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { + if ctx.User.IsAdmin || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { return true } return false @@ -463,10 +471,19 @@ func ShowModelPageInfo(ctx *context.Context) { return } - for _, model := range modelResult { + userIds := make([]int64, len(modelResult)) + for i, model := range modelResult { log.Info("model=" + model.Name) log.Info("model.UserId=" + fmt.Sprint(model.UserId)) model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + + userNames, err := models.GetUserNamesByIDs(userIds) + if err == nil { + for i, model := range modelResult { + model.UserName = userNames[i] + } } mapInterface := make(map[string]interface{}) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a18fe422c..a8f820dba 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,8 +315,6 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) - m.Get("/recommend/org", routers.RecommendOrgFromPromote) - m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") From b2d9c2e2d6ac363c0b8faf827dbb9138a2be7290 Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 9 Dec 2021 09:55:57 +0800 Subject: [PATCH 20/31] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E6=90=9C=E7=B4=A2,=E9=BB=98=E8=AE=A4=E7=83=AD=E9=97=A8?= =?UTF-8?q?=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/head_navbar.tmpl | 12 +- templates/base/head_navbar_fluid.tmpl | 4 +- templates/base/head_navbar_home.tmpl | 4 +- templates/base/head_navbar_pro.tmpl | 216 ++++++++++++++++++++++++++++++++++ templates/base/head_pro.tmpl | 208 ++++++++++++++++++++++++++++++++ templates/explore/repos.tmpl | 2 +- 6 files changed, 436 insertions(+), 10 deletions(-) create mode 100644 templates/base/head_navbar_pro.tmpl create mode 100644 templates/base/head_pro.tmpl diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index c87524afb..c475faaf6 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -17,7 +17,7 @@ {{if .IsSigned}} -
      +
      {{.i18n.Tr "index"}}   @@ -47,7 +47,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   @@ -94,13 +94,14 @@ */}} {{if .IsSigned}} -
      +
      + sss{{$.SortType}}}}
      - + @@ -190,12 +191,13 @@
      +
      - + diff --git a/templates/base/head_navbar_fluid.tmpl b/templates/base/head_navbar_fluid.tmpl index d323d2151..deecc3bf2 100644 --- a/templates/base/head_navbar_fluid.tmpl +++ b/templates/base/head_navbar_fluid.tmpl @@ -17,7 +17,7 @@ {{if .IsSigned}} -
      +
      {{.i18n.Tr "index"}}   @@ -46,7 +46,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   diff --git a/templates/base/head_navbar_home.tmpl b/templates/base/head_navbar_home.tmpl index 096722bf6..7a5258168 100644 --- a/templates/base/head_navbar_home.tmpl +++ b/templates/base/head_navbar_home.tmpl @@ -9,7 +9,7 @@
      {{if .IsSigned}} -
      +
      {{.i18n.Tr "index"}}   @@ -38,7 +38,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   diff --git a/templates/base/head_navbar_pro.tmpl b/templates/base/head_navbar_pro.tmpl new file mode 100644 index 000000000..561a49cd7 --- /dev/null +++ b/templates/base/head_navbar_pro.tmpl @@ -0,0 +1,216 @@ + diff --git a/templates/base/head_pro.tmpl b/templates/base/head_pro.tmpl new file mode 100644 index 000000000..0b92016d1 --- /dev/null +++ b/templates/base/head_pro.tmpl @@ -0,0 +1,208 @@ + + + + + + + {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} + + {{if UseServiceWorker}} + + {{else}} + + {{end}} + + + + + + + {{if .IsSigned}} + + {{end}} + {{if .ContextUser}} + + {{end}} + {{if .SearchLimit}} + + {{end}} +{{if .GoGetImport}} + + +{{end}} + + + + + + + + +{{if .RequireSimpleMDE}} + +{{end}} + +{{if .RequireTribute}} + +{{end}} + + + +{{if .RequireMinicolors}} + +{{end}} + +{{if .PageIsUserProfile}} + + + + + {{if .Owner.Description}} + + {{end}} +{{else if .Repository}} + {{if .Issue}} + + + {{if .Issue.Content}} + + {{end}} + {{else}} + + + {{if .Repository.Description}} + + {{end}} + {{end}} + + +{{else}} + + + + + +{{end}} + +{{if .IsSigned }} + {{ if ne .SignedUser.Theme "gitea" }} + + {{end}} +{{else if ne DefaultTheme "gitea"}} + +{{end}} +{{template "custom/header" .}} + + + + + + + {{template "custom/body_outer_pre" .}} + +
      + + + {{template "custom/body_inner_pre" .}} + + {{if not .PageIsInstall}} +
      + {{template "base/head_navbar_pro" .}} +
      + {{end}} +{{/* +
      + + +*/}} diff --git a/templates/explore/repos.tmpl b/templates/explore/repos.tmpl index f487c9e01..df305099a 100644 --- a/templates/explore/repos.tmpl +++ b/templates/explore/repos.tmpl @@ -1,4 +1,4 @@ -{{template "base/head" .}} +{{template "base/head_pro" .}}
      {{template "explore/repo_search" .}} From 9599b2dfd0dceaf4218ee67f9b0cb88ada148538 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 09:56:35 +0800 Subject: [PATCH 21/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 1 - routers/repo/ai_model_manage.go | 43 ++++++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 15f67e858..cf0065282 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -649,7 +649,6 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { } } return resultMap - } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index d3afe61a4..0fdf698c0 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -55,7 +55,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio cloudType = aiTask.Type //download model zip //train type if cloudType == models.TypeCloudBrainTwo { - modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "", aiTask.TrainUrl) if err != nil { log.Info("download model from CloudBrainTwo faild." + err.Error()) return err @@ -141,9 +141,12 @@ func SaveModel(ctx *context.Context) { log.Info("save model end.") } -func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { - +func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + if trainUrl != "" { + objectkey = trainUrl[len(setting.Bucket)+1:] + } + modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { @@ -392,6 +395,24 @@ func ShowSingleModel(ctx *context.Context) { ctx.JSON(http.StatusOK, models) } +func queryUserName(intSlice []int64) map[int64]string { + keys := make(map[int64]string) + uniqueElements := []int64{} + for _, entry := range intSlice { + if _, value := keys[entry]; !value { + keys[entry] = "" + uniqueElements = append(uniqueElements, entry) + } + } + userNames, err := models.GetUserNamesByIDs(uniqueElements) + if err == nil { + for i, userName := range userNames { + keys[uniqueElements[i]] = userName + } + } + return keys +} + func ShowOneVersionOtherModel(ctx *context.Context) { repoId := ctx.Repo.Repository.ID name := ctx.Query("name") @@ -404,12 +425,10 @@ func ShowOneVersionOtherModel(ctx *context.Context) { model.IsCanOper = isOper(ctx, model.UserId) userIds[i] = model.UserId } + userNameMap := queryUserName(userIds) - userNames, err := models.GetUserNamesByIDs(userIds) - if err == nil { - for i, model := range aimodels { - model.UserName = userNames[i] - } + for _, model := range aimodels { + model.UserName = userNameMap[model.UserId] } if len(aimodels) > 0 { @@ -479,11 +498,9 @@ func ShowModelPageInfo(ctx *context.Context) { userIds[i] = model.UserId } - userNames, err := models.GetUserNamesByIDs(userIds) - if err == nil { - for i, model := range modelResult { - model.UserName = userNames[i] - } + userNameMap := queryUserName(userIds) + for _, model := range modelResult { + model.UserName = userNameMap[model.UserId] } mapInterface := make(map[string]interface{}) From 1a6c5c02983a03aceb6531b5867fb9fcf644f4f9 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 10:04:40 +0800 Subject: [PATCH 22/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 0fdf698c0..02aa94c39 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -144,7 +144,7 @@ func SaveModel(ctx *context.Context) { func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") if trainUrl != "" { - objectkey = trainUrl[len(setting.Bucket)+1:] + objectkey = trainUrl[len(setting.Bucket)+2:] } modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") From f0e9abeeaa79dc132d554bcf54aa43603908b5b6 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 10:18:15 +0800 Subject: [PATCH 23/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 02aa94c39..5103c2a5e 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -144,7 +144,7 @@ func SaveModel(ctx *context.Context) { func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") if trainUrl != "" { - objectkey = trainUrl[len(setting.Bucket)+2:] + objectkey = strings.Trim(trainUrl[len(setting.Bucket)+1:], "/") } modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") From cd05bc1cdd1a1114e1c203dce1bcac674a2ebe00 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 10:28:15 +0800 Subject: [PATCH 24/31] =?UTF-8?q?=E6=81=A2=E5=A4=8D=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/setting/setting.go | 6 ++++++ routers/home.go | 42 ++++++++++++++++++++++++++++++++++++++++++ routers/routes/routes.go | 2 ++ 3 files changed, 50 insertions(+) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index dd51623c1..2d70e47b1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,6 +433,9 @@ var ( AuthUser string AuthPassword string + //home page + RecommentRepoAddr string + //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -1229,6 +1232,9 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") + sec = Cfg.Section("homepage") + RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") + sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") diff --git a/routers/home.go b/routers/home.go index d2c93c771..7cc353ed8 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,6 +7,8 @@ package routers import ( "bytes" + "fmt" + "io/ioutil" "net/http" "strings" @@ -511,3 +513,43 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } +func RecommendOrgFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "organizations" + recommendFromPromote(ctx, url) +} + +func recommendFromPromote(ctx *context.Context, url string) { + resp, err := http.Get(url) + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + + allLineStr := string(bytes) + lines := strings.Split(allLineStr, "\n") + result := make([]string, len(lines)) + for i, line := range lines { + + tmpIndex := strings.Index(line, ".") + log.Info("i=" + fmt.Sprint(i) + " line=" + line + " tmpIndex=" + fmt.Sprint(tmpIndex)) + if tmpIndex == -1 { + result[i] = strings.Trim(line, " ") + } else { + result[i] = strings.Trim(line[tmpIndex+1:], " ") + } + } + ctx.JSON(http.StatusOK, result) +} + +func RecommendRepoFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "projects" + recommendFromPromote(ctx, url) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a8f820dba..a18fe422c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) + m.Get("/recommend/org", routers.RecommendOrgFromPromote) + m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") From 7c11778adcbfa55043c92c18bf932ae61886e0cf Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 9 Dec 2021 11:21:38 +0800 Subject: [PATCH 25/31] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E6=90=9C=E7=B4=A2,=E9=BB=98=E8=AE=A4=E7=83=AD=E9=97=A8?= =?UTF-8?q?=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/head_navbar.tmpl | 7 +++---- templates/base/head_navbar_fluid.tmpl | 4 ++-- templates/base/head_navbar_home.tmpl | 4 ++-- templates/base/head_navbar_pro.tmpl | 4 ++-- web_src/less/openi.less | 8 +++++++- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index c475faaf6..a67df2186 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -17,7 +17,7 @@ {{if .IsSigned}} -
      +
      {{.i18n.Tr "index"}}   @@ -47,7 +47,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   @@ -94,8 +94,7 @@ */}} {{if .IsSigned}} -
      - sss{{$.SortType}}}} +
      +
      {{.i18n.Tr "index"}}   @@ -46,7 +46,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   diff --git a/templates/base/head_navbar_home.tmpl b/templates/base/head_navbar_home.tmpl index 7a5258168..7391fa3e5 100644 --- a/templates/base/head_navbar_home.tmpl +++ b/templates/base/head_navbar_home.tmpl @@ -9,7 +9,7 @@
      {{if .IsSigned}} -
      +
      {{.i18n.Tr "index"}}   @@ -38,7 +38,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   diff --git a/templates/base/head_navbar_pro.tmpl b/templates/base/head_navbar_pro.tmpl index 561a49cd7..086e4de02 100644 --- a/templates/base/head_navbar_pro.tmpl +++ b/templates/base/head_navbar_pro.tmpl @@ -18,7 +18,7 @@ {{if .IsSigned}} -
      +
      {{.i18n.Tr "index"}}   @@ -48,7 +48,7 @@
      {{else if .IsLandingPageHome}} -
      +
      {{.i18n.Tr "home"}}   diff --git a/web_src/less/openi.less b/web_src/less/openi.less index 0f3129cd5..a20563ea5 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -518,4 +518,10 @@ display: block; } .letf2{ margin-left: -2px; -} \ No newline at end of file +} +.edge{ + margin-left:0 !important; + margin-right: 0 !important; + padding-left:0 !important; + padding-right:0 !important; + } \ No newline at end of file From a8e4dbe698cf0d9b8b462293e1872e801943720e Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 9 Dec 2021 11:30:25 +0800 Subject: [PATCH 26/31] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E6=90=9C=E7=B4=A2,=E9=BB=98=E8=AE=A4=E7=83=AD=E9=97=A8?= =?UTF-8?q?=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/head_navbar.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index a67df2186..27c82a6bf 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -100,7 +100,7 @@ - + From 7028a604e8addca71552d03f5977f4c4a9b32996 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 9 Dec 2021 14:45:46 +0800 Subject: [PATCH 27/31] #917 delete dataset attachment record and remove related files when delete repository --- models/repo.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/models/repo.go b/models/repo.go index 7948346c5..fda5460f5 100755 --- a/models/repo.go +++ b/models/repo.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "html/template" + "xorm.io/xorm" "code.gitea.io/gitea/modules/blockchain" @@ -1632,6 +1633,9 @@ func DeleteRepository(doer *User, uid, repoID int64) error { releaseAttachments = append(releaseAttachments, attachments[i].RelativePath()) } + // Delete dataset attachment record and remove related files + deleteDatasetAttachmentByRepoId(sess, repoID) + if err = deleteBeans(sess, &Access{RepoID: repo.ID}, &Action{RepoID: repo.ID}, @@ -1817,6 +1821,20 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return nil } +func deleteDatasetAttachmentByRepoId(sess *xorm.Session, repoId int64) error { + attachments := make([]*Attachment, 0) + if err := sess.Join("INNER", "dataset", "dataset.id = attachment.dataset_id"). + Where("dataset.repo_id = ?", repoId). + Find(&attachments); err != nil { + return err + } + if len(attachments) == 0 { + return nil + } + _, err := DeleteAttachments(attachments, true) + return err +} + // GetRepositoryByOwnerAndName returns the repository by given ownername and reponame. func GetRepositoryByOwnerAndName(ownerName, repoName string) (*Repository, error) { return getRepositoryByOwnerAndName(x, ownerName, repoName) From 9606dee30a078c51dd3d42587c891ef44f7d8a2a Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 16:01:47 +0800 Subject: [PATCH 28/31] =?UTF-8?q?=E9=87=8D=E6=9E=84=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E5=88=86=E6=9E=90=E6=89=80=E6=9C=89=E6=95=B0=E6=8D=AE=E8=A1=A8?= =?UTF-8?q?=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 922 ++++++++++++++++++++++++------------- routers/repo/user_data_analysis.go | 12 +- 2 files changed, 614 insertions(+), 320 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index cf0065282..df419f127 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -7,10 +7,14 @@ import ( "strconv" "time" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" - "xorm.io/xorm" +) + +const ( + Page_SIZE = 2000 ) type UserBusinessAnalysisAll struct { @@ -199,7 +203,13 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } log.Info("query return total:" + fmt.Sprint(allCount)) if allCount == 0 { - RefreshUserStaticAllTabel() + CommitCodeSizeMap, err := GetAllUserKPIStats() + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + RefreshUserStaticAllTabel(make(map[string]int), CommitCodeSizeMap) } pageSize := 1000 totalPage := int(allCount) / pageSize @@ -336,47 +346,164 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func RefreshUserStaticAllTabel() { +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { + + sess := x.NewSession() + defer sess.Close() statictisSess := xStatistic.NewSession() defer statictisSess.Close() - log.Info("delete all data from table: user_business_analysis_all") - statictisSess.Exec("delete from user_business_analysis_all") + log.Info("truncate all data from table: user_business_analysis_all") + statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all") currentTimeNow := time.Now() - pageStartTime := getLastCountDate() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() - var cond = builder.NewCond() - cond = cond.And( - builder.Gte{"count_date": pageStartTime}, - ) - cond = cond.And( - builder.Lte{"count_date": pageEndTime}, - ) - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). - Find(&userBusinessAnalysisList); err != nil { + startTime := currentTimeNow.AddDate(0, 0, -1) + + pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) + log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) + + start_unix := pageStartTime.Unix() + end_unix := pageEndTime.Unix() + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryAction(start_unix, end_unix, 6) + + CommentCountMap := queryComment(start_unix, end_unix) + FocusRepoCountMap := queryWatch(start_unix, end_unix) + StarRepoCountMap := queryStar(start_unix, end_unix) + WatchedCountMap := queryFollow(start_unix, end_unix) + + CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + + OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) + + DataDate := currentTimeNow.Format("2006-01-02") + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") return } - log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) - allData := getAllData(userRecord.ID, statictisSess) - allData.ID = userRecord.ID - allData.CountDate = 0 - allData.DataDate = userRecord.DataDate - allData.Email = userRecord.Email - allData.OpenIIndex = userRecord.OpenIIndex - allData.GiteaAgeMonth = userRecord.GiteaAgeMonth - allData.Name = userRecord.Name - allData.RegistDate = userRecord.RegistDate - - _, err := statictisSess.Insert(&allData) - if err != nil { - log.Info("insert all data failed." + err.Error()) + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + for i, userRecord := range userList { + log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + var dateRecordAll UserBusinessAnalysisAll + dateRecordAll.ID = userRecord.ID + dateRecordAll.Email = userRecord.Email + dateRecordAll.RegistDate = userRecord.CreatedUnix + dateRecordAll.Name = userRecord.Name + dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecordAll.DataDate = DataDate + + if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CodeMergeCount = 0 + } else { + dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitCount = 0 + } else { + dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID] + } + + if _, ok := IssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.IssueCount = 0 + } else { + dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID] + } + + if _, ok := CommentCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommentCount = 0 + } else { + dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID] + } + + if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.FocusRepoCount = 0 + } else { + dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID] + } + + if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.StarRepoCount = 0 + } else { + dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID] + } + + if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.WatchedCount = 0 + } else { + dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + + if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitDatasetSize = 0 + } else { + dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID] + } + + if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.SolveIssueCount = 0 + } else { + dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID] + } + + if _, ok := wikiCountMap[dateRecordAll.Name]; !ok { + dateRecordAll.EncyclopediasCount = 0 + } else { + dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name] + } + + if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CreateRepoCount = 0 + } else { + dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID] + } + + if _, ok := LoginCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.LoginCount = 0 + } else { + dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID] + } + + if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok { + dateRecordAll.OpenIIndex = 0 + } else { + dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID] + } + + dateRecordAll.CommitModelCount = 0 + _, err = statictisSess.Insert(&dateRecordAll) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + log.Info("refresh all data finished.") } @@ -385,19 +512,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") - userList := make([]*User, 0) - sess.Find(&userList) currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - //yesterday := currentTimeNow.AddDate(0, 0, -1) - //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) if isReCount { @@ -429,194 +550,194 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, statictisSess := xStatistic.NewSession() defer statictisSess.Close() - for i, userRecord := range userList { - var dateRecord UserBusinessAnalysis - dateRecord.ID = userRecord.ID - log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) - dateRecord.CountDate = CountDate.Unix() - - statictisSess.Delete(&dateRecord) - - dateRecord.Email = userRecord.Email - dateRecord.RegistDate = userRecord.CreatedUnix - dateRecord.Name = userRecord.Name - dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) - dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") + return err + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysis + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate + if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { + dateRecord.CodeMergeCount = 0 + } else { + dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] + } - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } + if _, ok := CommitCountMap[dateRecord.ID]; !ok { + dateRecord.CommitCount = 0 + } else { + dateRecord.CommitCount = CommitCountMap[dateRecord.ID] + } - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } + if _, ok := IssueCountMap[dateRecord.ID]; !ok { + dateRecord.IssueCount = 0 + } else { + dateRecord.IssueCount = IssueCountMap[dateRecord.ID] + } - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } + if _, ok := CommentCountMap[dateRecord.ID]; !ok { + dateRecord.CommentCount = 0 + } else { + dateRecord.CommentCount = CommentCountMap[dateRecord.ID] + } - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } + if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { + dateRecord.FocusRepoCount = 0 + } else { + dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] + } - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } + if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { + dateRecord.StarRepoCount = 0 + } else { + dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] + } - if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { - dateRecord.CommitCodeSize = 0 - } else { - dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) - } + if _, ok := WatchedCountMap[dateRecord.ID]; !ok { + dateRecord.WatchedCount = 0 + } else { + dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] + } - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) + } - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } + if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { + dateRecord.CommitDatasetSize = 0 + } else { + dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] + } - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } + if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { + dateRecord.SolveIssueCount = 0 + } else { + dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] + } - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } + if _, ok := wikiCountMap[dateRecord.Name]; !ok { + dateRecord.EncyclopediasCount = 0 + } else { + dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] + } - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } + if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { + dateRecord.CreateRepoCount = 0 + } else { + dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] + } - if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { - dateRecord.OpenIIndex = 0 - } else { - dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] - } + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } - dateRecord.CommitModelCount = 0 + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] + } - _, err = statictisSess.Insert(&dateRecord) - if err != nil { - log.Info("insert daterecord failed." + err.Error()) - return err - } + dateRecord.CommitModelCount = 0 - if isExistUserInAllTable(dateRecord.ID, statictisSess) { - updateCurrentData(dateRecord.ID, statictisSess, dateRecord) - } else { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) - allData := getAllData(dateRecord.ID, statictisSess) - allData.ID = dateRecord.ID - allData.CountDate = 0 - allData.DataDate = dateRecord.DataDate - allData.Email = dateRecord.Email - allData.OpenIIndex = dateRecord.OpenIIndex - allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth - allData.Name = dateRecord.Name - allData.RegistDate = dateRecord.RegistDate - - _, err = statictisSess.Insert(&allData) + _, err = statictisSess.Insert(&dateRecord) if err != nil { - log.Info("insert all data failed." + err.Error()) + log.Info("insert daterecord failed." + err.Error()) return err } } - } - return nil -} - -func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { - - _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + - ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + - ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + - ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + - ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + - ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + - ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + - ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + - ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + - ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + - ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + - ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + - ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + - ",login_count+=" + fmt.Sprint(currentData.LoginCount) + - " where id=" + fmt.Sprint(userId)) - - if err != nil { - log.Info("update table failed." + err.Error()) + indexTotal += Page_SIZE + if indexTotal >= count { + break + } } -} - -func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { + RefreshUserStaticAllTabel(wikiCountMap, CommitCodeSizeMap) - allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) - if err != nil { - return false - } - return allCount > 0 + return nil } -func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { - var dateRecord UserBusinessAnalysisAll - - rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") - if err == nil { - for i, row := range rows { - log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) - dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) - dateRecord.CommitCount = getInt(string(row["commit_count"])) - dateRecord.IssueCount = getInt(string(row["issue_count"])) - dateRecord.CommentCount = getInt(string(row["comment_count"])) - dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) - dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) - dateRecord.WatchedCount = getInt(string(row["watched_count"])) - dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) - dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) - dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) - dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) - dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) - dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) - dateRecord.LoginCount = getInt(string(row["login_count"])) - } - } - return dateRecord -} +// func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { + +// _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + +// ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + +// ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + +// ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + +// ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + +// ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + +// ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + +// ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + +// ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + +// ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + +// ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + +// ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + +// ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + +// ",login_count+=" + fmt.Sprint(currentData.LoginCount) + +// " where id=" + fmt.Sprint(userId)) + +// if err != nil { +// log.Info("update table failed." + err.Error()) +// } + +// } + +// func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { + +// allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) +// if err != nil { +// return false +// } +// return allCount > 0 +// } + +// func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { +// var dateRecord UserBusinessAnalysisAll + +// rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") +// if err == nil { +// for i, row := range rows { +// log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) +// dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) +// dateRecord.CommitCount = getInt(string(row["commit_count"])) +// dateRecord.IssueCount = getInt(string(row["issue_count"])) +// dateRecord.CommentCount = getInt(string(row["comment_count"])) +// dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) +// dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) +// dateRecord.WatchedCount = getInt(string(row["watched_count"])) +// dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) +// dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) +// dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) +// dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) +// dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) +// dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) +// dateRecord.LoginCount = getInt(string(row["login_count"])) +// } +// } +// return dateRecord +// } func getInt(str string) int { re, err := strconv.ParseInt(str, 10, 32) @@ -631,43 +752,70 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { - //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() defer sess.Close() - sess.Select("issue_assignees.*").Table("issue_assignees"). - Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) - issueAssigneesList := make([]*IssueAssignees, 0) - sess.Find(&issueAssigneesList) resultMap := make(map[int64]int) - log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) - for _, issueAssigneesRecord := range issueAssigneesList { - if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { - resultMap[issueAssigneesRecord.AssigneeID] = 1 - } else { - resultMap[issueAssigneesRecord.AssigneeID] += 1 + cond := "issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Table("issue_assignees").Join("inner", "issue", "issue.id=issue_assignees.issue_id").Where(cond).Count(new(IssueAssignees)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueAssigneesList := make([]*IssueAssignees, 0) + sess.Select("issue_assignees.*").Table("issue_assignees"). + Join("inner", "issue", "issue.id=issue_assignees.issue_id"). + Where(cond).Limit(Page_SIZE, int(indexTotal)) + + sess.Find(&issueAssigneesList) + + log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) + for _, issueAssigneesRecord := range issueAssigneesList { + if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { + resultMap[issueAssigneesRecord.AssigneeID] = 1 + } else { + resultMap[issueAssigneesRecord.AssigneeID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - - sess.Select("issue.*").Table("issue"). - Join("inner", "pull_request", "issue.id=pull_request.issue_id"). - Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) - - issueList := make([]*Issue, 0) - sess.Find(&issueList) resultMap := make(map[int64]int) - log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) - for _, issueRecord := range issueList { - if _, ok := resultMap[issueRecord.PosterID]; !ok { - resultMap[issueRecord.PosterID] = 1 - } else { - resultMap[issueRecord.PosterID] += 1 + cond := "pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Count(new(Issue)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueList := make([]*Issue, 0) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&issueList) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -676,34 +824,68 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + + cond := "user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + cond := "op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query Action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -713,16 +895,30 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - commentList := make([]*Comment, 0) - sess.Find(&commentList) + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Comment size=" + fmt.Sprint(len(commentList))) - for _, commentRecord := range commentList { - if _, ok := resultMap[commentRecord.PosterID]; !ok { - resultMap[commentRecord.PosterID] = 1 - } else { - resultMap[commentRecord.PosterID] += 1 + count, err := sess.Where(cond).Count(new(Comment)) + if err != nil { + log.Info("query Comment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,type,poster_id").Table("comment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + commentList := make([]*Comment, 0) + sess.Find(&commentList) + log.Info("query Comment size=" + fmt.Sprint(len(commentList))) + for _, commentRecord := range commentList { + if _, ok := resultMap[commentRecord.PosterID]; !ok { + resultMap[commentRecord.PosterID] = 1 + } else { + resultMap[commentRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -732,18 +928,37 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - watchList := make([]*Watch, 0) - sess.Find(&watchList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + resultMap := make(map[int64]int) - log.Info("query Watch size=" + fmt.Sprint(len(watchList))) - for _, watchRecord := range watchList { - if _, ok := resultMap[watchRecord.UserID]; !ok { - resultMap[watchRecord.UserID] = 1 - } else { - resultMap[watchRecord.UserID] += 1 + count, err := sess.Where(cond).Count(new(Watch)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + watchList := make([]*Watch, 0) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&watchList) + + log.Info("query Watch size=" + fmt.Sprint(len(watchList))) + for _, watchRecord := range watchList { + if _, ok := resultMap[watchRecord.UserID]; !ok { + resultMap[watchRecord.UserID] = 1 + } else { + resultMap[watchRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } @@ -752,82 +967,151 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - starList := make([]*Star, 0) - sess.Find(&starList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Star size=" + fmt.Sprint(len(starList))) - for _, starRecord := range starList { - if _, ok := resultMap[starRecord.UID]; !ok { - resultMap[starRecord.UID] = 1 - } else { - resultMap[starRecord.UID] += 1 + + count, err := sess.Where(cond).Count(new(Star)) + if err != nil { + log.Info("query star error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uid,repo_id").Table("star").Where(cond).Limit(Page_SIZE, int(indexTotal)) + starList := make([]*Star, 0) + sess.Find(&starList) + + log.Info("query Star size=" + fmt.Sprint(len(starList))) + for _, starRecord := range starList { + if _, ok := resultMap[starRecord.UID]; !ok { + resultMap[starRecord.UID] = 1 + } else { + resultMap[starRecord.UID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap - } func queryFollow(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - followList := make([]*Follow, 0) - sess.Find(&followList) resultMap := make(map[int64]int) - log.Info("query Follow size=" + fmt.Sprint(len(followList))) - for _, followRecord := range followList { - if _, ok := resultMap[followRecord.FollowID]; !ok { - resultMap[followRecord.FollowID] = 1 - } else { - resultMap[followRecord.FollowID] += 1 + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Follow)) + if err != nil { + log.Info("query follow error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).Limit(Page_SIZE, int(indexTotal)) + followList := make([]*Follow, 0) + sess.Find(&followList) + + log.Info("query Follow size=" + fmt.Sprint(len(followList))) + for _, followRecord := range followList { + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 + } else { + resultMap[followRecord.FollowID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - attachmentList := make([]*Attachment, 0) - sess.Find(&attachmentList) resultMap := make(map[int64]int) - log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) - for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB - } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Attachment)) + if err != nil { + log.Info("query attachment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + attachmentList := make([]*Attachment, 0) + sess.Find(&attachmentList) + + log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) + for _, attachRecord := range attachmentList { + if _, ok := resultMap[attachRecord.UploaderID]; !ok { + resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + } else { + resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } - return resultMap + return resultMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - repoList := make([]*Repository, 0) - sess.Find(&repoList) resultMap := make(map[int64]int) - log.Info("query Repository size=" + fmt.Sprint(len(repoList))) - for _, repoRecord := range repoList { - if _, ok := resultMap[repoRecord.OwnerID]; !ok { - resultMap[repoRecord.OwnerID] = 1 - } else { - resultMap[repoRecord.OwnerID] += 1 + + cond := "is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Repository)) + if err != nil { + log.Info("query Repository error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,owner_id,name").Table("repository").Where(cond).Limit(Page_SIZE, int(indexTotal)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := resultMap[repoRecord.OwnerID]; !ok { + resultMap[repoRecord.OwnerID] = 1 + } else { + resultMap[repoRecord.OwnerID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + + statictisSess.Select("id,repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)).OrderBy("id desc") repoStatisticList := make([]*RepoStatistic, 0) statictisSess.Find(&repoStatisticList) repoOpenIIndexMap := make(map[int64]float64) @@ -883,18 +1167,34 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - userLoginLogList := make([]*UserLoginLog, 0) - statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) - log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) - for _, loginRecord := range userLoginLogList { - if _, ok := resultMap[loginRecord.UId]; !ok { - resultMap[loginRecord.UId] = 1 - } else { - resultMap[loginRecord.UId] += 1 + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := statictisSess.Where(cond).Count(new(UserLoginLog)) + if err != nil { + log.Info("query UserLoginLog error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2f37dc15e..42189c57f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -148,11 +148,6 @@ func QueryUserStaticDataPage(ctx *context.Context) { func TimingCountDataByDateAndReCount(date string, isReCount bool) { - if date == "refreshAll" { - models.RefreshUserStaticAllTabel() - return - } - t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -205,10 +200,9 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } - - if isReCount { - models.RefreshUserStaticAllTabel() - } + log.Info("start to count all user info data") + //models.RefreshUserStaticAllTabel(wikiMap) + log.Info("end to count all user info data") } func TimingCountDataByDate(date string) { From 73c0a959e5272ee90c37b723e553fecda458d0af Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 16:14:40 +0800 Subject: [PATCH 29/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index df419f127..595274340 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -14,7 +14,7 @@ import ( ) const ( - Page_SIZE = 2000 + Page_SIZE = 2 ) type UserBusinessAnalysisAll struct { From b40c97627c71ebfbe3217ccc82bb53616d196750 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 16:16:42 +0800 Subject: [PATCH 30/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 59 ---------------------------------------- 1 file changed, 59 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 595274340..9f25f63c4 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -680,65 +680,6 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, return nil } -// func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { - -// _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + -// ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + -// ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + -// ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + -// ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + -// ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + -// ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + -// ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + -// ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + -// ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + -// ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + -// ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + -// ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + -// ",login_count+=" + fmt.Sprint(currentData.LoginCount) + -// " where id=" + fmt.Sprint(userId)) - -// if err != nil { -// log.Info("update table failed." + err.Error()) -// } - -// } - -// func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { - -// allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) -// if err != nil { -// return false -// } -// return allCount > 0 -// } - -// func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { -// var dateRecord UserBusinessAnalysisAll - -// rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") -// if err == nil { -// for i, row := range rows { -// log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) -// dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) -// dateRecord.CommitCount = getInt(string(row["commit_count"])) -// dateRecord.IssueCount = getInt(string(row["issue_count"])) -// dateRecord.CommentCount = getInt(string(row["comment_count"])) -// dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) -// dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) -// dateRecord.WatchedCount = getInt(string(row["watched_count"])) -// dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) -// dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) -// dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) -// dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) -// dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) -// dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) -// dateRecord.LoginCount = getInt(string(row["login_count"])) -// } -// } -// return dateRecord -// } - func getInt(str string) int { re, err := strconv.ParseInt(str, 10, 32) if err != nil { From c6702e1b8e7fc04fd5cb348e5f0e6f7cf1051a65 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 9 Dec 2021 16:26:56 +0800 Subject: [PATCH 31/31] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 9f25f63c4..9b9d5e0ad 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -14,7 +14,7 @@ import ( ) const ( - Page_SIZE = 2 + Page_SIZE = 2000 ) type UserBusinessAnalysisAll struct {