From 33d029f74a954fe0cb723d2eb7dba75ce1010703 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 12 May 2022 11:12:31 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E5=9B=BD=E9=99=85=E5=8C=96=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E6=8F=90=E4=BA=A4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/cloudbrain/cloudbrain.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index f1d3cb344..6a648a94a 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -1,11 +1,12 @@ package cloudbrain import ( - "code.gitea.io/gitea/modules/timeutil" "encoding/json" "errors" "strconv" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/models" @@ -16,7 +17,8 @@ import ( ) const ( - Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + //Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + Command = `service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` CommandBenchmark = `echo "start benchmark";cd /benchmark && bash run_bk.sh;echo "end benchmark"` CodeMountPath = "/code" @@ -184,7 +186,6 @@ func AdminOrImageCreaterRight(ctx *context.Context) { } - func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description, branchName, bootFile, params string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + From 1532f003b9e9ccec6e3bde1fb547524ad6f219cb Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 12 May 2022 11:20:17 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E5=9B=BD=E9=99=85=E5=8C=96=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E6=8F=90=E4=BA=A4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/cloudbrain/cloudbrain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 6a648a94a..8888361e8 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -17,8 +17,8 @@ import ( ) const ( - //Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` - Command = `service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + //Command = `service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` CommandBenchmark = `echo "start benchmark";cd /benchmark && bash run_bk.sh;echo "end benchmark"` CodeMountPath = "/code" From bb642080ca3653cc4d74e2a9fc2cec9f96b111e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 15:38:24 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 4de0c6d1a..453838351 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -233,11 +233,7 @@ func QueryMetricsPage(start int64, end int64, page int, pageSize int) ([]*UserMe statictisSess := xStatistic.NewSession() defer statictisSess.Close() cond := "count_date >" + fmt.Sprint(start) + " and count_date<" + fmt.Sprint(end) - allCount, err := statictisSess.Where(cond).Count(new(UserMetrics)) - if err != nil { - log.Info("query error." + err.Error()) - return nil, 0 - } + userMetricsList := make([]*UserMetrics, 0) //.Limit(pageSize, page*pageSize) if err := statictisSess.Table(new(UserMetrics)).Where(cond).OrderBy("count_date desc"). @@ -245,7 +241,7 @@ func QueryMetricsPage(start int64, end int64, page int, pageSize int) ([]*UserMe return nil, 0 } postDeal(userMetricsList) - return userMetricsList, allCount + return userMetricsList, int64(len(userMetricsList)) } func QueryMetrics(start int64, end int64) ([]*UserMetrics, int) { @@ -260,7 +256,21 @@ func QueryMetrics(start int64, end int64) ([]*UserMetrics, int) { return userMetricsList, len(userMetricsList) } +func duplicateRemoval(userMetricsList []*UserMetrics) []*UserMetrics { + userMetricsResult := make([]*UserMetrics, 0) + for i := 0; i < len(userMetricsList); i++ { + if i > 0 { + if userMetricsList[i].DataDate == userMetricsList[i-1].DataDate { + continue + } + } + userMetricsResult = append(userMetricsResult, userMetricsList[i]) + } + return userMetricsResult +} + func postDeal(userMetricsList []*UserMetrics) { + userMetricsList = duplicateRemoval(userMetricsList) for _, userMetrics := range userMetricsList { userMetrics.DisplayDate = userMetrics.DataDate userMetrics.TotalRegistUser = userMetrics.ActivateRegistUser + userMetrics.NotActivateRegistUser @@ -276,6 +286,7 @@ func QueryMetricsForAll() []*UserMetrics { Find(&userMetricsList); err != nil { return nil } + userMetricsList = duplicateRemoval(userMetricsList) return makeResultForMonth(userMetricsList, len(userMetricsList)) } From 03dff4252eeecab741ed759a1c1e273568864553 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 15:48:46 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 453838351..cd270510e 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -240,8 +240,8 @@ func QueryMetricsPage(start int64, end int64, page int, pageSize int) ([]*UserMe Find(&userMetricsList); err != nil { return nil, 0 } - postDeal(userMetricsList) - return userMetricsList, int64(len(userMetricsList)) + postUserMetricsList := postDeal(userMetricsList) + return postUserMetricsList, int64(len(postUserMetricsList)) } func QueryMetrics(start int64, end int64) ([]*UserMetrics, int) { @@ -252,8 +252,8 @@ func QueryMetrics(start int64, end int64) ([]*UserMetrics, int) { Find(&userMetricsList); err != nil { return nil, 0 } - postDeal(userMetricsList) - return userMetricsList, len(userMetricsList) + postUserMetricsList := postDeal(userMetricsList) + return postUserMetricsList, int(len(postUserMetricsList)) } func duplicateRemoval(userMetricsList []*UserMetrics) []*UserMetrics { @@ -269,13 +269,14 @@ func duplicateRemoval(userMetricsList []*UserMetrics) []*UserMetrics { return userMetricsResult } -func postDeal(userMetricsList []*UserMetrics) { - userMetricsList = duplicateRemoval(userMetricsList) - for _, userMetrics := range userMetricsList { +func postDeal(userMetricsList []*UserMetrics) []*UserMetrics { + duplicateRemovalUserMetricsList := duplicateRemoval(userMetricsList) + for _, userMetrics := range duplicateRemovalUserMetricsList { userMetrics.DisplayDate = userMetrics.DataDate userMetrics.TotalRegistUser = userMetrics.ActivateRegistUser + userMetrics.NotActivateRegistUser userMetrics.TotalNotActivateRegistUser = userMetrics.TotalUser - userMetrics.TotalActivateRegistUser } + return duplicateRemovalUserMetricsList } func QueryMetricsForAll() []*UserMetrics { @@ -286,8 +287,8 @@ func QueryMetricsForAll() []*UserMetrics { Find(&userMetricsList); err != nil { return nil } - userMetricsList = duplicateRemoval(userMetricsList) - return makeResultForMonth(userMetricsList, len(userMetricsList)) + duplicateRemovalUserMetricsList := duplicateRemoval(userMetricsList) + return makeResultForMonth(duplicateRemovalUserMetricsList, len(duplicateRemovalUserMetricsList)) } func QueryMetricsForYear() []*UserMetrics { From b1b05ae44d3442105d6c33d8335c4048db319181 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 16:20:45 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/cloudbrain/cloudbrain.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 8888361e8..a71389741 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -18,7 +18,6 @@ import ( const ( Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` - //Command = `service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` CommandBenchmark = `echo "start benchmark";cd /benchmark && bash run_bk.sh;echo "end benchmark"` CodeMountPath = "/code" @@ -30,6 +29,8 @@ const ( Snn4imagenetMountPath = "/snn4imagenet" BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" + Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s'` + BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s'` SubTaskName = "task1" @@ -348,7 +349,7 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, } stringId := strconv.FormatInt(task.ID, 10) - if string(models.JobTypeBenchmark) == jobType { + if IsBenchmarkJob(jobType) { notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateBenchMarkTask) } else if string(models.JobTypeTrain) == jobType { notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, displayJobName, models.ActionCreateGPUTrainTask) @@ -359,6 +360,10 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, return nil } +func IsBenchmarkJob(jobType string) bool { + return string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType +} + func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + From 67f7cac54553bb868786c54cdac30880bac04148 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 16:51:07 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BC=80=E5=A7=8B=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- routers/private/internal.go | 5 +++-- routers/private/tool.go | 5 +++++ routers/repo/user_data_analysis.go | 10 ++++++++++ web_src/js/components/UserTrend.vue | 6 +++--- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index cd270510e..f53039ebd 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -891,9 +891,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, //insert userMetrics table var useMetrics UserMetrics useMetrics.CountDate = CountDate.Unix() - useMetrics.DataDate = DataDate statictisSess.Delete(&useMetrics) + useMetrics.DataDate = DataDate useMetrics.ActivateRegistUser = getMapKeyStringValue("ActivateRegistUser", userMetrics) useMetrics.HasActivityUser = getMapKeyStringValue("HasActivityUser", userMetrics) useMetrics.RegistActivityUser = 0 diff --git a/routers/private/internal.go b/routers/private/internal.go index a0c2843be..4731463b1 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -6,9 +6,10 @@ package private import ( - "code.gitea.io/gitea/routers/repo" "strings" + "code.gitea.io/gitea/routers/repo" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" @@ -46,7 +47,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues) m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat/:date", RepoStatisticManually) - + m.Post("/tool/user_stat/:date", UserStatisticManually) m.Get("/tool/org_stat", OrgStatisticManually) m.Post("/tool/update_repo_visit/:date", UpdateRepoVisit) m.Post("/task/history_handle/duration", repo.HandleTaskWithNoDuration) diff --git a/routers/private/tool.go b/routers/private/tool.go index 122a41afe..f8b1a2c3f 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -45,6 +45,11 @@ func RepoStatisticManually(ctx *macaron.Context) { repo.TimingCountDataByDate(date) } +func UserStatisticManually(ctx *macaron.Context) { + date := ctx.Params("date") + repo.TimingCountDataByDate(date) +} + func OrgStatisticManually() { models.UpdateOrgStatistics() } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2823f9c87..cd17e5202 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -247,6 +247,7 @@ func queryUserDataPage(ctx *context.Context, tableName string, queryObj interfac mapInterface := make(map[string]interface{}) mapInterface["data"] = re mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) } } @@ -303,6 +304,15 @@ func queryMetrics(ctx *context.Context, tableName string, startTime time.Time, e mapInterface := make(map[string]interface{}) mapInterface["data"] = result mapInterface["count"] = count + if tableName == "public.user_business_analysis_yesterday" { + mapInterface["datarecordbegintime"] = setting.RadarMap.RecordBeginTime + if len(result) > 0 { + dateTime := time.Unix(result[0].CountDate, 0) + mapInterface["lastUpdatedTime"] = dateTime.Format("2006-01-02 15:04:05") + } else { + mapInterface["lastUpdatedTime"] = "" + } + } ctx.JSON(http.StatusOK, mapInterface) } diff --git a/web_src/js/components/UserTrend.vue b/web_src/js/components/UserTrend.vue index 489d65cbb..a8a573247 100644 --- a/web_src/js/components/UserTrend.vue +++ b/web_src/js/components/UserTrend.vue @@ -257,11 +257,11 @@ return days; }, getUpdateTime(){ - this.$axios.get('../../api/v1/projectboard/project',{ + this.$axios.get('../../api/v1//query_metrics_yesterday',{ params:this.params_pro }).then((res)=>{ - this.recordBeginTime=res.data.recordBeginTime - this.lastUpdatedTime=res.data.lastUpdatedTime + this.recordBeginTime=res.datarecordbegintime + this.lastUpdatedTime=res.lastUpdatedTime }) }, getUserList(type_val,index){ From 2e8368e6414a1aacdadb1cf52ec8e0465bec47c3 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 16:59:48 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BC=80=E5=A7=8B=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserTrend.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/components/UserTrend.vue b/web_src/js/components/UserTrend.vue index a8a573247..22bbf2798 100644 --- a/web_src/js/components/UserTrend.vue +++ b/web_src/js/components/UserTrend.vue @@ -257,7 +257,7 @@ return days; }, getUpdateTime(){ - this.$axios.get('../../api/v1//query_metrics_yesterday',{ + this.$axios.get('../../api/v1/query_metrics_yesterday',{ params:this.params_pro }).then((res)=>{ this.recordBeginTime=res.datarecordbegintime From 397f9c12691c743381f23583e0f6fbb2cf84ab2a Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 17:30:07 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BC=80=E5=A7=8B=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserTrend.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web_src/js/components/UserTrend.vue b/web_src/js/components/UserTrend.vue index 22bbf2798..fcc4ede4b 100644 --- a/web_src/js/components/UserTrend.vue +++ b/web_src/js/components/UserTrend.vue @@ -260,8 +260,8 @@ this.$axios.get('../../api/v1/query_metrics_yesterday',{ params:this.params_pro }).then((res)=>{ - this.recordBeginTime=res.datarecordbegintime - this.lastUpdatedTime=res.lastUpdatedTime + this.recordBeginTime=res.data.datarecordbegintime + this.lastUpdatedTime=res.data.lastUpdatedTime }) }, getUserList(type_val,index){ From e82b8b70c856f33f82015a78c2b369ef5c38160d Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 16 May 2022 17:32:23 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BC=80=E5=A7=8B=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index cd17e5202..9707f8df2 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -305,7 +305,7 @@ func queryMetrics(ctx *context.Context, tableName string, startTime time.Time, e mapInterface["data"] = result mapInterface["count"] = count if tableName == "public.user_business_analysis_yesterday" { - mapInterface["datarecordbegintime"] = setting.RadarMap.RecordBeginTime + mapInterface["datarecordbegintime"] = setting.RadarMap.GrowthBeginTime if len(result) > 0 { dateTime := time.Unix(result[0].CountDate, 0) mapInterface["lastUpdatedTime"] = dateTime.Format("2006-01-02 15:04:05")