From 6d4a039df93e13234093d2720eda9e011e5f4293 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 16:13:45 +0800 Subject: [PATCH 01/28] =?UTF-8?q?issue=E9=97=AE=E9=A2=98=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/issue_list.go | 19 +++++++++++++++++++ routers/repo/issue.go | 6 +++++- templates/repo/issue/list.tmpl | 6 +++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/models/issue_list.go b/models/issue_list.go index 628058eb3..17934c004 100644 --- a/models/issue_list.go +++ b/models/issue_list.go @@ -511,6 +511,25 @@ func (issues IssueList) LoadComments() error { return issues.loadComments(x, builder.NewCond()) } +// LoadClosedComments loads discuss comments +func (issues IssueList) LoadClosedComments() error { + err := issues.loadComments(x, builder.Eq{"comment.type": CommentTypeClose}) + if err == nil { + for _, issue := range issues { + if issue.Comments != nil { + tmp := make([]*Comment, 0) + tmp = append(tmp, issue.Comments[len(issue.Comments)-1]) + issue.Comments = tmp + for i, comm := range issue.Comments { + fmt.Printf("comm load poster.i=" + fmt.Sprint(i)) + comm.LoadPoster() + } + } + } + } + return err +} + // LoadDiscussComments loads discuss comments func (issues IssueList) LoadDiscussComments() error { return issues.loadComments(x, builder.Eq{"comment.type": CommentTypeComment}) diff --git a/routers/repo/issue.go b/routers/repo/issue.go index d28936594..ffe8cb188 100755 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -224,7 +224,11 @@ func issues(ctx *context.Context, milestoneID int64, isPullOption util.OptionalB } } - approvalCounts, err := models.IssueList(issues).GetApprovalCounts() + issueList := models.IssueList(issues) + if isShowClosed { + issueList.LoadClosedComments() + } + approvalCounts, err := issueList.GetApprovalCounts() if err != nil { ctx.ServerError("ApprovalCounts", err) return diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 1e29b6512..8c15ef060 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -270,7 +270,11 @@ {{if .OriginalAuthor }} {{$.i18n.Tr .GetLastEventLabelFake $timeStr .OriginalAuthor | Safe}} {{else if gt .Poster.ID 0}} - {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} + {{if .IsClosed}} + {{$.i18n.Tr .GetLastEventLabelFake $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} + {{eles}} + {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} + {{end}} {{else}} {{$.i18n.Tr .GetLastEventLabelFake $timeStr (.Poster.GetDisplayName | Escape) | Safe}} {{end}} From 5556541117b67517057059ce78231ecdb71eb008 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:09:38 +0800 Subject: [PATCH 02/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/issue.go | 21 +++++++++++++++++++++ routers/repo/issue.go | 6 +----- templates/repo/issue/list.tmpl | 2 +- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/models/issue.go b/models/issue.go index 3ed49ce42..b951fabe9 100755 --- a/models/issue.go +++ b/models/issue.go @@ -840,6 +840,27 @@ func (issue *Issue) GetLastEventLabel() string { } // GetLastComment return last comment for the current issue. +func (issue *Issue) GetCloseUser() (*User, error) { + var c Comment + exist, err := x.Where("type = ?", CommentTypeClose). + And("issue_id = ?", issue.ID).Desc("id").Get(&c) + if err != nil { + return nil, err + } + if !exist { + return nil, nil + } + err = c.LoadPoster() + if err == nil { + return c.Poster, nil + } + if issue.Poster == nil { + issue.LoadPoster() + } + return issue.Poster, nil +} + +// GetLastComment return last comment for the current issue. func (issue *Issue) GetLastComment() (*Comment, error) { var c Comment exist, err := x.Where("type = ?", CommentTypeComment). diff --git a/routers/repo/issue.go b/routers/repo/issue.go index ffe8cb188..d28936594 100755 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -224,11 +224,7 @@ func issues(ctx *context.Context, milestoneID int64, isPullOption util.OptionalB } } - issueList := models.IssueList(issues) - if isShowClosed { - issueList.LoadClosedComments() - } - approvalCounts, err := issueList.GetApprovalCounts() + approvalCounts, err := models.IssueList(issues).GetApprovalCounts() if err != nil { ctx.ServerError("ApprovalCounts", err) return diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 8c15ef060..ab29a3c8f 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -271,7 +271,7 @@ {{$.i18n.Tr .GetLastEventLabelFake $timeStr .OriginalAuthor | Safe}} {{else if gt .Poster.ID 0}} {{if .IsClosed}} - {{$.i18n.Tr .GetLastEventLabelFake $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} + {{$.i18n.Tr .GetLastEventLabelFake $timeStr .GetCloseUser.HomeLink (.GetCloseUser.GetDisplayName | Escape) | Safe}} {{eles}} {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} {{end}} From 04ee8beb28b282c6273232d398d7ab2f9b1d7820 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:11:29 +0800 Subject: [PATCH 03/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/issue/list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index ab29a3c8f..a715c929c 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -272,7 +272,7 @@ {{else if gt .Poster.ID 0}} {{if .IsClosed}} {{$.i18n.Tr .GetLastEventLabelFake $timeStr .GetCloseUser.HomeLink (.GetCloseUser.GetDisplayName | Escape) | Safe}} - {{eles}} + {{else}} {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} {{end}} {{else}} From 49534218db432381f3a46aa366a92ffce2b585a1 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:13:49 +0800 Subject: [PATCH 04/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/issue/list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index a715c929c..42181fb86 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -271,7 +271,7 @@ {{$.i18n.Tr .GetLastEventLabelFake $timeStr .OriginalAuthor | Safe}} {{else if gt .Poster.ID 0}} {{if .IsClosed}} - {{$.i18n.Tr .GetLastEventLabelFake $timeStr .GetCloseUser.HomeLink (.GetCloseUser.GetDisplayName | Escape) | Safe}} + {{$.i18n.Tr .GetLastEventLabelFake $timeStr (.GetCloseUser.GetDisplayName | Escape) | Safe}} {{else}} {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} {{end}} From 779bb38cbc7d53b8aa74cd52acf2b3711eb85853 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:16:22 +0800 Subject: [PATCH 05/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/issue/list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 42181fb86..282774ddc 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -271,7 +271,7 @@ {{$.i18n.Tr .GetLastEventLabelFake $timeStr .OriginalAuthor | Safe}} {{else if gt .Poster.ID 0}} {{if .IsClosed}} - {{$.i18n.Tr .GetLastEventLabelFake $timeStr (.GetCloseUser.GetDisplayName | Escape) | Safe}} + {{$.i18n.Tr .GetLastEventLabel $timeStr .GetCloseUser.HomeLink (.GetCloseUser.GetDisplayName | Escape) | Safe}} {{else}} {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} {{end}} From 40ab6143029addbd6162c9dc6cdd2488d728ce94 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:18:31 +0800 Subject: [PATCH 06/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_zh-CN.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index f4e8f1aea..fdaf90de5 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1666,7 +1666,7 @@ issues.action_assignee_no_select=未指派 issues.opened_by=由 %[3]s 于 %[1]s创建 pulls.merged_by=由 %[3]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 -issues.closed_by=按 %[3]s 关闭%[1]s +issues.closed_by=由 %[3]s 关闭%[1]s issues.opened_by_fake=由 %[2]s 于 %[1]s创建 issues.closed_by_fake=通过 %[2]s 关闭 %[1]s issues.previous=上一页 From 6435b186b9fe5b4e1fd828337a44b98afb737971 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:20:47 +0800 Subject: [PATCH 07/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_zh-CN.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index fdaf90de5..efaf031c6 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1666,7 +1666,7 @@ issues.action_assignee_no_select=未指派 issues.opened_by=由 %[3]s 于 %[1]s创建 pulls.merged_by=由 %[3]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 -issues.closed_by=由 %[3]s 关闭%[1]s +issues.closed_by=由 %[3]s 关闭于 %[1]s issues.opened_by_fake=由 %[2]s 于 %[1]s创建 issues.closed_by_fake=通过 %[2]s 关闭 %[1]s issues.previous=上一页 From 3e68da48dfaf504fd0ed8ab249b192650adba1af Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:28:22 +0800 Subject: [PATCH 08/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/issue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/issue.go b/models/issue.go index b951fabe9..b943aea79 100755 --- a/models/issue.go +++ b/models/issue.go @@ -851,7 +851,7 @@ func (issue *Issue) GetCloseUser() (*User, error) { return nil, nil } err = c.LoadPoster() - if err == nil { + if c.Poster != nil { return c.Poster, nil } if issue.Poster == nil { From 439094a43b8fddc8385f69141841914c6e935f1f Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 13 Dec 2022 17:33:27 +0800 Subject: [PATCH 09/28] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/issue_list.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/models/issue_list.go b/models/issue_list.go index 17934c004..628058eb3 100644 --- a/models/issue_list.go +++ b/models/issue_list.go @@ -511,25 +511,6 @@ func (issues IssueList) LoadComments() error { return issues.loadComments(x, builder.NewCond()) } -// LoadClosedComments loads discuss comments -func (issues IssueList) LoadClosedComments() error { - err := issues.loadComments(x, builder.Eq{"comment.type": CommentTypeClose}) - if err == nil { - for _, issue := range issues { - if issue.Comments != nil { - tmp := make([]*Comment, 0) - tmp = append(tmp, issue.Comments[len(issue.Comments)-1]) - issue.Comments = tmp - for i, comm := range issue.Comments { - fmt.Printf("comm load poster.i=" + fmt.Sprint(i)) - comm.LoadPoster() - } - } - } - } - return err -} - // LoadDiscussComments loads discuss comments func (issues IssueList) LoadDiscussComments() error { return issues.loadComments(x, builder.Eq{"comment.type": CommentTypeComment}) From bd6f783cc1851eb07fb69c97b4c42fbf5424a523 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 14 Dec 2022 10:46:41 +0800 Subject: [PATCH 10/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ca1eb4c7b..4ae9f6ced 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -2106,8 +2106,8 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) (map[int64]int, map[s detailInfoMap[key] = getMapKeyStringValue(key, detailInfoMap) + int(repoRecord.CloneCnt) key = fmt.Sprint(repoRecord.OwnerID) + "_most_download" - if int(repoRecord.CloneCnt) > getMapKeyStringValue(key, detailInfoMap) { - detailInfoMap[key] = int(repoRecord.CloneCnt) + if int(repoRecord.GitCloneCnt) > getMapKeyStringValue(key, detailInfoMap) { + detailInfoMap[key] = int(repoRecord.GitCloneCnt) mostDownloadMap[repoRecord.OwnerID] = repoRecord.DisplayName() } } From 8bad9d982e3630351750ebd84abca3dc5097d778 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 14 Dec 2022 10:55:17 +0800 Subject: [PATCH 11/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/issue.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/models/issue.go b/models/issue.go index b943aea79..0161af427 100755 --- a/models/issue.go +++ b/models/issue.go @@ -842,13 +842,20 @@ func (issue *Issue) GetLastEventLabel() string { // GetLastComment return last comment for the current issue. func (issue *Issue) GetCloseUser() (*User, error) { var c Comment - exist, err := x.Where("type = ?", CommentTypeClose). + tmp := CommentTypeClose + if issue.IsPull { + tmp = CommentTypeMergePull + } + exist, err := x.Where("type = ?", tmp). And("issue_id = ?", issue.ID).Desc("id").Get(&c) if err != nil { return nil, err } if !exist { - return nil, nil + if issue.Poster == nil { + issue.LoadPoster() + } + return issue.Poster, nil } err = c.LoadPoster() if c.Poster != nil { From 00da5e9b7947f3168cd77cee09ee0a80c626bfdb Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 14 Dec 2022 15:50:50 +0800 Subject: [PATCH 12/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E6=A8=A1=E5=9E=8B=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 48 +++++++++++++++++++++++++++++++++++--- models/user_business_struct.go | 10 ++++++++ options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/repo/user_data_analysis.go | 5 ++++ 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index b8e7a422d..e224c6c9b 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -355,6 +355,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) @@ -427,6 +428,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) @@ -546,6 +548,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].ModelConvertCount += userRecord.ModelConvertCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount @@ -607,6 +610,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix) @@ -687,6 +691,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) + dateRecordAll.ModelConvertCount = getMapValue(dateRecordAll.ID, AiModelConvertMap) dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset) dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset) dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset) @@ -998,7 +1003,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + - "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num) " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num,model_convert_count) " + "VALUES" for i, record := range dateRecords { @@ -1007,7 +1012,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," + - fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + ")" + fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + "," + fmt.Sprint(record.ModelConvertCount) + ")" if i < (len(dateRecords) - 1) { insertBatchSql += "," } @@ -1098,6 +1103,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix) + AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) @@ -1179,7 +1185,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) - + dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset) @@ -1368,6 +1374,7 @@ func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) @@ -1393,6 +1400,7 @@ func getUserActivateAll(dateRecord UserBusinessAnalysisAll) int { result += dateRecord.CreateRepoCount result += dateRecord.CloudBrainTaskNum result += dateRecord.CommitModelCount + result += dateRecord.ModelConvertCount result += dateRecord.CommitDatasetNum result += dateRecord.FocusOtherUser result += dateRecord.CollectDataset @@ -1414,6 +1422,7 @@ func getUserActivate(dateRecord UserBusinessAnalysis) int { result += dateRecord.CreateRepoCount result += dateRecord.CloudBrainTaskNum result += dateRecord.CommitModelCount + result += dateRecord.ModelConvertCount result += dateRecord.CommitDatasetNum result += dateRecord.FocusOtherUser result += dateRecord.CollectDataset @@ -1450,6 +1459,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64 result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) @@ -2278,6 +2288,38 @@ func queryUserModel(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserModelConvert(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(AiModelConvert)) + if err != nil { + log.Info("query AiModelConvert error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id").Table("ai_model_convert").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + aiModelList := make([]*AiModelConvert, 0) + sess.Find(&aiModelList) + log.Info("query AiModelConvert size=" + fmt.Sprint(len(aiModelList))) + for _, aiModelRecord := range aiModelList { + if _, ok := resultMap[aiModelRecord.UserId]; !ok { + resultMap[aiModelRecord.UserId] = 1 + } else { + resultMap[aiModelRecord.UserId] += 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) { sess := x.NewSession() defer sess.Close() diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 9dcc12342..e59945abe 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -89,6 +89,7 @@ type UserBusinessAnalysisCurrentYear struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLast30Day struct { @@ -157,6 +158,7 @@ type UserBusinessAnalysisLast30Day struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLastMonth struct { @@ -225,6 +227,7 @@ type UserBusinessAnalysisLastMonth struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisCurrentMonth struct { @@ -293,6 +296,7 @@ type UserBusinessAnalysisCurrentMonth struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisCurrentWeek struct { @@ -362,6 +366,7 @@ type UserBusinessAnalysisCurrentWeek struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisYesterday struct { @@ -431,6 +436,7 @@ type UserBusinessAnalysisYesterday struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysisLastWeek struct { @@ -500,6 +506,7 @@ type UserBusinessAnalysisLastWeek struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserAnalysisPara struct { @@ -616,6 +623,8 @@ type UserBusinessAnalysisAll struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } type UserBusinessAnalysis struct { @@ -704,4 +713,5 @@ type UserBusinessAnalysis struct { Phone string `xorm:"NULL"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` + ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"` } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 082b35ca8..c3a1fa748 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -577,6 +577,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count static.CloudBrainRunTime=CloudBrain Run Time static.CommitDatasetNum=Commit Dataset Count static.CommitModelCount=Commit Model Count +static.ModelConvertCount=Model Convert Count static.UserIndex=Normalized user index static.UserIndexPrimitive=User Index static.countdate=Count Date diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index efaf031c6..363bd6b82 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -581,6 +581,7 @@ static.CloudBrainTaskNum=云脑任务数 static.CloudBrainRunTime=云脑运行时间（小时） static.CommitDatasetNum=上传（提交）数据集文件数 static.CommitModelCount=提交模型数 +static.ModelConvertCount=模型转换数 static.UserIndex=归一化用户指数 static.UserIndexPrimitive=用户指数 static.countdate=系统统计时间 diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index a6de283a4..a6fb7d82c 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -104,6 +104,7 @@ func getExcelHeader(ctx *context.Context) map[string]string { excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.ModelConvertCount")) excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser")) excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset")) @@ -178,6 +179,8 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount) + tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) @@ -256,6 +259,8 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord * tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount) + tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) From 858850150979dbbb0eed45d257c637b4027463dd Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 15 Dec 2022 14:53:02 +0800 Subject: [PATCH 13/28] =?UTF-8?q?=E8=A7=A3=E5=86=B3issue#3388?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/aisafety.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 6176fcda5..55f25dba6 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -847,6 +847,9 @@ func createForGPU(ctx *context.Context, jobName string) error { codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath os.RemoveAll(codePath) + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(cloudbrain.DefaultBranchName) + if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) return errors.New("system error") @@ -891,7 +894,7 @@ func createForGPU(ctx *context.Context, jobName string) error { BranchName: cloudbrain.DefaultBranchName, BootFile: BootFile, Params: Params, - CommitID: "", + CommitID: commitID, ModelName: modelName, ModelVersion: modelVersion, CkptName: CkptName, From ca5130010bc08745507a1b573e4a59406e4a06ec Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 16 Dec 2022 09:24:59 +0800 Subject: [PATCH 14/28] =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=B9=B4=E5=BA=A6?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E6=8A=A5=E5=91=8A=E4=BF=A1=E6=81=AF=E6=9B=B4?= =?UTF-8?q?=E6=94=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 171 +++++++++++++++++++++++++++---------- routers/repo/cloudbrain.go | 22 ++++- routers/repo/modelarts.go | 38 +++++++++ routers/repo/user_data_analysis.go | 7 ++ routers/routes/routes.go | 4 +- 5 files changed, 192 insertions(+), 50 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index e224c6c9b..dd72fac4a 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -586,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS startTime := currentTimeNow.AddDate(0, 0, -1) CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5) + CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) @@ -602,9 +602,9 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS //log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson)) } //CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix()) - CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix) + CommitDatasetSizeMap, CommitDatasetNumMap, _ := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) - CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix) + CreateRepoCountMap, _, _ := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) @@ -613,19 +613,14 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS AiModelConvertMap := queryUserModelConvert(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) - RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix) + RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) CollectImage, CollectedImage := queryImageStars(start_unix, end_unix) RecommendImage := queryRecommedImage(start_unix, end_unix) InvitationMap := queryUserInvitationCount(start_unix, end_unix) DataDate := currentTimeNow.Format("2006-01-02") + " 00:01" - bonusMap := make(map[string]map[string]int) - if isUserYearData(tableName) { - bonusMap = getBonusMap() - log.Info("truncate all data from table:user_summary_current_year ") - statictisSess.Exec("TRUNCATE TABLE user_summary_current_year") - } + cond := "type != 1 and is_active=true" count, err := sess.Where(cond).Count(new(User)) if err != nil { @@ -724,37 +719,6 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1 } } - if isUserYearData(tableName) { - //年度数据 - subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC()) - mostActiveDay := "" - if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok { - mostActiveDay = getMostActiveJson(userInfo) - } - scoreMap := make(map[string]float64) - repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap) - dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset) - scoreMap["datasetscore"] = datasetscore - codeInfo, codescore := getCodeInfo(dateRecordAll) - scoreMap["codescore"] = codescore - cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap) - playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap) - re := &UserSummaryCurrentYear{ - ID: dateRecordAll.ID, - Name: dateRecordAll.Name, - Email: dateRecordAll.Email, - Phone: dateRecordAll.Phone, - RegistDate: dateRecordAll.RegistDate, - DateCount: int(subTime.Hours()) / 24, - MostActiveDay: mostActiveDay, - RepoInfo: repoInfo, - DataSetInfo: dataSetInfo, - CodeInfo: codeInfo, - CloudBrainInfo: cloudBrainInfo, - PlayARoll: playARoll, - } - statictisSess.Insert(re) - } } if len(dateRecordBatch) > 0 { err := insertTable(dateRecordBatch, tableName, statictisSess) @@ -784,6 +748,127 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount)) } +func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) { + sess := x.NewSession() + defer sess.Close() + log.Info("RefreshUserYearTable start....") + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + log.Info("UserYear StartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + log.Info("UserYear EndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) + + start_unix := pageStartTime.Unix() + end_unix := pageEndTime.Unix() + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryCreateIssue(start_unix, end_unix) + + CommentCountMap := queryComment(start_unix, end_unix) + + CommitCodeSizeMap, err := GetAllUserKPIStats(pageStartTime, pageEndTime) + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix) + + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + + _, CollectedDataset := queryDatasetStars(start_unix, end_unix) + _, CreatedDataset := queryRecommedDataSet(start_unix, end_unix) + + bonusMap := getBonusMap() + log.Info("truncate all data from table:user_summary_current_year ") + statictisSess.Exec("TRUNCATE TABLE user_summary_current_year") + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") + return + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + for _, userRecord := range userList { + var dateRecordAll UserBusinessAnalysisAll + dateRecordAll.ID = userRecord.ID + dateRecordAll.Email = userRecord.Email + dateRecordAll.Phone = userRecord.PhoneNumber + dateRecordAll.RegistDate = userRecord.CreatedUnix + dateRecordAll.Name = userRecord.Name + + dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap) + dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap) + dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap) + dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap) + + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + //dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap) + dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap) + dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap) + dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap) + dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap) + + dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap) + dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) + + //年度数据 + subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC()) + mostActiveDay := "" + if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok { + mostActiveDay = getMostActiveJson(userInfo) + } + scoreMap := make(map[string]float64) + repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap) + dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset) + scoreMap["datasetscore"] = datasetscore + codeInfo, codescore := getCodeInfo(dateRecordAll) + scoreMap["codescore"] = codescore + cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap) + playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap) + re := &UserSummaryCurrentYear{ + ID: dateRecordAll.ID, + Name: dateRecordAll.Name, + Email: dateRecordAll.Email, + Phone: dateRecordAll.Phone, + RegistDate: dateRecordAll.RegistDate, + DateCount: int(subTime.Hours()) / 24, + MostActiveDay: mostActiveDay, + RepoInfo: repoInfo, + DataSetInfo: dataSetInfo, + CodeInfo: codeInfo, + CloudBrainInfo: cloudBrainInfo, + PlayARoll: playARoll, + } + statictisSess.Insert(re) + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + log.Info("update user year data finished. ") +} + func isUserYearData(tableName string) bool { if tableName == "user_business_analysis_current_year" { currentTimeNow := time.Now() @@ -1504,10 +1589,6 @@ func getInt(str string) int { return int(re) } -func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { - CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) -} - func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a23cd5462..27c1ece39 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -711,7 +711,6 @@ func CloudBrainRestart(ctx *context.Context) { }) } - func getOldJobPath(task *models.Cloudbrain) string { return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName } @@ -854,10 +853,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile!=""{ - ctx.Redirect(getFileUrl(debugUrl,task.BootFile)) + if task.BootFile != "" { + ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) - }else{ + } else { ctx.Redirect(debugUrl) } @@ -1581,6 +1580,21 @@ func CloudBrainDownloadModel(ctx *context.Context) { ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } + +func CloudBrainDownloadMultiModel(ctx *context.Context) { + parentDir := ctx.Query("parentDir") + jobName := ctx.Query("jobName") + filePath := "jobs/" + jobName + "/model/" + parentDir + allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath) + if err == nil { + returnFileName := jobName + ".zip" + MinioDownloadManyFile(filePath, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } +} + func CloudBrainDownloadInferenceResult(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 4e30e625d..e0eb678e3 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2582,6 +2582,44 @@ func InferenceJobShow(ctx *context.Context) { ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow) } +func MultiModelDownload(ctx *context.Context) { + var ( + err error + ) + jobID := ctx.Params(":jobid") + versionName := ctx.Query("version_name") + parentDir := ctx.Query("parent_dir") + + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error()) + return + } + + if task.ComputeResource == models.NPUResource { + path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir), "/") + allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) + if err == nil { + returnFileName := task.DisplayJobName + ".zip" + ObsDownloadManyFile(path, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } + } else if task.ComputeResource == models.GPUResource { + filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir + allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath) + if err == nil { + returnFileName := task.DisplayJobName + ".zip" + MinioDownloadManyFile(filePath, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } + } + +} + func ModelDownload(ctx *context.Context) { var ( err error diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index a6fb7d82c..9a5661c65 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -21,6 +21,7 @@ import ( const ( PAGE_SIZE = 2000 Excel_File_Path = "/useranalysis/" + USER_YEAR = 2022 ) func getUserMetricsExcelHeader(ctx *context.Context) map[string]string { @@ -719,6 +720,12 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Info("startTime time:" + startTime.Format("2006-01-02 15:04:05")) log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05")) warnEmailMessage := "用户统计信息入库失败，请尽快定位。" + + startYear := time.Date(USER_YEAR, 0, 0, 0, 0, 0, 1, t.Location()) + endYear := startYear.AddDate(1, 0, 0) + + models.RefreshUserYearTable(startYear, endYear) + //query wiki data log.Info("start to time count data") wikiMap, err := queryWikiCountMap(startTime, endTime) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 80ff96364..9976be902 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1175,6 +1175,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel) + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel) }) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) @@ -1198,6 +1199,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel) //m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel) + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel) //m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir) //m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew) @@ -1210,7 +1212,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult) - + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel) m.Get("/downloadall", repo.DownloadInferenceResultFile) }) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew) From 86e0a2c8d18ab3b77e35b435a1ede14e6821ca94 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 16 Dec 2022 09:39:43 +0800 Subject: [PATCH 15/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=97=A5=E6=9C=9F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 9a5661c65..8121f167c 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -721,7 +721,7 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05")) warnEmailMessage := "用户统计信息入库失败，请尽快定位。" - startYear := time.Date(USER_YEAR, 0, 0, 0, 0, 0, 1, t.Location()) + startYear := time.Date(USER_YEAR, 1, 1, 0, 0, 0, 1, t.Location()) endYear := startYear.AddDate(1, 0, 0) models.RefreshUserYearTable(startYear, endYear) From 7e7f3eb3e7d0a556656a3c408a66bffe7b7d90a2 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 20 Dec 2022 16:11:26 +0800 Subject: [PATCH 16/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/cloudbrain.go | 12 ++++++------ routers/api/v1/api.go | 2 ++ routers/api/v1/repo/modelmanage.go | 32 ++++++++++++++++++++++++++++++++ routers/repo/ai_model_manage.go | 33 ++++++++++++++++++++------------- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 37965c73a..409bb494a 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -205,7 +205,7 @@ type Cloudbrain struct { BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp - Cleared bool `xorm:"DEFAULT false"` + Cleared bool `xorm:"DEFAULT false"` Spec *Specification `xorm:"-"` } @@ -1988,7 +1988,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } -func IsCloudbrainExistByJobName(jobName string)(bool,error){ +func IsCloudbrainExistByJobName(jobName string) (bool, error) { return x.Unscoped().Exist(&Cloudbrain{ JobName: jobName, }) @@ -2172,25 +2172,25 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra Limit(limit). Find(&cloudbrains) } + /** 本方法考虑了再次调试的情况，多次调试取最后一次的任务的结束时间 - */ +*/ func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 - sql:=`SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) + sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) id, job_name, job_id,status,end_time,updated_unix,cleared FROM cloudbrain where type=0 and job_type='DEBUG' ORDER BY job_name, updated_unix DESC) a where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix Date: Tue, 20 Dec 2022 16:24:18 +0800 Subject: [PATCH 17/28] =?UTF-8?q?issue=E5=85=B3=E9=97=AD=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_zh-CN.ini | 2 +- templates/repo/issue/list.tmpl | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index f003c7d85..bf1eb1740 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1675,7 +1675,7 @@ pulls.merged_by=由 %[3]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 issues.closed_by=由 %[3]s 关闭于 %[1]s issues.opened_by_fake=由 %[2]s 于 %[1]s创建 -issues.closed_by_fake=通过 %[2]s 关闭 %[1]s +issues.closed_by_fake=由 %[2]s 创建，被关闭于 %[1]s issues.previous=上一页 issues.next=下一页 issues.open_title=开启中 diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 282774ddc..1e29b6512 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -270,11 +270,7 @@ {{if .OriginalAuthor }} {{$.i18n.Tr .GetLastEventLabelFake $timeStr .OriginalAuthor | Safe}} {{else if gt .Poster.ID 0}} - {{if .IsClosed}} - {{$.i18n.Tr .GetLastEventLabel $timeStr .GetCloseUser.HomeLink (.GetCloseUser.GetDisplayName | Escape) | Safe}} - {{else}} - {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} - {{end}} + {{$.i18n.Tr .GetLastEventLabel $timeStr .Poster.HomeLink (.Poster.GetDisplayName | Escape) | Safe}} {{else}} {{$.i18n.Tr .GetLastEventLabelFake $timeStr (.Poster.GetDisplayName | Escape) | Safe}} {{end}} From 122d4b228a52f0ad69320fc800a5bf8989d61a6b Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 20 Dec 2022 16:31:46 +0800 Subject: [PATCH 18/28] =?UTF-8?q?issue=E5=85=B3=E9=97=AD=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_zh-CN.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index bf1eb1740..a79091932 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1673,9 +1673,9 @@ issues.action_assignee_no_select=未指派 issues.opened_by=由 %[3]s 于 %[1]s创建 pulls.merged_by=由 %[3]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 -issues.closed_by=由 %[3]s 关闭于 %[1]s +issues.closed_by=由 %[3]s 创建，被关闭于 %[1]s issues.opened_by_fake=由 %[2]s 于 %[1]s创建 -issues.closed_by_fake=由 %[2]s 创建，被关闭于 %[1]s +issues.closed_by_fake=通过 %[2]s 关闭 %[1]s issues.previous=上一页 issues.next=下一页 issues.open_title=开启中 From d8ec649a37f4ea6a006249e680c94b62f12109fc Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 20 Dec 2022 16:43:48 +0800 Subject: [PATCH 19/28] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=97=B6=E9=80=89=E6=8B=A9=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=9A=84api=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/cloudbrain.go | 6 +++--- routers/api/v1/repo/modelmanage.go | 2 +- routers/repo/ai_model_manage.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 409bb494a..7e736e19d 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1823,7 +1823,7 @@ func QueryModelTrainJobVersionList(jobId string) ([]*Cloudbrain, int, error) { return cloudbrains, int(len(cloudbrains)), nil } -func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) { +func QueryModelTrainJobList(repoId int64) ([]*Cloudbrain, int, error) { sess := x.NewSession() defer sess.Close() var cond = builder.NewCond() @@ -1840,14 +1840,14 @@ func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) { // builder.In("type", 0, 1), // ) - cloudbrains := make([]*CloudbrainInfo, 0) + cloudbrains := make([]*Cloudbrain, 0) if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC"). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } keys := make(map[string]string) - uniqueElements := make([]*CloudbrainInfo, 0) + uniqueElements := make([]*Cloudbrain, 0) for _, entry := range cloudbrains { if _, value := keys[entry.JobID]; !value { keys[entry.JobID] = entry.DisplayJobName diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index efa215259..696231fea 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -64,7 +64,7 @@ func QueryTrainJobList(ctx *context.APIContext) { } else { re := make([]*api.Cloudbrain, 0) for _, task := range result { - conRe := convert.ToCloudBrain(&task.Cloudbrain) + conRe := convert.ToCloudBrain(task) re = append(re, conRe) } ctx.JSON(http.StatusOK, re) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 74bd45988..a074119fc 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -741,7 +741,7 @@ func QueryTrainJobList(ctx *context.Context) { } } -func QueryTrainJobListApi(ctx *context.Context) ([]*models.CloudbrainInfo, error) { +func QueryTrainJobListApi(ctx *context.Context) ([]*models.Cloudbrain, error) { repoId := ctx.QueryInt64("repoId") VersionListTasks, count, err := models.QueryModelTrainJobList(repoId) log.Info("query return count=" + fmt.Sprint(count)) From 2367ba88cdd577272ec927e99ea2deadc72850d8 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 20 Dec 2022 16:47:58 +0800 Subject: [PATCH 20/28] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=97=B6=E9=80=89=E6=8B=A9=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=9A=84api=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/convert/cloudbrain.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/convert/cloudbrain.go b/modules/convert/cloudbrain.go index 599da4800..13c1d7c31 100644 --- a/modules/convert/cloudbrain.go +++ b/modules/convert/cloudbrain.go @@ -34,8 +34,7 @@ func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain { StartTime: int64(task.StartTime), EndTime: int64(task.EndTime), - - Spec: ToSpecification(task.Spec), + Spec: ToSpecification(task.Spec), } } func ToAttachment(attachment *models.Attachment) *api.AttachmentShow { @@ -89,6 +88,9 @@ func ToDataset(dataset *models.Dataset) *api.Dataset { } func ToSpecification(s *models.Specification) *api.SpecificationShow { + if s == nil { + return nil + } return &api.SpecificationShow{ ID: s.ID, AccCardsNum: s.AccCardsNum, From ab729b4f28a34f31f0b36a3ac3eac383d812a44c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 21 Dec 2022 12:42:23 +0800 Subject: [PATCH 21/28] notebook multifile --- models/cloudbrain.go | 8 + modules/cloudbrain/cloudbrain.go | 1 + modules/modelarts/modelarts.go | 84 +-------- modules/modelarts_cd/modelarts.go | 1 + modules/notebook/contentManager.go | 198 ++++++++++++++++++++ modules/structs/cloudbrain.go | 1 + options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 2 +- routers/api/v1/api.go | 3 +- routers/api/v1/repo/cloudbrain.go | 3 + routers/repo/modelarts.go | 5 +- services/cloudbrain/cloudbrainTask/notebook.go | 243 ++++++++++++++++++++++--- 12 files changed, 435 insertions(+), 116 deletions(-) create mode 100644 modules/notebook/contentManager.go diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 37965c73a..099f07c3a 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2126,6 +2126,14 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobName: jobName} return getRepoCloudBrain(cb) } +func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) { + cloudBrain := new(Cloudbrain) + has, err := x.In("status", JobWaiting, JobRunning).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) + if has { + return cloudBrain, err + } + return nil, err +} func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index c85f4b8cd..827e2b736 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -709,6 +709,7 @@ type GenerateModelArtsNotebookReq struct { ImageId string AutoStopDurationMs int64 + BranchName string Spec *models.Specification ModelName string diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index dcad1eb00..bd7f848fc 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,9 @@ package modelarts import ( - "encoding/base64" "encoding/json" "errors" "fmt" - "io/ioutil" - "net/http" - "path" "strconv" "strings" @@ -239,6 +235,7 @@ func GenerateNotebook2(ctx *context.Context, req cloudbrain.GenerateModelArtsNot ComputeResource: models.NPUResource, Image: imageName, BootFile: req.BootFile, + BranchName: req.BranchName, Description: req.Description, CreatedUnix: createTime, UpdatedUnix: createTime, @@ -830,10 +827,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { task.FlavorCode = result.Flavor } - if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" { - uploadNoteBookFile(task, result) - - } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -844,81 +837,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { return nil } -func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { - jupyterUrl := result.Url + "?token=" + result.Token - - cookies, xsrf := getCookiesAndCsrf(jupyterUrl) - if xsrf == "" { - log.Error("browser jupyterUrl failed:%v", task.DisplayJobName) - } else { - - codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath - fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) - if err != nil { - log.Error("read jupyter file failed:%v", task.DisplayJobName, err) - } - - base64Content := base64.StdEncoding.EncodeToString(fileContents) - client := getRestyClient() - uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile) - res, err := client.R(). - SetCookies(cookies). - SetHeader("X-XSRFToken", xsrf). - SetBody(map[string]interface{}{ - "type": "file", - "format": "base64", - "name": path.Base(task.BootFile), - "path": path.Base(task.BootFile), - "content": base64Content}). - Put(uploadUrl) - if err != nil { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } else if res.StatusCode() != http.StatusCreated { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } - - } - -} - -func getJupyterBaseUrl(url string) string { - jupyterUrlLength := len(url) - baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))] - return baseUrl -} - -func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) { - log.Info("jupyter url:" + jupyterUrl) - var cookies []*http.Cookie - const retryTimes = 10 - for i := 0; i < retryTimes; i++ { - res, err := http.Get(jupyterUrl) - if err != nil { - log.Error("browser jupyterUrl failed.", err) - if i == retryTimes-1 { - return cookies, "" - } - - } else { - cookies = res.Cookies() - xsrf := "" - for _, cookie := range cookies { - if cookie.Name == "_xsrf" { - xsrf = cookie.Value - break - } - - } - if xsrf != "" { - return cookies, xsrf - } - - } - } - return cookies, "" - -} - func SyncTempStatusJob() { jobs, err := models.GetCloudBrainTempJobs() if err != nil { diff --git a/modules/modelarts_cd/modelarts.go b/modules/modelarts_cd/modelarts.go index bdc42002a..9d74c0919 100755 --- a/modules/modelarts_cd/modelarts.go +++ b/modules/modelarts_cd/modelarts.go @@ -148,6 +148,7 @@ func GenerateNotebook(ctx *context.Context, req cloudbrain.GenerateModelArtsNote UpdatedUnix: createTime, Spec: req.Spec, BootFile: req.BootFile, + BranchName: req.BranchName, ModelName: req.ModelName, ModelVersion: req.ModelVersion, LabelName: req.LabelName, diff --git a/modules/notebook/contentManager.go b/modules/notebook/contentManager.go new file mode 100644 index 000000000..823e59665 --- /dev/null +++ b/modules/notebook/contentManager.go @@ -0,0 +1,198 @@ +package notebook + +import ( + "crypto/tls" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "path" + "strings" + + "github.com/go-resty/resty/v2" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/log" +) + +var restyClient *resty.Client + +type NotebookApiResponse struct { + Name string `json:"name"` + Path string `json:"path"` +} + +type NotebookContent struct { + Url string + Path string + Cookies []*http.Cookie + Xsrf string + PathType string //file directory + Token string +} + +func (c *NotebookContent) IsNotebookFileCanBrowser() bool { + if c.Xsrf == "" { + c.SetCookiesAndCsrf() + } + if c.Xsrf == "" { + log.Warn("xsrf is empty, can not broswer url:" + c.Url) + return false + } + return c.IsNoteBookContentsExist() + +} + +func (c *NotebookContent) SetCookiesAndCsrf() { + log.Info("jupyter url:" + c.Url) + var cookies []*http.Cookie + const retryTimes = 10 + url := c.Url + if c.Token != "" { + url = c.Url + "?token=" + c.Token + } + for i := 0; i < retryTimes; i++ { + res, err := http.Get(url) + if err != nil { + log.Error("browser jupyterUrl failed.", err) + if i == retryTimes-1 { + c.Cookies = cookies + } + + } else { + cookies = res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + if len(cookies) > 1 { + break + } + + } + + } + if xsrf != "" { + c.Cookies = cookies + c.Xsrf = xsrf + } + + } + } + c.Cookies = cookies + +} + +func (c *NotebookContent) IsNoteBookContentsExist() bool { + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + "?type=" + c.PathType + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + Get(uploadUrl) + if err != nil { + log.Warn("browser url error:"+uploadUrl, err) + return false + } + return res.StatusCode() == http.StatusOK +} + +func (c *NotebookContent) UploadNoteBookFile(task *models.Cloudbrain) error { + + err := c.MakeNoteBookDir() + if err != nil { + return err + } + + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath + fileContents, err := ioutil.ReadFile(codePath + "/" + c.Path) + if err != nil { + log.Error("read jupyter file failed:%v", task.DisplayJobName, err) + } + + base64Content := base64.StdEncoding.EncodeToString(fileContents) + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": "file", + "format": "base64", + "name": path.Base(c.Path), + "path": c.Path, + "content": base64Content}). + Put(uploadUrl) + if err != nil { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + return err + } else if res.StatusCode() != http.StatusCreated { + log.Error("upload jupyter file failed:%v, status is %s", task.DisplayJobName, res.Status(), err) + return fmt.Errorf("status:", res.StatusCode()) + } + return nil +} + +/** + if c.Path is a/b/c.txt + makedir a/b + if c.Path is a/b/c + makedir a/b +*/ +func (c *NotebookContent) MakeNoteBookDir() error { + filePaths := strings.Split(c.Path, "/") + + for i := 0; i < len(filePaths)-1; i++ { + cTemp := &NotebookContent{ + Url: c.Url, + Cookies: c.Cookies, + Path: path.Join(filePaths[0 : i+1]...), + PathType: "directory", + Xsrf: c.Xsrf, + } + if !cTemp.IsNoteBookContentsExist() { + + createTempDirUrl := getJupyterBaseUrl(cTemp.Url) + "api/contents/" + cTemp.Path + client := getRestyClient() + var jobResult NotebookApiResponse + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": cTemp.PathType, + "path": cTemp.Path, + }).SetResult(&jobResult). + Put(createTempDirUrl) + if err != nil { + return err + } + if res.StatusCode() != http.StatusCreated { + return fmt.Errorf("status code:" + res.Status()) + } + + } + + } + return nil +} + +func getJupyterBaseUrl(url string) string { + jupyterUrlLength := len(url) + baseUrl := url + if strings.HasSuffix(url, "lab") { + baseUrl = url[0 : jupyterUrlLength-len(path.Base(url))] + } + + return baseUrl +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + return restyClient +} diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 9ea5601c9..e4403ed04 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -47,6 +47,7 @@ type CreateFileNotebookJobOption struct { BranchName string `json:"branch_name" binding:"Required"` OwnerName string `json:"owner_name" binding:"Required"` ProjectName string `json:"project_name" binding:"Required"` + JobId string `json:"job_id"` } type Cloudbrain struct { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 702c17bce..f86879b06 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1061,7 +1061,7 @@ model_rename=Duplicate model name, please modify model name. notebook_file_not_exist=Notebook file does not exist. notebook_select_wrong=Please select a Notebook(.ipynb) file first. notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. -debug_again_fail=Fail to restart debug task, please try again later. +notebook_repo_conflict=The files in different branches of the same repository can not run together. date=Date repo_add=Project Increment diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index bf609ba14..cfd58d6b9 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1060,7 +1060,7 @@ model_rename=模型名称重复，请修改模型名称 notebook_file_not_exist=Notebook文件不存在。 notebook_select_wrong=请先选择Notebook（.ipynb）文件。 notebook_file_no_right=您没有这个Notebook文件的读权限。 -debug_again_fail=再次调试失败，请稍后再试。 +notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。 date=日期 repo_add=新增项目 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 4936c2362..6a8090cd4 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -745,7 +745,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/file_notebook", func() { m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) - + //m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) + m.Post("/status", bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) }) m.Group("/repos", func() { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 805443788..250a08423 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -110,6 +110,9 @@ func GeneralCloudBrainJobStop(ctx *context.APIContext) { func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { cloudbrainTask.FileNotebookCreate(ctx.Context, option) } +func FileNoteBookStatus(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { + cloudbrainTask.FileNotebookStatus(ctx.Context, option) +} func GetFileNoteBookInfo(ctx *context.APIContext) { //image description spec description waiting count diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 3dbe101a8..76de80f71 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -440,10 +440,9 @@ func NotebookDebug2(ctx *context.Context) { return } if task.BootFile != "" { - ctx.Redirect(getFileUrl(result.Url, task.BootFile) + "?token=" + result.Token) - } else { - ctx.Redirect(result.Url + "?token=" + result.Token) + go cloudbrainTask.UploadNotebookFiles(task) } + ctx.Redirect(result.Url + "?token=" + result.Token) } diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index cc9563520..766966843 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -4,6 +4,9 @@ import ( "fmt" "net/http" "path" + "strings" + + "code.gitea.io/gitea/modules/notebook" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts_cd" @@ -29,6 +32,9 @@ import ( ) const NoteBookExtension = ".ipynb" +const CPUType = 0 +const GPUType = 1 +const NPUType = 2 func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { @@ -66,7 +72,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } //create repo if not exist - repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) + repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ Name: setting.FileNoteBook.ProjectName, @@ -80,17 +86,200 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp AutoInit: true, DefaultBranch: "master", }) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + return + } + } else { + + noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type)) + if noteBook != nil { + + if isRepoConfilcts(option, noteBook) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict"))) + return + } + + if isNotebookSpecMath(option, noteBook) { + + err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) + noteBook.BranchName += ";" + option.BranchName + err := models.UpdateJob(noteBook) + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: noteBook.JobID, + }) + return + } + + } + } + + if option.Type <= GPUType { + cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + } else { + modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) } + +} +func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) { + if ctx.Written() { + return + } + + if path.Ext(option.File) != NoteBookExtension { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) + return + } + + isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) + if !isNotebookFileExist { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + task, err := models.GetCloudbrainByJobID(option.JobId) if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + log.Error("job not found:"+option.JobId, err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job.")) return } - if option.Type <= 1 { - cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) { + log.Warn("Boot file is empty or status is running. ") + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running.")) + return + } + if !isRepoFileMatch(option, task) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file.")) + return + } + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) { + ctx.JSON(http.StatusOK, models.BaseOKMessageApi) } else { - modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed.")) + + } + +} + +func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) { + var debugBaseUrl string + var token string + if task.Type == models.TypeCloudBrainOne { + debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab" + + } else { + var result *models.GetNotebook2Result + var err error + if task.Type == models.TypeCloudBrainTwo { + result, err = modelarts.GetNotebook2(task.JobID) + } else if task.Type == models.TypeCDCenter { + result, err = modelarts_cd.GetNotebook(task.JobID) + } + if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" { + log.Error("notebook job not found:"+task.JobID, err) + return "", "", fmt.Errorf("can not get job or job is invalid.") + } + + debugBaseUrl = result.Url + token = result.Token + + } + return debugBaseUrl, token, nil +} + +func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool { + c := ¬ebook.NotebookContent{ + Url: debugBaseUrl, + Path: bootFile, + PathType: "file", + Token: token, + } + if c.IsNotebookFileCanBrowser() { + return true + } else { + c.SetCookiesAndCsrf() + c.UploadNoteBookFile(task) + return c.IsNotebookFileCanBrowser() + } + +} + +func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + if option.Type == NPUType || option.Type == CPUType { + return true + } + spec, err := models.GetCloudbrainSpecByID(book.ID) + if err != nil { + log.Warn("can not get spec ", err) + return false + } + return spec.AccCardsNum > 0 +} + +func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName { + return true + } + } } + return false + +} + +func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile { + return true + } + } + + return false + +} +func UploadNotebookFiles(task *models.Cloudbrain) { + if task.Status == string(models.JobRunning) && task.BootFile != "" { + + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + log.Error("can not get base url:", err) + return + } + bootFiles := strings.Split(task.BootFile, ";") + + for _, bootFile := range bootFiles { + uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token) + } + + } } func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { @@ -131,17 +320,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return } } - errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) - if errStr != "" { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return } command := cloudbrain.GetCloudbrainDebugCommand() @@ -185,7 +375,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot JobType: jobType, Description: getDescription(option), BranchName: option.BranchName, - BootFile: option.File, + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), Params: "{\"parameter\":[]}", CommitID: "", BenchmarkTypeID: 0, @@ -206,8 +396,15 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } -func getCodePath(jobName string) string { - return setting.JobPath + jobName + cloudbrain.CodeMountPath +func getCloudbrainType(optionType int) int { + if optionType < 1 { + return models.TypeCloudBrainOne + } + return models.TypeCloudBrainTwo +} + +func getCodePath(jobName string, repo *models.Repository) string { + return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name } func getDescription(option api.CreateFileNotebookJobOption) string { @@ -237,8 +434,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return @@ -260,7 +457,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } } - err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return @@ -299,6 +496,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote Spec: spec, BootFile: "", AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, + BranchName: option.BranchName, } if setting.ModelartsCD.Enabled { @@ -347,17 +545,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO return true, nil } -func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { - err := downloadCode(repo, codePath, branchName) - if err != nil { - return "cloudbrain.load_code_failed" - } - - err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") - if err != nil { - return "cloudbrain.load_code_failed" - } - return "" +func getBootFile(filePath string, ownerName string, projectName string) string { + return ownerName + "/" + projectName + "/" + filePath } func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { From 2dd91ddba92049cb60016b90ce0a0ed721e817e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 21 Dec 2022 15:03:33 +0800 Subject: [PATCH 22/28] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/issue.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/models/issue.go b/models/issue.go index 0161af427..3ed49ce42 100755 --- a/models/issue.go +++ b/models/issue.go @@ -840,34 +840,6 @@ func (issue *Issue) GetLastEventLabel() string { } // GetLastComment return last comment for the current issue. -func (issue *Issue) GetCloseUser() (*User, error) { - var c Comment - tmp := CommentTypeClose - if issue.IsPull { - tmp = CommentTypeMergePull - } - exist, err := x.Where("type = ?", tmp). - And("issue_id = ?", issue.ID).Desc("id").Get(&c) - if err != nil { - return nil, err - } - if !exist { - if issue.Poster == nil { - issue.LoadPoster() - } - return issue.Poster, nil - } - err = c.LoadPoster() - if c.Poster != nil { - return c.Poster, nil - } - if issue.Poster == nil { - issue.LoadPoster() - } - return issue.Poster, nil -} - -// GetLastComment return last comment for the current issue. func (issue *Issue) GetLastComment() (*Comment, error) { var c Comment exist, err := x.Where("type = ?", CommentTypeComment). From 1418e860bc4a7f986b3fd778ba51c8a3d2d00dcd Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 21 Dec 2022 15:04:56 +0800 Subject: [PATCH 23/28] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 0cbcfba8f..d5ab871ce 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -2199,8 +2199,8 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) (map[int64]int, map[s detailInfoMap[key] = getMapKeyStringValue(key, detailInfoMap) + int(repoRecord.CloneCnt) key = fmt.Sprint(repoRecord.OwnerID) + "_most_download" - if int(repoRecord.GitCloneCnt) > getMapKeyStringValue(key, detailInfoMap) { - detailInfoMap[key] = int(repoRecord.GitCloneCnt) + if int(repoRecord.CloneCnt) > getMapKeyStringValue(key, detailInfoMap) { + detailInfoMap[key] = int(repoRecord.CloneCnt) mostDownloadMap[repoRecord.OwnerID] = repoRecord.DisplayName() } } From 5ebd80c32ef7b37b553a0e4293883c0a71ceef48 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 21 Dec 2022 15:49:43 +0800 Subject: [PATCH 24/28] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 3 +- routers/api/v1/api.go | 3 +- routers/repo/cloudbrain.go | 7 ++-- routers/repo/modelarts.go | 13 ++++--- services/cloudbrain/cloudbrainTask/notebook.go | 50 ++++++++++++++++++-------- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 099f07c3a..f94969605 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2128,7 +2128,8 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { } func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) { cloudBrain := new(Cloudbrain) - has, err := x.In("status", JobWaiting, JobRunning).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) + has, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, + ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) if has { return cloudBrain, err } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6a8090cd4..cae6c01bf 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -745,8 +745,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/file_notebook", func() { m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) - //m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) - m.Post("/status", bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) + m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) }) m.Group("/repos", func() { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 905c25a64..3b45e3074 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -911,10 +911,13 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile != "" { - ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(debugUrl, ctx.QueryTrim("file"))) } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } ctx.Redirect(debugUrl) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 76de80f71..2d555fc9a 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -439,10 +439,15 @@ func NotebookDebug2(ctx *context.Context) { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) return } - if task.BootFile != "" { - go cloudbrainTask.UploadNotebookFiles(task) + + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(result.Url, ctx.QueryTrim("file")) + "?token=" + result.Token) + } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } + ctx.Redirect(result.Url + "?token=" + result.Token) } - ctx.Redirect(result.Url + "?token=" + result.Token) } @@ -629,7 +634,7 @@ func NotebookStop(ctx *context.Context) { if task.Status != string(models.ModelArtsRunning) { log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" - errorMsg = "the job is not running" + errorMsg = ctx.Tr("cloudbrain.Already_stopped") break } diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 766966843..4a06bbc79 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -101,20 +101,25 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } if isNotebookSpecMath(option, noteBook) { - - err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) - if err != nil { - log.Error("download code failed", err) - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) - return + if !isRepoMatch(option, noteBook) { + err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } } - noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) - noteBook.BranchName += ";" + option.BranchName - err := models.UpdateJob(noteBook) - if err != nil { - log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) - return + if !isRepoFileMatch(option, noteBook) { + noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) + noteBook.BranchName += ";" + option.BranchName + noteBook.Description += ";" + getDescription(option) + + err := models.UpdateJob(noteBook) + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } } ctx.JSON(http.StatusOK, models.BaseMessageApi{ @@ -252,6 +257,21 @@ func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudb } +func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + + for _, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName { + return true + } + } + } + return false + +} + func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { bootFiles := strings.Split(book.BootFile, ";") branches := strings.Split(book.BranchName, ";") @@ -494,9 +514,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote Description: getDescription(option), ImageId: setting.FileNoteBook.ImageIdNPU, Spec: spec, - BootFile: "", + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, - BranchName: option.BranchName, + BranchName: option.BranchName, } if setting.ModelartsCD.Enabled { From ec1cb958d10cc21d702cb1090fb76f7ea8147a30 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 22 Dec 2022 16:00:03 +0800 Subject: [PATCH 25/28] fix issue --- web_src/vuepages/apis/modules/notobook.js | 13 +++- web_src/vuepages/pages/notebook/debug/index.vue | 92 +++++++++++++++++-------- 2 files changed, 74 insertions(+), 31 deletions(-) diff --git a/web_src/vuepages/apis/modules/notobook.js b/web_src/vuepages/apis/modules/notobook.js index 4c59a253b..1dd35e337 100644 --- a/web_src/vuepages/apis/modules/notobook.js +++ b/web_src/vuepages/apis/modules/notobook.js @@ -29,7 +29,7 @@ export const getCb1Notebook = (path,jobid) => { }); }; -// Notebook获取云脑I调试任务状态 +// Notebook获取云脑II调试任务状态 export const getCb2Notebook = (path,jobid) => { return service({ url: `/api/v1/${path}/modelarts/notebook/${jobid}`, @@ -37,7 +37,16 @@ export const getCb2Notebook = (path,jobid) => { params: {}, }); }; - +// Notebook查询文件在环境中是否已准备好 +// type, file, branch_name, owner_name, project_name,job_id +export const getFileInfoNotebook = (data) => { + return service({ + url: "/api/v1/file_notebook/status", + method: "post", + data, + params: {}, + }); +}; export const stopNotebook = (url) => { return service({ url: url, diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue index 20af220f5..63d45b5ee 100644 --- a/web_src/vuepages/pages/notebook/debug/index.vue +++ b/web_src/vuepages/pages/notebook/debug/index.vue @@ -178,9 +178,9 @@