From 692385810ae03b9ad7f613c9645b1dd424ac94cf Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 12 Apr 2022 19:46:36 +0800 Subject: [PATCH 01/29] opt --- routers/repo/cloudbrain.go | 6 ++++-- routers/repo/modelarts.go | 14 ++++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 0d007a27d..fecd34faf 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -53,6 +53,7 @@ var ( ) const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types" +const CLONE_FILE_PREFIX = "file:///" var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0) @@ -909,7 +910,8 @@ func GetRate(ctx *context.Context) { } func downloadCode(repo *models.Repository, codePath, branchName string) error { - if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName}); err != nil { + //add "file:///" prefix to make the depth valid + if err := git.Clone(CLONE_FILE_PREFIX+repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil { log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) return err } @@ -969,7 +971,7 @@ func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepo return err } - if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + if err := git.Clone(CLONE_FILE_PREFIX+repoExt.RepoPath(), codePath, git.CloneRepoOptions{Depth: 1}); err != nil { log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 8b785a395..29387133d 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1023,10 +1023,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ - Branch: branch_name, - }); err != nil { - log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) + if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { + log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Create task failed, server timed out", tplModelArtsTrainJobNew, &form) return @@ -1241,9 +1239,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ - Branch: branch_name, - }); err != nil { + if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { log.Error("Failed git clone repo to local(！: %s (%v)", repo.FullName(), err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed git clone repo to local！", tplModelArtsTrainJobVersionNew, &form) @@ -1867,9 +1863,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ - Branch: branch_name, - }); err != nil { + if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Create task failed, server timed out", tplModelArtsInferenceJobNew, &form) From 4910fbe08e1e25612f7435431380e18c1654c039 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 10:51:52 +0800 Subject: [PATCH 02/29] no need reqin for complete multipart --- routers/repo/attachment.go | 43 ------------------------------------------- routers/routes/routes.go | 3 +-- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 3c66a3537..b2ded263e 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -11,7 +11,6 @@ import ( "fmt" "mime/multipart" "net/http" - "path" "strconv" "strings" @@ -830,20 +829,6 @@ func GetMultipartUploadUrl(ctx *context.Context) { }) } -func GetObsKey(ctx *context.Context) { - uuid := gouuid.NewV4().String() - key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/") - - ctx.JSON(200, map[string]string{ - "uuid": uuid, - "key": key, - "access_key_id": setting.AccessKeyID, - "secret_access_key": setting.SecretAccessKey, - "server": setting.Endpoint, - "bucket": setting.Bucket, - }) -} - func CompleteMultipart(ctx *context.Context) { uuid := ctx.Query("uuid") uploadID := ctx.Query("uploadID") @@ -947,34 +932,6 @@ func CompleteMultipart(ctx *context.Context) { }) } -func UpdateMultipart(ctx *context.Context) { - uuid := ctx.Query("uuid") - partNumber := ctx.QueryInt("chunkNumber") - etag := ctx.Query("etag") - - fileChunk, err := models.GetFileChunkByUUID(uuid) - if err != nil { - if models.IsErrFileChunkNotExist(err) { - ctx.Error(404) - } else { - ctx.ServerError("GetFileChunkByUUID", err) - } - return - } - - fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber)+"-"+strings.Replace(etag, "\"", "", -1)) - - err = models.UpdateFileChunk(fileChunk) - if err != nil { - ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) - return - } - - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) -} - func HandleUnDecompressAttachment() { attachs, err := models.GetUnDecompressAttachments() if err != nil { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 9df429e8b..e6bda142b 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -608,12 +608,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Put("/obs_proxy_multipart", repo.PutOBSProxyUpload) m.Get("/obs_proxy_download", repo.GetOBSProxyDownload) m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) - m.Post("/complete_multipart", repo.CompleteMultipart) - m.Post("/update_chunk", repo.UpdateMultipart) }, reqSignIn) m.Group("/attachments", func() { m.Post("/decompress_done_notify", repo.UpdateAttachmentDecompressState) + m.Post("/complete_multipart", repo.CompleteMultipart) }) m.Group("/attachments", func() { From adf0af490068eb1b697c22b6de9dff5e8dbc419c Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 11:31:06 +0800 Subject: [PATCH 03/29] fix 1939 --- modules/storage/obs.go | 85 ++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 08a354359..44de3a25e 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -59,21 +59,55 @@ func ObsHasObject(path string) (bool, error) { return hasObject, nil } +func listAllParts(uuid, uploadID, key string) (output *obs.ListPartsOutput, err error) { + output = nil + partNumberMarker := 0 + for { + temp, err := ObsCli.ListParts(&obs.ListPartsInput{ + Bucket: setting.Bucket, + Key: key, + UploadId: uploadID, + MaxParts: MAX_LIST_PARTS, + PartNumberMarker: partNumberMarker, + }) + if err != nil { + log.Error("ListParts failed:", err.Error()) + return output, err + } + + partNumberMarker = temp.NextPartNumberMarker + log.Info("uuid:%s, MaxParts:%d, PartNumberMarker:%d, NextPartNumberMarker:%d, len:%d", uuid, temp.MaxParts, temp.PartNumberMarker, temp.NextPartNumberMarker, len(temp.Parts)) + + for _, partInfo := range temp.Parts { + output.Parts = append(output.Parts, obs.Part{ + PartNumber: partInfo.PartNumber, + ETag: partInfo.ETag, + }) + } + + if len(temp.Parts) < temp.MaxParts { + break + } else { + continue + } + + break + } + + return output, nil +} + func GetObsPartInfos(uuid, uploadID, fileName string) (string, error) { key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") - output, err := ObsCli.ListParts(&obs.ListPartsInput{ - Bucket: setting.Bucket, - Key: key, - UploadId: uploadID, - }) + allParts, err := listAllParts(uuid, uploadID, key) if err != nil { - log.Error("ListParts failed:", err.Error()) + log.Error("listAllParts failed: %v", err) return "", err } var chunks string - for _, partInfo := range output.Parts { + for _, partInfo := range allParts.Parts { chunks += strconv.Itoa(partInfo.PartNumber) + "-" + partInfo.ETag + "," } @@ -100,39 +134,14 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadID - partNumberMarker := 0 - for { - output, err := ObsCli.ListParts(&obs.ListPartsInput{ - Bucket: setting.Bucket, - Key: input.Key, - UploadId: uploadID, - MaxParts: MAX_LIST_PARTS, - PartNumberMarker: partNumberMarker, - }) - if err != nil { - log.Error("ListParts failed:", err.Error()) - return err - } - - partNumberMarker = output.NextPartNumberMarker - log.Info("uuid:%s, MaxParts:%d, PartNumberMarker:%d, NextPartNumberMarker:%d, len:%d", uuid, output.MaxParts, output.PartNumberMarker, output.NextPartNumberMarker, len(output.Parts)) - - for _, partInfo := range output.Parts { - input.Parts = append(input.Parts, obs.Part{ - PartNumber: partInfo.PartNumber, - ETag: partInfo.ETag, - }) - } - - if len(output.Parts) < output.MaxParts { - break - } else { - continue - } - - break + allParts, err := listAllParts(uuid, uploadID, input.Key) + if err != nil { + log.Error("listAllParts failed: %v", err) + return err } + input.Parts = allParts.Parts + output, err := ObsCli.CompleteMultipartUpload(input) if err != nil { log.Error("CompleteMultipartUpload failed:", err.Error()) From f1108953bfa2f8e2ee8d077f4bc58095ef462c4e Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:08:34 +0800 Subject: [PATCH 04/29] =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/routers/search.go b/routers/search.go index 1cf78666e..457916d5a 100644 --- a/routers/search.go +++ b/routers/search.go @@ -183,7 +183,7 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) topicsQuery := elastic.NewMatchQuery("topics", Key) boolQ.Should(topicsQuery) - res, err := client.Search("repository-es-index").Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("topics")).Do(ctx.Req.Context()) + res, err := client.Search("repository-es-index").Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("topics")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -200,16 +200,15 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) } } -func getSort(SortBy string, ascending bool) elastic.Sorter { - var sort elastic.Sorter - sort = elastic.NewScoreSort() - if SortBy != "" { - if SortBy == "default" { - return sort - } - return elastic.NewFieldSort(SortBy).Order(ascending) +func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter { + sort := make([]elastic.Sorter, 0) + if SortBy == "default" { + sort = append(sort, elastic.NewScoreSort()) + sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) + return sort + } else { + return append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) } - return sort } func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) { @@ -308,7 +307,7 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, descriptionQuery, topicsQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -330,7 +329,7 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa } else { log.Info("query all content.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -691,7 +690,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "created_unix.keyword", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -849,7 +848,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -864,7 +863,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1057,7 +1056,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 067d4ad7ef1d767fd53ad2f7f6030f6bb75b6ed1 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:17:59 +0800 Subject: [PATCH 05/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/search.go b/routers/search.go index 457916d5a..ab9e902da 100644 --- a/routers/search.go +++ b/routers/search.go @@ -848,7 +848,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "download_times", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 2c4907301cfeeb43faf0ad30b4e2a19036f55404 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:32:29 +0800 Subject: [PATCH 06/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/routers/search.go b/routers/search.go index ab9e902da..64832d794 100644 --- a/routers/search.go +++ b/routers/search.go @@ -204,7 +204,9 @@ func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending sort := make([]elastic.Sorter, 0) if SortBy == "default" { sort = append(sort, elastic.NewScoreSort()) - sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) + if secondSortBy != "" { + sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) + } return sort } else { return append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) @@ -690,7 +692,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "created_unix.keyword", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -848,7 +850,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "download_times", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -863,7 +865,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1056,7 +1058,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 86c540783b893000d6836ecd80e0ccded35418f2 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:50:35 +0800 Subject: [PATCH 07/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/search.go b/routers/search.go index 64832d794..a83b53261 100644 --- a/routers/search.go +++ b/routers/search.go @@ -205,6 +205,7 @@ func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending if SortBy == "default" { sort = append(sort, elastic.NewScoreSort()) if secondSortBy != "" { + log.Info("SortBy=" + SortBy + " secondSortBy=" + secondSortBy) sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) } return sort From 30e32a4a620fed50d4a15431921124e23f3936bd Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:55:58 +0800 Subject: [PATCH 08/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/search.go b/routers/search.go index a83b53261..58e39b476 100644 --- a/routers/search.go +++ b/routers/search.go @@ -208,10 +208,11 @@ func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending log.Info("SortBy=" + SortBy + " secondSortBy=" + secondSortBy) sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) } - return sort } else { - return append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) + sort = append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) } + log.Info("sort size=" + fmt.Sprint(len(sort))) + return sort } func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) { From c2010ce0b4bd2c0dc3f370c21029fe02880867f8 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 15:59:54 +0800 Subject: [PATCH 09/29] fix bug --- modules/storage/obs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 44de3a25e..2b52ab603 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -60,7 +60,7 @@ func ObsHasObject(path string) (bool, error) { } func listAllParts(uuid, uploadID, key string) (output *obs.ListPartsOutput, err error) { - output = nil + output = &obs.ListPartsOutput{} partNumberMarker := 0 for { temp, err := ObsCli.ListParts(&obs.ListPartsInput{ From 3a60fbdf99256ced9142d9d33f833d1ac5456330 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 16:09:31 +0800 Subject: [PATCH 10/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/routers/search.go b/routers/search.go index 58e39b476..f2a2f30d6 100644 --- a/routers/search.go +++ b/routers/search.go @@ -200,6 +200,16 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) } } +func getSortSingle(SortBy string, ascending bool) elastic.Sorter { + var sort elastic.Sorter + if SortBy == "default" { + sort = elastic.NewScoreSort() + } else { + sort = elastic.NewFieldSort(SortBy).Order(ascending) + } + return sort +} + func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter { sort := make([]elastic.Sorter, 0) if SortBy == "default" { @@ -694,7 +704,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -852,7 +862,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -867,7 +877,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1060,7 +1070,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 079e2aed7a99673c5f4359b4b6a7330a5df6a730 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 16:16:40 +0800 Subject: [PATCH 11/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/routers/search.go b/routers/search.go index f2a2f30d6..5dd8a9b7e 100644 --- a/routers/search.go +++ b/routers/search.go @@ -202,17 +202,21 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) func getSortSingle(SortBy string, ascending bool) elastic.Sorter { var sort elastic.Sorter - if SortBy == "default" { - sort = elastic.NewScoreSort() + if SortBy != "" { + if SortBy == "default" { + sort = elastic.NewScoreSort() + } else { + sort = elastic.NewFieldSort(SortBy).Order(ascending) + } } else { - sort = elastic.NewFieldSort(SortBy).Order(ascending) + sort = elastic.NewScoreSort() } return sort } func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter { sort := make([]elastic.Sorter, 0) - if SortBy == "default" { + if SortBy == "default" || SortBy == "" { sort = append(sort, elastic.NewScoreSort()) if secondSortBy != "" { log.Info("SortBy=" + SortBy + " secondSortBy=" + secondSortBy) From 82fd6b72533674565f2b3eb8a5fcab8006db8b91 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 16:19:40 +0800 Subject: [PATCH 12/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/search.go b/routers/search.go index 5dd8a9b7e..a2c465757 100644 --- a/routers/search.go +++ b/routers/search.go @@ -708,7 +708,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From c20142f8282924abcfe00804e614da8fa583b693 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 17:00:09 +0800 Subject: [PATCH 13/29] test --- web_src/js/components/MinioUploader.vue | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/web_src/js/components/MinioUploader.vue b/web_src/js/components/MinioUploader.vue index 8c33608e7..dd2548f40 100755 --- a/web_src/js/components/MinioUploader.vue +++ b/web_src/js/components/MinioUploader.vue @@ -27,6 +27,7 @@ import createDropzone from '../features/dropzone.js'; const {_AppSubUrl, _StaticUrlPrefix, csrf} = window.config; const chunkSize = 1024 * 1024 * 64; +const md5ChunkSize = 1024 * 1024 * 1; export default { props:{ @@ -190,7 +191,6 @@ export default { let currentChunk = 0; const time = new Date().getTime(); - // console.log('计算MD5...') this.status = this.dropzoneParams.data('md5-computing'); file.totalChunkCounts = chunks; loadNext(); @@ -207,13 +207,12 @@ export default { spark.append(e.target.result); // Append array buffer currentChunk++; if (currentChunk < chunks) { - // console.log(`第${currentChunk}分片解析完成, 开始第${currentChunk +1}/${chunks}分片解析`); this.status = `${this.dropzoneParams.data('loading-file')} ${( (currentChunk / chunks) * 100 ).toFixed(2)}% (${currentChunk}/${chunks})`; this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2)); - loadNext(); + loadMd5Next(); return; } @@ -235,6 +234,13 @@ export default { start + chunkSize >= file.size ? file.size : start + chunkSize; fileReader.readAsArrayBuffer(blobSlice.call(file, start, end)); } + + function loadMd5Next() { + const start = currentChunk * chunkSize; + const end = + start + md5ChunkSize >= file.size ? file.size : start + md5ChunkSize; + fileReader.readAsArrayBuffer(blobSlice.call(file, start, end)); + } }, async computeMD5Success(md5edFile) { From b4442bdeaaed235b0ec4120fd1d4898a41875f7e Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 17:03:01 +0800 Subject: [PATCH 14/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/routers/search.go b/routers/search.go index a2c465757..8240b59d4 100644 --- a/routers/search.go +++ b/routers/search.go @@ -200,20 +200,6 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) } } -func getSortSingle(SortBy string, ascending bool) elastic.Sorter { - var sort elastic.Sorter - if SortBy != "" { - if SortBy == "default" { - sort = elastic.NewScoreSort() - } else { - sort = elastic.NewFieldSort(SortBy).Order(ascending) - } - } else { - sort = elastic.NewScoreSort() - } - return sort -} - func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter { sort := make([]elastic.Sorter, 0) if SortBy == "default" || SortBy == "" { @@ -866,7 +852,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -881,7 +867,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1074,7 +1060,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 6f2d744e7ccbd70ff010e5b9e8ff8d072deb1fad Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 17:27:31 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- public/home/search.js | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/public/home/search.js b/public/home/search.js index e23d27549..6de502721 100644 --- a/public/home/search.js +++ b/public/home/search.js @@ -101,16 +101,20 @@ function initPageInfo(){ function searchItem(type,sortType){ console.log("enter item 2."); - currentSearchKeyword = document.getElementById("keyword_input").value; - if(!isEmpty(currentSearchKeyword)){ - initPageInfo(); - currentSearchTableName = itemType[type]; - currentSearchSortBy = sortBy[sortType]; - currentSearchAscending = sortAscending[sortType]; - OnlySearchLabel =false; - page(currentPage); + if(OnlySearchLabel){ + doSearchLabel(currentSearchTableName,currentSearchKeyword,sortBy[sortType],sortAscending[sortType]) }else{ - emptySearch(); + currentSearchKeyword = document.getElementById("keyword_input").value; + if(!isEmpty(currentSearchKeyword)){ + initPageInfo(); + currentSearchTableName = itemType[type]; + currentSearchSortBy = sortBy[sortType]; + currentSearchAscending = sortAscending[sortType]; + OnlySearchLabel =false; + page(currentPage); + }else{ + emptySearch(); + } } } From 0f370c459044709fe2e327fa30565b7e3b97d60f Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 17:42:05 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/attachment.go | 5 +++-- services/socketwrap/clientManager.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 3c66a3537..da9f17dad 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -909,8 +909,9 @@ func CompleteMultipart(ctx *context.Context) { } repository, _ := models.GetRepositoryByID(dataset.RepoID) - notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(attachment.Type), attachment.Name, models.ActionUploadAttachment) - + if !repository.IsPrivate && !attachment.IsPrivate { + notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(attachment.Type), attachment.Name, models.ActionUploadAttachment) + } if attachment.DatasetID != 0 { if isCanDecompress(attachment.Name) { if typeCloudBrain == models.TypeCloudBrainOne { diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go index 61f356a66..6ffa96933 100755 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 25, 26, 27, 28, 29, 30, 31} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} type ClientsManager struct { Clients *orderedmap.OrderedMap From 5f03007bce04cd33f6b98f8e0acd021d29fe6670 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 17:49:42 +0800 Subject: [PATCH 17/29] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=8C=87=E6=95=B0=E4=BD=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 16 ++++++++-------- web_src/js/components/UserAnalysis.vue | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 8bc9dc458..995465b09 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -40,8 +40,8 @@ func queryUserDataPage(ctx *context.Context, tableName string, queryObj interfac dataHeader := map[string]string{ "A1": ctx.Tr("user.static.id"), "B1": ctx.Tr("user.static.name"), - "C1": ctx.Tr("user.static.codemergecount"), - "D1": ctx.Tr("user.static.UserIndex"), + "C1": ctx.Tr("user.static.UserIndex"), + "D1": ctx.Tr("user.static.codemergecount"), "E1": ctx.Tr("user.static.commitcount"), "F1": ctx.Tr("user.static.issuecount"), "G1": ctx.Tr("user.static.commentcount"), @@ -77,8 +77,8 @@ func queryUserDataPage(ctx *context.Context, tableName string, queryObj interfac rows := fmt.Sprint(row) xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) - xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) - xlsx.SetCellValue(sheetName, "D"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) + xlsx.SetCellValue(sheetName, "C"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CodeMergeCount) xlsx.SetCellValue(sheetName, "E"+rows, userRecord.CommitCount) xlsx.SetCellValue(sheetName, "F"+rows, userRecord.IssueCount) xlsx.SetCellValue(sheetName, "G"+rows, userRecord.CommentCount) @@ -239,8 +239,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { dataHeader := map[string]string{ "A1": ctx.Tr("user.static.id"), "B1": ctx.Tr("user.static.name"), - "C1": ctx.Tr("user.static.codemergecount"), - "D1": ctx.Tr("user.static.UserIndex"), + "C1": ctx.Tr("user.static.UserIndex"), + "D1": ctx.Tr("user.static.codemergecount"), "E1": ctx.Tr("user.static.commitcount"), "F1": ctx.Tr("user.static.issuecount"), "G1": ctx.Tr("user.static.commentcount"), @@ -270,8 +270,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) - xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) - xlsx.SetCellValue(sheetName, "D"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) + xlsx.SetCellValue(sheetName, "C"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CodeMergeCount) xlsx.SetCellValue(sheetName, "E"+rows, userRecord.CommitCount) xlsx.SetCellValue(sheetName, "F"+rows, userRecord.IssueCount) xlsx.SetCellValue(sheetName, "G"+rows, userRecord.CommentCount) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index 596ecd602..117984f95 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -63,11 +63,6 @@ - - + + From 274430598c2b1fd97df4aeeb87d6913febfe9909 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 08:51:42 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/search.go b/routers/search.go index 8240b59d4..fe1643c80 100644 --- a/routers/search.go +++ b/routers/search.go @@ -311,7 +311,7 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, descriptionQuery, topicsQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "num_stars", false)...).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 9f11ea564f0ea3c05b5b82273ff849ed1df0e7ee Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 10:53:25 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- public/home/search.js | 23 ++++++++++++++--------- routers/repo/attachment.go | 4 +--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/public/home/search.js b/public/home/search.js index 6de502721..ca23e55be 100644 --- a/public/home/search.js +++ b/public/home/search.js @@ -810,17 +810,21 @@ var repoAndOrgEN={ function page(current){ currentPage=current; + startIndex = currentPage -1; + if(startIndex < 1){ + startIndex = 1; + } + endIndex = currentPage + 1; + if(endIndex >= totalPage){ + endIndex = totalPage; + } doSearch(currentSearchTableName,currentSearchKeyword,current,pageSize,false,currentSearchSortBy,OnlySearchLabel); - } function nextPage(){ currentPage = currentPage+1; console.log("currentPage=" + currentPage); - if(currentPage >= endIndex){ - startIndex=startIndex+1; - endIndex = endIndex +1; - } + page(currentPage); } @@ -828,10 +832,6 @@ function page(current){ console.log("currentPage=" + currentPage); if(currentPage > 1){ currentPage = currentPage-1; - if(currentPage <= startIndex && startIndex > 1){ - startIndex = startIndex -1; - endIndex = endIndex - 1; - } console.log("currentPage=" + (currentPage)); page(currentPage); } @@ -912,6 +912,11 @@ function getYPosition(e){ } } + if (endIndex < totalPage-1){ + html += "..."; + html += "" + totalPage + ""; + } + if(currentPage >=totalPage){ html += ""; html += "" + getLabel(isZh,"search_last_page") + ""; diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index da9f17dad..8511a1f29 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -909,9 +909,7 @@ func CompleteMultipart(ctx *context.Context) { } repository, _ := models.GetRepositoryByID(dataset.RepoID) - if !repository.IsPrivate && !attachment.IsPrivate { - notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(attachment.Type), attachment.Name, models.ActionUploadAttachment) - } + notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(repository.IsPrivate, attachment.IsPrivate), attachment.Name, models.ActionUploadAttachment) if attachment.DatasetID != 0 { if isCanDecompress(attachment.Name) { if typeCloudBrain == models.TypeCloudBrainOne { From e1365f74dde87f45a55c062decca93c790993b43 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 19 Apr 2022 11:15:00 +0800 Subject: [PATCH 20/29] opt-for-compute-md5 --- web_src/js/components/MinioUploader.vue | 2 +- web_src/js/components/ObsUploader.vue | 484 -------------------------------- 2 files changed, 1 insertion(+), 485 deletions(-) delete mode 100755 web_src/js/components/ObsUploader.vue diff --git a/web_src/js/components/MinioUploader.vue b/web_src/js/components/MinioUploader.vue index dd2548f40..95f5e7e59 100755 --- a/web_src/js/components/MinioUploader.vue +++ b/web_src/js/components/MinioUploader.vue @@ -193,7 +193,7 @@ export default { const time = new Date().getTime(); this.status = this.dropzoneParams.data('md5-computing'); file.totalChunkCounts = chunks; - loadNext(); + loadMd5Next(); fileReader.onload = (e) => { fileLoaded.call(this, e); diff --git a/web_src/js/components/ObsUploader.vue b/web_src/js/components/ObsUploader.vue deleted file mode 100755 index 7ceb91a27..000000000 --- a/web_src/js/components/ObsUploader.vue +++ /dev/null @@ -1,484 +0,0 @@ - - - - - \ No newline at end of file From 4be35d2e4f9f9c7ed5009b3db432cdab4619f28f Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 11:16:36 +0800 Subject: [PATCH 21/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_en-US.ini | 2 +- public/home/home.js | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index fb38fe067..b35bf74b4 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2844,7 +2844,7 @@ mirror_sync_create = synced new reference %[2]s to %[2]s at %[3]s from mirror approve_pull_request = `approved %s#%[2]s` reject_pull_request = `suggested changes for %s#%[2]s` -upload_dataset=`upload dataset %s` +upload_dataset=`upload dataset %s` task_gpudebugjob=`created CPU/GPU type debugging task%s` task_npudebugjob=`created NPU type debugging task %s` task_nputrainjob=`created NPU training task%s` diff --git a/public/home/home.js b/public/home/home.js index 478c70f21..d8e423def 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -99,6 +99,11 @@ socket.onmessage = function (e) { console.log("receive action type=" + record.OpType + " name=" + actionName + " but user is null."); continue; } + if(record.OpType == "24"){ + if(record.Content.indexOf("true") != -1){ + continue; + } + } var recordPrefix = getMsg(record); if(record.OpType == "6" || record.OpType == "10" || record.OpType == "12" || record.OpType == "13"){ html += recordPrefix + actionName; @@ -162,7 +167,7 @@ socket.onmessage = function (e) { function getTaskLink(record){ var re = getRepoLink(record); if(record.OpType == 24){ - re = re + "/datasets?type=" + record.Content; + re = re + "/datasets"; }else if(record.OpType == 25){ re = re + "/cloudbrain/" + record.Content; }else if(record.OpType == 26){ From f1cb3b3afdd9630a1cdd2f7fc4e44cb632529057 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 11:17:00 +0800 Subject: [PATCH 22/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/user/dashboard/feeds.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index 614c328c6..171287acb 100755 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -71,7 +71,7 @@ {{ $index := index .GetIssueInfos 0}} {{$.i18n.Tr "action.comment_pull" .GetRepoLink $index .ShortRepoPath | Str2html}} {{else if eq .GetOpType 24}} - {{$.i18n.Tr "action.upload_dataset" .GetRepoLink .Content .RefName | Str2html}} + {{$.i18n.Tr "action.upload_dataset" .GetRepoLink .RefName | Str2html}} {{else if eq .GetOpType 25}} {{$.i18n.Tr "action.task_gpudebugjob" .GetRepoLink .Content .RefName | Str2html}} {{else if eq .GetOpType 26}} From 3fb2b297b5935596e6a4ac351f97120bd141844a Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 11:19:28 +0800 Subject: [PATCH 23/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_zh-CN.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index ea40f9aa8..480eb8fc2 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -2852,7 +2852,7 @@ mirror_sync_create=从镜像同步了新的引用 %[2]s mirror_sync_delete=从镜像同步并从 %[3]s 删除了引用 %[2]s approve_pull_request=`同意了 %s#%[2]s` reject_pull_request=`建议变更 %s#%[2]s` -upload_dataset=`上传了数据集文件 %s` +upload_dataset=`上传了数据集文件 %s` task_gpudebugjob=`创建了CPU/GPU类型调试任务 %s` task_npudebugjob=`创建了NPU类型调试任务 %s` task_nputrainjob=`创建了NPU类型训练任务 %s` From 92d67c4e6b43c5e37999b654592306118d4e49e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 11:34:35 +0800 Subject: [PATCH 24/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- public/home/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/home/search.js b/public/home/search.js index ca23e55be..5e647070a 100644 --- a/public/home/search.js +++ b/public/home/search.js @@ -866,7 +866,7 @@ function getYPosition(e){ showTip(getLabel(isZh,"search_input_large_0"),"warning",left+5,top); } else if(goNum<=totalPage){ - page(goNum); + page(parseInt(goNum,10)); } else{ showTip(getLabel(isZh,"search_input_maxed"),"warning",left+5,top); From 8e724b56bfdeeb74ee96f047f9a0601753fe4aa2 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 11:37:23 +0800 Subject: [PATCH 25/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- public/home/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/home/search.js b/public/home/search.js index 5e647070a..c55d1807c 100644 --- a/public/home/search.js +++ b/public/home/search.js @@ -814,7 +814,7 @@ function page(current){ if(startIndex < 1){ startIndex = 1; } - endIndex = currentPage + 1; + endIndex = currentPage + 2; if(endIndex >= totalPage){ endIndex = totalPage; } From 33b641467f6255bb05f3bdb527e56f3eba263595 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 11:52:01 +0800 Subject: [PATCH 26/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ca8c5071f..84972ca9d 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -831,7 +831,12 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64 result += float64(dateRecord.StarRepoCount) * getParaWeightValue("StarRepoCount", ParaWeight, 0.1) result += float64(dateRecord.LoginCount) * getParaWeightValue("LoginCount", ParaWeight, 0.1) result += float64(dateRecord.WatchedCount) * getParaWeightValue("WatchedCount", ParaWeight, 0.3) - result += float64(dateRecord.CommitCodeSize) * getParaWeightValue("CommitCodeSize", ParaWeight, 0.1) + codeLine := float64(dateRecord.CommitCodeSize) / 1000 + limitCodeLine := getParaWeightValue("LimitCommitCodeSize", ParaWeight, 100) + if codeLine > limitCodeLine { + codeLine = limitCodeLine + } + result += codeLine * getParaWeightValue("CommitCodeSize", ParaWeight, 0.1) result += float64(dateRecord.SolveIssueCount) * getParaWeightValue("SolveIssueCount", ParaWeight, 0.2) result += float64(dateRecord.EncyclopediasCount) * getParaWeightValue("EncyclopediasCount", ParaWeight, 0.1) result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) From 1749d7695e440affa31c969b27bc3de81012c4ca Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 14:30:40 +0800 Subject: [PATCH 27/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/attachment.go | 9 +++++++++ models/user_business_analysis.go | 2 +- routers/repo/attachment.go | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 7c95a73dd..40fcd7f08 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -10,6 +10,7 @@ import ( "io" "path" "strings" + "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/obs" @@ -104,6 +105,14 @@ func (a *Attachment) IncreaseDownloadCount() error { return nil } +func (a *Attachment) UpdateDatasetUpdateUnix() error { + // Update download count. + if _, err := x.Exec("UPDATE `dataset` SET updated_unix="+fmt.Sprint(time.Now().Unix())+" WHERE id=?", a.DatasetID); err != nil { + return fmt.Errorf("UpdateDatasetUpdateUnix: %v", err) + } + return nil +} + // APIFormat converts models.Attachment to api.Attachment func (a *Attachment) APIFormat() *api.Attachment { return &api.Attachment{ diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 84972ca9d..34d84555c 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -833,7 +833,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64 result += float64(dateRecord.WatchedCount) * getParaWeightValue("WatchedCount", ParaWeight, 0.3) codeLine := float64(dateRecord.CommitCodeSize) / 1000 limitCodeLine := getParaWeightValue("LimitCommitCodeSize", ParaWeight, 100) - if codeLine > limitCodeLine { + if codeLine >= limitCodeLine { codeLine = limitCodeLine } result += codeLine * getParaWeightValue("CommitCodeSize", ParaWeight, 0.1) diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 8511a1f29..fa9d21963 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -907,7 +907,7 @@ func CompleteMultipart(ctx *context.Context) { ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err)) return } - + attachment.UpdateDatasetUpdateUnix() repository, _ := models.GetRepositoryByID(dataset.RepoID) notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(repository.IsPrivate, attachment.IsPrivate), attachment.Name, models.ActionUploadAttachment) if attachment.DatasetID != 0 { From 1427ee25b75bbd522e5e0c519940746525ffd819 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 19 Apr 2022 16:23:23 +0800 Subject: [PATCH 28/29] del md5 show --- templates/repo/datasets/index.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index e8d9b65e3..ba4842f57 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -281,7 +281,7 @@ {{$.i18n.Tr "dataset.copy_url"}} - {{$.i18n.Tr "dataset.copy_md5"}} + {{if and ($.CanWrite) (eq .DecompressState 1) }} {{$.i18n.Tr "dataset.annotation"}} {{end}} From 747c4c9d44a1d3063d9db738cc8ec8d00b6eafc9 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 17:27:21 +0800 Subject: [PATCH 29/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/modelarts.go | 4 +++- templates/repo/cloudbrain/show.tmpl | 2 +- templates/repo/modelarts/notebook/show.tmpl | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 318726e8e..1147dc5a9 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -247,7 +247,9 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm func NotebookShow(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true debugListType := ctx.Query("debugListType") - + if debugListType == "" { + debugListType = "all" + } var ID = ctx.Params(":id") task, err := models.GetCloudbrainByIDWithDeleted(ID) if err != nil { diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 7e16c36ff..5fabaa937 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -187,7 +187,7 @@ td, th { {{.i18n.Tr "repo.cloudbrain"}}
/
- + {{$.i18n.Tr "repo.modelarts.notebook"}}
/
diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index 0d7a01efb..26839345e 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -193,7 +193,7 @@ td, th { {{.i18n.Tr "repo.cloudbrain"}}
/
- + {{$.i18n.Tr "repo.modelarts.notebook"}}
/