From 692385810ae03b9ad7f613c9645b1dd424ac94cf Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 12 Apr 2022 19:46:36 +0800 Subject: [PATCH 001/125] opt --- routers/repo/cloudbrain.go | 6 ++++-- routers/repo/modelarts.go | 14 ++++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 0d007a27d..fecd34faf 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -53,6 +53,7 @@ var ( ) const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types" +const CLONE_FILE_PREFIX = "file:///" var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0) @@ -909,7 +910,8 @@ func GetRate(ctx *context.Context) { } func downloadCode(repo *models.Repository, codePath, branchName string) error { - if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName}); err != nil { + //add "file:///" prefix to make the depth valid + if err := git.Clone(CLONE_FILE_PREFIX+repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil { log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) return err } @@ -969,7 +971,7 @@ func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepo return err } - if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + if err := git.Clone(CLONE_FILE_PREFIX+repoExt.RepoPath(), codePath, git.CloneRepoOptions{Depth: 1}); err != nil { log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 8b785a395..29387133d 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1023,10 +1023,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ - Branch: branch_name, - }); err != nil { - log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) + if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { + log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Create task failed, server timed out", tplModelArtsTrainJobNew, &form) return @@ -1241,9 +1239,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ - Branch: branch_name, - }); err != nil { + if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { log.Error("Failed git clone repo to local(！: %s (%v)", repo.FullName(), err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed git clone repo to local！", tplModelArtsTrainJobVersionNew, &form) @@ -1867,9 +1863,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) - if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ - Branch: branch_name, - }); err != nil { + if err := downloadCode(repo, codeLocalPath, branch_name); err != nil { log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Create task failed, server timed out", tplModelArtsInferenceJobNew, &form) From 212b3c217c1701c55d14aed20fe0a25b60628233 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 18 Apr 2022 09:24:21 +0800 Subject: [PATCH 002/125] =?UTF-8?q?fix-1913=20=E6=8F=90=E4=BA=A4=E9=95=9C?= =?UTF-8?q?=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 2 ++ models/cloudbrain_image.go | 4 ++-- modules/auth/cloudbrain.go | 10 +++++++++ modules/cloudbrain/resty.go | 41 +++++++++++++++++++++++++++++++++++- routers/admin/cloudbrains.go | 7 +++++++ routers/repo/cloudbrain.go | 49 ++++++++++++++++++++++++++++++++++++++++++++ routers/routes/routes.go | 2 ++ 7 files changed, 112 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 0465faf9a..44b177471 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -580,6 +580,8 @@ type CommitImageParams struct { Topics []string CloudBrainType int UID int64 + Place string + Type int } type CommitImageResult struct { diff --git a/models/cloudbrain_image.go b/models/cloudbrain_image.go index c88db0f67..eb21e0d87 100644 --- a/models/cloudbrain_image.go +++ b/models/cloudbrain_image.go @@ -567,12 +567,12 @@ func isImageStaring(e Engine, userID, imageID int64) bool { } func RecommendImage(imageId int64, recommond bool) error { - image := Image{Type: getRecommondType(recommond)} + image := Image{Type: GetRecommondType(recommond)} _, err := x.ID(imageId).Cols("type").Update(image) return err } -func getRecommondType(recommond bool) int { +func GetRecommondType(recommond bool) int { if recommond { return RECOMMOND_TYPE diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go index 85f3a2127..e5be38084 100755 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -33,6 +33,16 @@ type CommitImageCloudBrainForm struct { Topics string `form:"topics"` } +type CommitAdminImageCloudBrainForm struct { + Description string `form:"description" binding:"Required"` + Type int `form:"type" binding:"Required"` + Tag string `form:"tag" binding:"Required;MaxSize(100)" ` + IsPrivate bool `form:"isPrivate" binding:"Required"` + Topics string `form:"topics"` + Place string `form:"place" binding:"Required"` + IsRecommend bool `form:"isRecommend" binding:"Required"` +} + type EditImageCloudBrainForm struct { ID int64 `form:"id" binding:"Required"` Description string `form:"description" binding:"Required"` diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index f1f213bea..05f110856 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -312,12 +312,51 @@ sendjob: return nil }) if err == nil { - go updateImageStatus(image, isSetCreatedUnix, createTime) } return err } +func CommitAdminImage(jobID string, params models.CommitImageParams) error { + + exist, err := models.IsImageExist(params.ImageTag) + + if err != nil { + return fmt.Errorf("resty CommitImage: %v", err) + } + if exist { + return models.ErrorImageTagExist{ + Tag: params.ImageTag, + } + } + + image := models.Image{ + CloudbrainType: params.CloudBrainType, + UID: params.UID, + IsPrivate: params.IsPrivate, + Tag: params.ImageTag, + Description: params.ImageDescription, + Place: params.Place, + Status: models.IMAGE_STATUS_SUCCESS, + Type: params.Type, + } + + err = models.WithTx(func(ctx models.DBContext) error { + + if err := models.CreateLocalImage(&image); err != nil { + log.Error("Failed to insert image record.", err) + return fmt.Errorf("resty CommitImage: %v", err) + } + + if err := models.SaveImageTopics(image.ID, params.Topics...); err != nil { + log.Error("Failed to insert image record.", err) + return fmt.Errorf("resty CommitImage: %v", err) + } + return nil + }) + return err +} + func updateImageStatus(image models.Image, isSetCreatedUnix bool, createTime time.Time) { attemps := 5 commitSuccess := false diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 91d866093..5ad1246ff 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -21,6 +21,7 @@ import ( const ( tplCloudBrains base.TplName = "admin/cloudbrain/list" tplImages base.TplName = "admin/cloudbrain/images" + tplCommitImages base.TplName = "admin/cloudbrain/imagecommit" EXCEL_DATE_FORMAT = "20060102150405" CREATE_TIME_FORMAT = "2006/01/02 15:04:05" ) @@ -114,6 +115,12 @@ func Images(ctx *context.Context) { } +func CloudBrainCommitImageShow(ctx *context.Context) { + ctx.Data["PageIsAdminImages"] = true + ctx.HTML(200, tplImages) + +} + func DownloadCloudBrains(ctx *context.Context) { page := 1 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5daf96e78..c2eaf1468 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -702,6 +702,55 @@ func CloudBrainCommitImageCheck(ctx *context.Context, form auth.CommitImageCloud } +func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImageCloudBrainForm) { + + if !NamePattern.MatchString(form.Tag) { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) + return + } + + if utf8.RuneCountInString(form.Description) > 255 { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255))) + return + } + + validTopics, errMessage := checkTopics(form.Topics) + if errMessage != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage))) + return + } + + err := cloudbrain.CommitAdminImage(ctx.Cloudbrain.JobID, models.CommitImageParams{ + CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{ + Ip: ctx.Cloudbrain.ContainerIp, + TaskContainerId: ctx.Cloudbrain.ContainerID, + ImageDescription: form.Description, + ImageTag: form.Tag, + }, + IsPrivate: form.IsPrivate, + CloudBrainType: form.Type, + Topics: validTopics, + UID: ctx.User.ID, + Type: models.GetRecommondType(form.IsRecommend), + Place: form.Place, + }) + if err != nil { + log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"]) + if models.IsErrImageTagExist(err) { + ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist"))) + + } else if models.IsErrorImageCommitting(err) { + ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing"))) + } else { + ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail"))) + } + + return + } + + ctx.JSON(200, models.BaseOKMessage) +} + func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) { if !NamePattern.MatchString(form.Tag) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 9df429e8b..cf1a18b31 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -534,6 +534,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/images", func() { m.Get("", admin.Images) m.Get("/data", repo.GetAllImages) + m.Get("/commit_image", admin.CloudBrainCommitImageShow) + m.Post("/commit_image", bindIgnErr(auth.CommitAdminImageCloudBrainForm{}), repo.CloudBrainAdminCommitImage) }) m.Put("/image/:id/action/:action", image.Action) From f008690b00c05b81a60d2ef5fd7de5cb64206ce4 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 18 Apr 2022 10:46:12 +0800 Subject: [PATCH 003/125] fix issu --- templates/repo/modelarts/trainjob/show.tmpl | 34 ++++++++++++++++++++++++++++- web_src/less/openi.less | 17 +++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 0b420c484..dea9b6aa0 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -420,7 +420,13 @@ td, th {
-
+
+ + + + + + @@ -879,4 +885,30 @@ td, th { }); } } + function scrollAnimation(dom, currentY, targetY, currentX) { + let needScrollTop = targetY - currentY; + let _currentY = currentY; + let context = this; + setTimeout(() => { + // 一次调用滑动帧数，每次调用会不一样 + //取总距离的十分之一 + const dist = Math.ceil(needScrollTop / 10); + _currentY += dist; + //移动一个十分之一 + + dom.scrollTo(currentX || 0, _currentY); + // 如果移动幅度小于十个像素，直接移动，否则递归调用，实现动画效果 + if (needScrollTop > 10 || needScrollTop < -10) { + context.scrollAnimation(dom, _currentY, targetY) + } else { + dom.scrollTo(_currentY, targetY) + } + }, 1) + } + function scrollTop(){ + let logContentDom = document.querySelector('.log'), context = this; + if(!logContentDom) + return + scrollAnimation(logContentDom, logContentDom.scrollTop, 0); + } diff --git a/web_src/less/openi.less b/web_src/less/openi.less index c195bac38..31a8932c0 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -250,6 +250,23 @@ footer .column{margin-bottom:0!important; padding-bottom:0!important;} .CREATING, .STOPPING, .DELETING, .STARTING, i.WAITING ,.INIT,.KILLING{display:inline-block;background-image:url('/img/loading.gif');background-repeat:no-repeat;width:16px;height:16px;background-size:16px 16px;margin-right:5px;} i.COMPLETED,i.SUCCEEDED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -441px -52px;} +.icon-to-bottom{ + background:url("/img/icons.svg"); + background-position: -574px -208px; + width: 30px; + height: 30px; + display: inline-block; +} +.icon-to-top{ + background:url("/img/icons.svg"); + background-position: -540px -208px; + width: 30px; + height: 30px; + display: inline-block; +} + + + .text_over{ overflow: hidden; text-overflow: ellipsis; From 39cdb60b8f61779cc1d32b578cb4cace014ff9b5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 18 Apr 2022 10:48:03 +0800 Subject: [PATCH 004/125] fix issue --- templates/repo/debugjob/index.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/debugjob/index.tmpl b/templates/repo/debugjob/index.tmpl index b99a50f1f..dbdf6af09 100755 --- a/templates/repo/debugjob/index.tmpl +++ b/templates/repo/debugjob/index.tmpl @@ -380,7 +380,7 @@
{{if .CanDebug}} - {{$.i18n.Tr "repo.submit_image"}} + {{$.i18n.Tr "repo.submit_image"}} {{else}} {{$.i18n.Tr "repo.submit_image"}} {{end}} From 4910fbe08e1e25612f7435431380e18c1654c039 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 10:51:52 +0800 Subject: [PATCH 005/125] no need reqin for complete multipart --- routers/repo/attachment.go | 43 ------------------------------------------- routers/routes/routes.go | 3 +-- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 3c66a3537..b2ded263e 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -11,7 +11,6 @@ import ( "fmt" "mime/multipart" "net/http" - "path" "strconv" "strings" @@ -830,20 +829,6 @@ func GetMultipartUploadUrl(ctx *context.Context) { }) } -func GetObsKey(ctx *context.Context) { - uuid := gouuid.NewV4().String() - key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/") - - ctx.JSON(200, map[string]string{ - "uuid": uuid, - "key": key, - "access_key_id": setting.AccessKeyID, - "secret_access_key": setting.SecretAccessKey, - "server": setting.Endpoint, - "bucket": setting.Bucket, - }) -} - func CompleteMultipart(ctx *context.Context) { uuid := ctx.Query("uuid") uploadID := ctx.Query("uploadID") @@ -947,34 +932,6 @@ func CompleteMultipart(ctx *context.Context) { }) } -func UpdateMultipart(ctx *context.Context) { - uuid := ctx.Query("uuid") - partNumber := ctx.QueryInt("chunkNumber") - etag := ctx.Query("etag") - - fileChunk, err := models.GetFileChunkByUUID(uuid) - if err != nil { - if models.IsErrFileChunkNotExist(err) { - ctx.Error(404) - } else { - ctx.ServerError("GetFileChunkByUUID", err) - } - return - } - - fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber)+"-"+strings.Replace(etag, "\"", "", -1)) - - err = models.UpdateFileChunk(fileChunk) - if err != nil { - ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) - return - } - - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) -} - func HandleUnDecompressAttachment() { attachs, err := models.GetUnDecompressAttachments() if err != nil { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 9df429e8b..e6bda142b 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -608,12 +608,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Put("/obs_proxy_multipart", repo.PutOBSProxyUpload) m.Get("/obs_proxy_download", repo.GetOBSProxyDownload) m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) - m.Post("/complete_multipart", repo.CompleteMultipart) - m.Post("/update_chunk", repo.UpdateMultipart) }, reqSignIn) m.Group("/attachments", func() { m.Post("/decompress_done_notify", repo.UpdateAttachmentDecompressState) + m.Post("/complete_multipart", repo.CompleteMultipart) }) m.Group("/attachments", func() { From 093ed5bfdc38e0c2451cc8ea8fbfe6f3eab98d2b Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 18 Apr 2022 11:03:47 +0800 Subject: [PATCH 006/125] =?UTF-8?q?fix-1898=201801=201761=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 8 ++++++++ models/dataset.go | 16 ++++++++++++++-- routers/admin/dataset.go | 19 +++++++++++++++++++ routers/home.go | 1 + routers/repo/dataset.go | 3 +++ routers/routes/routes.go | 1 + routers/user/profile.go | 8 +++++--- 7 files changed, 51 insertions(+), 5 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 7c95a73dd..778b9095f 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -64,6 +64,7 @@ type AttachmentInfo struct { Repo *Repository `xorm:"extends"` RelAvatarLink string `xorm:"extends"` UserName string `xorm:"extends"` + Recommend bool `xorm:"-"` } type AttachmentsOptions struct { @@ -78,6 +79,7 @@ type AttachmentsOptions struct { JustNeedZipFile bool NeedRepoInfo bool Keyword string + RecommendOnly bool } func (a *Attachment) AfterUpdate() { @@ -570,6 +572,11 @@ func Attachments(opts *AttachmentsOptions) ([]*AttachmentInfo, int64, error) { builder.Eq{"attachment.is_private": opts.IsPrivate}, ) } + if opts.RecommendOnly { + cond = cond.And(builder.In("attachment.id", builder.Select("attachment.id"). + From("attachment"). + Join("INNER", "dataset", "attachment.dataset_id = dataset.id and dataset.recommend=true"))) + } if opts.JustNeedZipFile { var DecompressState []int32 @@ -618,6 +625,7 @@ func Attachments(opts *AttachmentsOptions) ([]*AttachmentInfo, int64, error) { if err != nil { return nil, 0, fmt.Errorf("GetDatasetByID failed error: %v", err) } + attachment.Recommend = dataset.Recommend repo, err := GetRepositoryByID(dataset.RepoID) if err == nil { attachment.Repo = repo diff --git a/models/dataset.go b/models/dataset.go index 95800100c..e841261c7 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -23,7 +23,8 @@ type Dataset struct { Category string Description string `xorm:"TEXT"` DownloadTimes int64 - NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` + NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` + Recommend bool `xorm:"INDEX NOT NULL DEFAULT false"` License string Task string ReleaseID int64 `xorm:"INDEX"` @@ -99,6 +100,7 @@ type SearchDatasetOptions struct { OwnerID int64 RepoID int64 IncludePublic bool + RecommendOnly bool Category string Task string License string @@ -132,6 +134,13 @@ func CreateDataset(dataset *Dataset) (err error) { } +func RecommendDataset(dataSetId int64, recommend bool) error { + + dataset := Dataset{Recommend: recommend} + _, err := x.ID(dataSetId).Cols("recommend").Update(dataset) + return err +} + func SearchDataset(opts *SearchDatasetOptions) (DatasetList, int64, error) { cond := SearchDatasetCondition(opts) return SearchDatasetByCondition(opts, cond) @@ -146,6 +155,9 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { if opts.RepoID > 0 { cond = cond.And(builder.Eq{"dataset.repo_id": opts.RepoID}) } + if opts.RecommendOnly { + cond = cond.And(builder.Eq{"dataset.recommend": opts.RecommendOnly}) + } if opts.IncludePublic { cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) @@ -198,7 +210,7 @@ func SearchDatasetByCondition(opts *SearchDatasetOptions, cond builder.Cond) (Da defer sess.Close() datasets := make(DatasetList, 0, opts.PageSize) - selectColumnsSql := "distinct dataset.id,dataset.title, dataset.status, dataset.category, dataset.description, dataset.download_times, dataset.license, dataset.task, dataset.release_id, dataset.user_id, dataset.repo_id, dataset.created_unix,dataset.updated_unix,dataset.num_stars" + selectColumnsSql := "distinct dataset.id,dataset.title, dataset.status, dataset.category, dataset.description, dataset.download_times, dataset.license, dataset.task, dataset.release_id, dataset.user_id, dataset.repo_id, dataset.created_unix,dataset.updated_unix,dataset.num_stars,dataset.recommend" count, err := sess.Distinct("dataset.id").Join("INNER", "repository", "repository.id = dataset.repo_id"). Join("INNER", "attachment", "attachment.dataset_id=dataset.id"). diff --git a/routers/admin/dataset.go b/routers/admin/dataset.go index a4378cf67..c233c388c 100644 --- a/routers/admin/dataset.go +++ b/routers/admin/dataset.go @@ -1,6 +1,8 @@ package admin import ( + "net/http" + "strconv" "strings" "code.gitea.io/gitea/models" @@ -88,6 +90,23 @@ func Datasets(ctx *context.Context) { ctx.HTML(200, tplDatasets) } +func DatasetAction(ctx *context.Context) { + var err error + datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) + switch ctx.Params(":action") { + + case "recommend": + err = models.RecommendDataset(datasetId, true) + case "unrecommend": + err = models.RecommendDataset(datasetId, false) + } + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action")))) + } else { + ctx.JSON(http.StatusOK, models.BaseOKMessage) + } +} + func DeleteDataset(ctx *context.Context) { dataset, err := models.GetDatasetByID(ctx.QueryInt64("id")) if err != nil { diff --git a/routers/home.go b/routers/home.go index 324bb1032..274e8af69 100755 --- a/routers/home.go +++ b/routers/home.go @@ -331,6 +331,7 @@ func ExploreDatasets(ctx *context.Context) { Task: task, License: license, OwnerID: ownerID, + RecommendOnly: ctx.QueryBool("recommend"), ListOptions: models.ListOptions{ Page: page, PageSize: 30, diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 1a3762be3..73036a2cc 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -358,6 +358,7 @@ func MyDatasets(ctx *context.Context) { NeedIsPrivate: false, JustNeedZipFile: true, NeedRepoInfo: true, + RecommendOnly: ctx.QueryBool("recommend"), }) if err != nil { ctx.ServerError("datasets", err) @@ -398,6 +399,7 @@ func PublicDataset(ctx *context.Context) { Type: cloudbrainType, JustNeedZipFile: true, NeedRepoInfo: true, + RecommendOnly: ctx.QueryBool("recommend"), }) if err != nil { ctx.ServerError("datasets", err) @@ -454,6 +456,7 @@ func MyFavoriteDataset(ctx *context.Context) { Type: cloudbrainType, JustNeedZipFile: true, NeedRepoInfo: true, + RecommendOnly: ctx.QueryBool("recommend"), }) if err != nil { ctx.ServerError("datasets", err) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index cf1a18b31..c281d832b 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -525,6 +525,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/datasets", func() { m.Get("", admin.Datasets) + m.Put(":id/action/:action", admin.DatasetAction) // m.Post("/delete", admin.DeleteDataset) }) m.Group("/cloudbrains", func() { diff --git a/routers/user/profile.go b/routers/user/profile.go index 41d8561d6..f82c03a75 100755 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -106,9 +106,9 @@ func Profile(ctx *context.Context) { for _, org := range orgs { _, repoCount, err := models.SearchRepository(&models.SearchRepoOptions{ - OwnerID: org.ID, - Private: ctx.IsSigned, - Actor: ctx.User, + OwnerID: org.ID, + Private: ctx.IsSigned, + Actor: ctx.User, }) if err != nil { ctx.ServerError("SearchRepository", err) @@ -175,6 +175,8 @@ func Profile(ctx *context.Context) { orderBy = models.SearchOrderByAlphabeticallyReverse case "alphabetically": orderBy = models.SearchOrderByAlphabetically + case "downloadtimes": + orderBy = models.SearchOrderByDownloadTimes case "moststars": orderBy = models.SearchOrderByStarsReverse case "feweststars": From adf0af490068eb1b697c22b6de9dff5e8dbc419c Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 11:31:06 +0800 Subject: [PATCH 007/125] fix 1939 --- modules/storage/obs.go | 85 ++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 08a354359..44de3a25e 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -59,21 +59,55 @@ func ObsHasObject(path string) (bool, error) { return hasObject, nil } +func listAllParts(uuid, uploadID, key string) (output *obs.ListPartsOutput, err error) { + output = nil + partNumberMarker := 0 + for { + temp, err := ObsCli.ListParts(&obs.ListPartsInput{ + Bucket: setting.Bucket, + Key: key, + UploadId: uploadID, + MaxParts: MAX_LIST_PARTS, + PartNumberMarker: partNumberMarker, + }) + if err != nil { + log.Error("ListParts failed:", err.Error()) + return output, err + } + + partNumberMarker = temp.NextPartNumberMarker + log.Info("uuid:%s, MaxParts:%d, PartNumberMarker:%d, NextPartNumberMarker:%d, len:%d", uuid, temp.MaxParts, temp.PartNumberMarker, temp.NextPartNumberMarker, len(temp.Parts)) + + for _, partInfo := range temp.Parts { + output.Parts = append(output.Parts, obs.Part{ + PartNumber: partInfo.PartNumber, + ETag: partInfo.ETag, + }) + } + + if len(temp.Parts) < temp.MaxParts { + break + } else { + continue + } + + break + } + + return output, nil +} + func GetObsPartInfos(uuid, uploadID, fileName string) (string, error) { key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") - output, err := ObsCli.ListParts(&obs.ListPartsInput{ - Bucket: setting.Bucket, - Key: key, - UploadId: uploadID, - }) + allParts, err := listAllParts(uuid, uploadID, key) if err != nil { - log.Error("ListParts failed:", err.Error()) + log.Error("listAllParts failed: %v", err) return "", err } var chunks string - for _, partInfo := range output.Parts { + for _, partInfo := range allParts.Parts { chunks += strconv.Itoa(partInfo.PartNumber) + "-" + partInfo.ETag + "," } @@ -100,39 +134,14 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadID - partNumberMarker := 0 - for { - output, err := ObsCli.ListParts(&obs.ListPartsInput{ - Bucket: setting.Bucket, - Key: input.Key, - UploadId: uploadID, - MaxParts: MAX_LIST_PARTS, - PartNumberMarker: partNumberMarker, - }) - if err != nil { - log.Error("ListParts failed:", err.Error()) - return err - } - - partNumberMarker = output.NextPartNumberMarker - log.Info("uuid:%s, MaxParts:%d, PartNumberMarker:%d, NextPartNumberMarker:%d, len:%d", uuid, output.MaxParts, output.PartNumberMarker, output.NextPartNumberMarker, len(output.Parts)) - - for _, partInfo := range output.Parts { - input.Parts = append(input.Parts, obs.Part{ - PartNumber: partInfo.PartNumber, - ETag: partInfo.ETag, - }) - } - - if len(output.Parts) < output.MaxParts { - break - } else { - continue - } - - break + allParts, err := listAllParts(uuid, uploadID, input.Key) + if err != nil { + log.Error("listAllParts failed: %v", err) + return err } + input.Parts = allParts.Parts + output, err := ObsCli.CompleteMultipartUpload(input) if err != nil { log.Error("CompleteMultipartUpload failed:", err.Error()) From f1108953bfa2f8e2ee8d077f4bc58095ef462c4e Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:08:34 +0800 Subject: [PATCH 008/125] =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/routers/search.go b/routers/search.go index 1cf78666e..457916d5a 100644 --- a/routers/search.go +++ b/routers/search.go @@ -183,7 +183,7 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) topicsQuery := elastic.NewMatchQuery("topics", Key) boolQ.Should(topicsQuery) - res, err := client.Search("repository-es-index").Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("topics")).Do(ctx.Req.Context()) + res, err := client.Search("repository-es-index").Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("topics")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -200,16 +200,15 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) } } -func getSort(SortBy string, ascending bool) elastic.Sorter { - var sort elastic.Sorter - sort = elastic.NewScoreSort() - if SortBy != "" { - if SortBy == "default" { - return sort - } - return elastic.NewFieldSort(SortBy).Order(ascending) +func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter { + sort := make([]elastic.Sorter, 0) + if SortBy == "default" { + sort = append(sort, elastic.NewScoreSort()) + sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) + return sort + } else { + return append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) } - return sort } func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) { @@ -308,7 +307,7 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third") boolQ.Should(nameQuery, descriptionQuery, topicsQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -330,7 +329,7 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa } else { log.Info("query all content.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -691,7 +690,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "created_unix.keyword", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -849,7 +848,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -864,7 +863,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1057,7 +1056,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 067d4ad7ef1d767fd53ad2f7f6030f6bb75b6ed1 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:17:59 +0800 Subject: [PATCH 009/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/search.go b/routers/search.go index 457916d5a..ab9e902da 100644 --- a/routers/search.go +++ b/routers/search.go @@ -848,7 +848,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "download_times", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 2c4907301cfeeb43faf0ad30b4e2a19036f55404 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:32:29 +0800 Subject: [PATCH 010/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/routers/search.go b/routers/search.go index ab9e902da..64832d794 100644 --- a/routers/search.go +++ b/routers/search.go @@ -204,7 +204,9 @@ func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending sort := make([]elastic.Sorter, 0) if SortBy == "default" { sort = append(sort, elastic.NewScoreSort()) - sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) + if secondSortBy != "" { + sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) + } return sort } else { return append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) @@ -690,7 +692,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "created_unix.keyword", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -848,7 +850,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "download_times", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -863,7 +865,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1056,7 +1058,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "updated_unix.keyword", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 86c540783b893000d6836ecd80e0ccded35418f2 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:50:35 +0800 Subject: [PATCH 011/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/search.go b/routers/search.go index 64832d794..a83b53261 100644 --- a/routers/search.go +++ b/routers/search.go @@ -205,6 +205,7 @@ func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending if SortBy == "default" { sort = append(sort, elastic.NewScoreSort()) if secondSortBy != "" { + log.Info("SortBy=" + SortBy + " secondSortBy=" + secondSortBy) sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) } return sort From 30e32a4a620fed50d4a15431921124e23f3936bd Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 15:55:58 +0800 Subject: [PATCH 012/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/search.go b/routers/search.go index a83b53261..58e39b476 100644 --- a/routers/search.go +++ b/routers/search.go @@ -208,10 +208,11 @@ func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending log.Info("SortBy=" + SortBy + " secondSortBy=" + secondSortBy) sort = append(sort, elastic.NewFieldSort(secondSortBy).Order(secondAscending)) } - return sort } else { - return append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) + sort = append(sort, elastic.NewFieldSort(SortBy).Order(ascending)) } + log.Info("sort size=" + fmt.Sprint(len(sort))) + return sort } func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) { From c2010ce0b4bd2c0dc3f370c21029fe02880867f8 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 18 Apr 2022 15:59:54 +0800 Subject: [PATCH 013/125] fix bug --- modules/storage/obs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 44de3a25e..2b52ab603 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -60,7 +60,7 @@ func ObsHasObject(path string) (bool, error) { } func listAllParts(uuid, uploadID, key string) (output *obs.ListPartsOutput, err error) { - output = nil + output = &obs.ListPartsOutput{} partNumberMarker := 0 for { temp, err := ObsCli.ListParts(&obs.ListPartsInput{ From 3a60fbdf99256ced9142d9d33f833d1ac5456330 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 18 Apr 2022 16:09:31 +0800 Subject: [PATCH 014/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/search.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/routers/search.go b/routers/search.go index 58e39b476..f2a2f30d6 100644 --- a/routers/search.go +++ b/routers/search.go @@ -200,6 +200,16 @@ func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) } } +func getSortSingle(SortBy string, ascending bool) elastic.Sorter { + var sort elastic.Sorter + if SortBy == "default" { + sort = elastic.NewScoreSort() + } else { + sort = elastic.NewFieldSort(SortBy).Order(ascending) + } + return sort +} + func getSort(SortBy string, ascending bool, secondSortBy string, secondAscending bool) []elastic.Sorter { sort := make([]elastic.Sorter, 0) if SortBy == "default" { @@ -694,7 +704,7 @@ func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(UserOrOrgQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -852,7 +862,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -867,7 +877,7 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, } else { log.Info("query all datasets.") //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} - res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Do(ctx.Req.Context()) + res, err := client.Search(TableName).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) @@ -1060,7 +1070,7 @@ func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page in boolQ.Must(isIssueQuery) } - res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "", false)...).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSortSingle(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) if err == nil { searchJson, _ := json.Marshal(res) log.Info("searchJson=" + string(searchJson)) From 784b948c748862237225cd68bca362889d5181a5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 18 Apr 2022 16:12:27 +0800 Subject: [PATCH 015/125] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 45 +++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index dea9b6aa0..88cabb4ab 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -422,10 +422,10 @@ td, th {
- + - + @@ -274,19 +279,20 @@ $('.menu .item') .tab(); - let sever_num = $('#trainjob_work_server_num') - $('.add').click(function(){ - sever_num.val(parseInt(sever_num.val())+1) - if(sever_num.val()>=26){ - sever_num.val(parseInt(sever_num.val())-1) - } - }) - $('.min').click(function(){ - sever_num.val(parseInt(sever_num.val())-1) - if(sever_num.val()<=0){ - sever_num.val(parseInt(sever_num.val())+1) - } - }) + // let sever_num = $("#trainjob_work_server_num_select .text").text() //$('#trainjob_work_server_num') + // console.log("sever_num:",sever_num) + // $('.add').click(function(){ + // sever_num.val(parseInt(sever_num.val())+1) + // if(sever_num.val()>=26){ + // sever_num.val(parseInt(sever_num.val())-1) + // } + // }) + // $('.min').click(function(){ + // sever_num.val(parseInt(sever_num.val())-1) + // if(sever_num.val()<=0){ + // sever_num.val(parseInt(sever_num.val())+1) + // } + // }) // 参数增加、删除、修改、保存 function Add_parameter(i){ value = '
' + @@ -360,7 +366,7 @@ // $("select[name='pool_id']").val(parameters[i]); // break; case (6): - $("input[name='work_server_number']").val(parameters[i]); + // $("input[name='work_server_number']").val(parameters[i]); break; } } @@ -467,6 +473,10 @@ $("input#ai_engine_name").val(name1) $("input#ai_flaver_name").val(name2) + let val_server_num_select = $("#trainjob_work_server_num_select .text").text() + // console.log("val_server_num_select:",val_server_num_select) + $("input#trainjob_work_server_num").val(val_server_num_select) + } $('.ui.create_train_job.green.button').click(function(e) { get_name() From eaf8090feff19e7e8545031ad4d4d2985e44fd12 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 19 Apr 2022 15:31:34 +0800 Subject: [PATCH 041/125] fix issue --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + templates/admin/dataset/list.tmpl | 45 ++++++++++++++++++++++++++++----------- templates/admin/repo/list.tmpl | 37 +++++++++++++++++++++++++++++++- templates/explore/datasets.tmpl | 3 ++- 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 14d4e19d1..58c6cc84a 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1389,6 +1389,7 @@ issues.filter_sort.feweststars = Fewest stars issues.filter_sort.mostforks = Most forks issues.filter_sort.fewestforks = Fewest forks issues.filter_sort.downloadtimes = Most downloaded +issues.filter_sort.moststars = Most star issues.action_open = Open issues.action_close = Close issues.action_label = Label diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index de9f79319..3e010893a 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1401,6 +1401,7 @@ issues.filter_sort.feweststars=点赞由少到多 issues.filter_sort.mostforks=派生由多到少 issues.filter_sort.fewestforks=派生由少到多 issues.filter_sort.downloadtimes=下载次数 +issues.filter_sort.moststars=收藏数量 issues.action_open=开启 issues.action_close=关闭 issues.action_label=标签 diff --git a/templates/admin/dataset/list.tmpl b/templates/admin/dataset/list.tmpl index 1044e4c28..4e73d10b5 100644 --- a/templates/admin/dataset/list.tmpl +++ b/templates/admin/dataset/list.tmpl @@ -27,7 +27,7 @@ {{.Title}} {{.CreatedUnix.FormatShort}} - + {{if .Recommend}}取消推荐{{else}}设为推荐{{end}} {{end}} @@ -37,16 +37,35 @@ {{template "base/paginate" .}}
- -
-
- - {{.i18n.Tr "dataset.settings.delete"}} -
-
-

{{.i18n.Tr "dataset.settings.delete_desc"}}

- {{.i18n.Tr "dataset.settings.delete_notices_2" `` | Safe}}
-
- {{template "base/delete_modal_actions" .}} -
{{template "base/footer" .}} + \ No newline at end of file diff --git a/templates/admin/repo/list.tmpl b/templates/admin/repo/list.tmpl index 3dee62bfb..ddc4ea81a 100644 --- a/templates/admin/repo/list.tmpl +++ b/templates/admin/repo/list.tmpl @@ -44,7 +44,7 @@ {{.NumIssues}} {{SizeFmt .Size}} {{.CreatedUnix.FormatShort}} - + {{end}} @@ -68,3 +68,38 @@ {{template "base/delete_modal_actions" .}}
{{template "base/footer" .}} + \ No newline at end of file diff --git a/templates/explore/datasets.tmpl b/templates/explore/datasets.tmpl index a48dbf478..20eafb26f 100644 --- a/templates/explore/datasets.tmpl +++ b/templates/explore/datasets.tmpl @@ -125,7 +125,8 @@ {{.i18n.Tr "repo.issues.filter_sort.oldest"}} {{.i18n.Tr "repo.issues.filter_sort.recentupdate"}} {{.i18n.Tr "repo.issues.filter_sort.leastupdate"}} - + {{.i18n.Tr "repo.issues.filter_sort.downloadtimes"}} + {{.i18n.Tr "repo.issues.issues.filter_sort.moststars"}}
From 7cb3a03fffd1f56c170ffb5578efd86d1fcdd69d Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 19 Apr 2022 15:53:14 +0800 Subject: [PATCH 042/125] =?UTF-8?q?=E4=BF=AE=E6=94=B9bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/routes/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index c281d832b..a7238e602 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -525,7 +525,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/datasets", func() { m.Get("", admin.Datasets) - m.Put(":id/action/:action", admin.DatasetAction) + m.Put("/:id/action/:action", admin.DatasetAction) // m.Post("/delete", admin.DeleteDataset) }) m.Group("/cloudbrains", func() { From ebe56325ff92ca63bd058a2c054c9a6c5de86dcf Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 19 Apr 2022 16:15:13 +0800 Subject: [PATCH 043/125] fix-1656-patch1 --- models/cloudbrain.go | 4 ++-- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 0465faf9a..45ed52bd5 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1629,12 +1629,12 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { var err error condition := "cloudbrain.user_id = `user`.id" if len(opts.Keyword) == 0 { - count, err = sess.Where(cond).Count(new(Cloudbrain)) + count, err = sess.Unscoped().Where(cond).Count(new(Cloudbrain)) } else { lowerKeyWord := strings.ToLower(opts.Keyword) cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) - count, err = sess.Table(&Cloudbrain{}).Where(cond). + count, err = sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Join("left", "`user`", condition).Count(new(CloudbrainInfo)) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index b979729a8..2090a2cf2 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -23,7 +23,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { _, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, - PageSize: 1, + PageSize: pageSize, }, Type: models.TypeCloudBrainAll, NeedRepoInfo: false, From 1427ee25b75bbd522e5e0c519940746525ffd819 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 19 Apr 2022 16:23:23 +0800 Subject: [PATCH 044/125] del md5 show --- templates/repo/datasets/index.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index e8d9b65e3..ba4842f57 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -281,7 +281,7 @@ {{$.i18n.Tr "dataset.copy_url"}} - {{$.i18n.Tr "dataset.copy_md5"}} + {{if and ($.CanWrite) (eq .DecompressState 1) }} {{$.i18n.Tr "dataset.annotation"}} {{end}} From 747c4c9d44a1d3063d9db738cc8ec8d00b6eafc9 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 19 Apr 2022 17:27:21 +0800 Subject: [PATCH 045/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/modelarts.go | 4 +++- templates/repo/cloudbrain/show.tmpl | 2 +- templates/repo/modelarts/notebook/show.tmpl | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 318726e8e..1147dc5a9 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -247,7 +247,9 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm func NotebookShow(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true debugListType := ctx.Query("debugListType") - + if debugListType == "" { + debugListType = "all" + } var ID = ctx.Params(":id") task, err := models.GetCloudbrainByIDWithDeleted(ID) if err != nil { diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 7e16c36ff..5fabaa937 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -187,7 +187,7 @@ td, th { {{.i18n.Tr "repo.cloudbrain"}}
/
- + {{$.i18n.Tr "repo.modelarts.notebook"}}
/
diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index 0d7a01efb..26839345e 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -193,7 +193,7 @@ td, th { {{.i18n.Tr "repo.cloudbrain"}}
/
- + {{$.i18n.Tr "repo.modelarts.notebook"}}
/
From bc02f0851b5facf1aa1954429998ef4fc52bd05a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 19 Apr 2022 17:37:58 +0800 Subject: [PATCH 046/125] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 76a825fa6..dcf999001 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -733,7 +733,7 @@ func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImage Place: form.Place, }) if err != nil { - log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"]) + log.Error("CommitImagefailed") if models.IsErrImageTagExist(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist"))) From f9342252f1d0d046fa48600a38870e16215eb91b Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 20 Apr 2022 09:36:29 +0800 Subject: [PATCH 047/125] fix issue --- templates/admin/dataset/list.tmpl | 38 ++++------------------------ templates/admin/repo/list.tmpl | 37 +-------------------------- templates/custom/select_dataset.tmpl | 8 +++--- templates/custom/select_dataset_train.tmpl | 8 +++--- templates/explore/datasets.tmpl | 3 ++- web_src/js/components/images/adminImages.vue | 26 ++++++++++--------- web_src/js/features/images.js | 5 ++-- web_src/js/index.js | 26 +++++++++++++++++++ 8 files changed, 58 insertions(+), 93 deletions(-) diff --git a/templates/admin/dataset/list.tmpl b/templates/admin/dataset/list.tmpl index 4e73d10b5..afdfdf7f6 100644 --- a/templates/admin/dataset/list.tmpl +++ b/templates/admin/dataset/list.tmpl @@ -3,6 +3,9 @@ {{template "admin/navbar" .}}
{{template "base/alert" .}} +

{{.i18n.Tr "admin.datasets.dataset_manage_panel"}} ({{.i18n.Tr "admin.total" .Total}})

@@ -24,7 +27,7 @@ {{range .Datasets}} {{.ID}} - {{.Title}} + {{.Title}}{{if .Recommend}}{{end}} {{.CreatedUnix.FormatShort}} {{if .Recommend}}取消推荐{{else}}设为推荐{{end}} @@ -37,35 +40,4 @@ {{template "base/paginate" .}}
-{{template "base/footer" .}} - \ No newline at end of file +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/admin/repo/list.tmpl b/templates/admin/repo/list.tmpl index ddc4ea81a..3dee62bfb 100644 --- a/templates/admin/repo/list.tmpl +++ b/templates/admin/repo/list.tmpl @@ -44,7 +44,7 @@ {{.NumIssues}} {{SizeFmt .Size}} {{.CreatedUnix.FormatShort}} - + {{end}} @@ -68,38 +68,3 @@ {{template "base/delete_modal_actions" .}}
{{template "base/footer" .}} - \ No newline at end of file diff --git a/templates/custom/select_dataset.tmpl b/templates/custom/select_dataset.tmpl index cd4ce50da..befd186c5 100644 --- a/templates/custom/select_dataset.tmpl +++ b/templates/custom/select_dataset.tmpl @@ -23,7 +23,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias} ${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
@@ -49,7 +49,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
@@ -74,7 +74,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
@@ -99,7 +99,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
diff --git a/templates/custom/select_dataset_train.tmpl b/templates/custom/select_dataset_train.tmpl index 3da9be9aa..b8260f279 100644 --- a/templates/custom/select_dataset_train.tmpl +++ b/templates/custom/select_dataset_train.tmpl @@ -23,7 +23,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias} ${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias} ${dataset.Name}
@@ -49,7 +49,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
@@ -74,7 +74,7 @@
-
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}