From a1c59d0a8f6e64b270074fe45a9ef353a3ba7fa9 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 16 Aug 2022 11:11:39 +0800 Subject: [PATCH 01/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/dataset.go | 8 +- models/dataset_reference.go | 42 ++++++++++ modules/auth/dataset.go | 8 ++ modules/dataset/dataset.go | 18 +++++ modules/setting/repository.go | 7 +- modules/templates/helper.go | 51 ++++++------ options/locale/locale_en-US.ini | 2 + options/locale/locale_zh-CN.ini | 2 + routers/home.go | 19 ++--- routers/repo/dataset.go | 167 +++++++++++++++++++++++++++++++++++----- routers/routes/routes.go | 5 ++ services/repository/dataset.go | 19 +++++ 12 files changed, 283 insertions(+), 65 deletions(-) create mode 100644 models/dataset_reference.go create mode 100644 services/repository/dataset.go diff --git a/models/dataset.go b/models/dataset.go index 7f049f068..45dfad735 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -121,7 +121,7 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) if attachment.DatasetID == datasets[i].ID { if opts.StarByMe { - permission,ok := permissionMap[datasets[i].ID]; + permission, ok := permissionMap[datasets[i].ID] if !ok { permission = false @@ -159,8 +159,8 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) } for i := range datasets { - if datasets[i].Attachments==nil{ - datasets[i].Attachments=[]*Attachment{} + if datasets[i].Attachments == nil { + datasets[i].Attachments = []*Attachment{} } datasets[i].Repo.Owner = nil } @@ -178,7 +178,7 @@ type SearchDatasetOptions struct { Category string Task string License string - DatasetIDs []int64 // 目前只在StarByMe为true时起作用 + DatasetIDs []int64 ListOptions SearchOrderBy IsOwner bool diff --git a/models/dataset_reference.go b/models/dataset_reference.go new file mode 100644 index 000000000..f3083ca32 --- /dev/null +++ b/models/dataset_reference.go @@ -0,0 +1,42 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type DatasetReference struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX unique(repo_dataset)"` + DatasetID int64 `xorm:"INDEX unique(repo_dataset)"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + +func GetDatasetIdsByRepoID(repoID int64) []int64 { + var datasets []int64 + _ = x.Table("dataset_reference").Where("repo_id=?", repoID). + Cols("dataset_reference.dataset_id").Find(&datasets) + return datasets +} + +func DeleteDatasetIdsByRepoID(repoID int64, datasetIds []int64) error { + if len(datasetIds) == 0 { + return nil + } + _, err := x.In("dataset_id", datasetIds).And("repo_id", repoID).Delete(new(DatasetReference)) + return err +} + +func NewDatasetIdsByRepoID(repoID int64, datasetIds []int64) error { + var datasetReference []DatasetReference + for _, datasetId := range datasetIds { + datasetReference = append(datasetReference, DatasetReference{ + DatasetID: datasetId, + RepoID: repoID, + }) + } + _, err := x.Insert(datasetReference) + return err +} + +func DeleteReferenceDatasetByDatasetID(datasetId int64) error { + _, err := x.Delete(&DatasetReference{DatasetID: datasetId}) + return err +} diff --git a/modules/auth/dataset.go b/modules/auth/dataset.go index 71b5ac938..8aed3a8c2 100755 --- a/modules/auth/dataset.go +++ b/modules/auth/dataset.go @@ -44,3 +44,11 @@ type EditAttachmentForm struct { func (f *EditAttachmentForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { return validate(errs, ctx.Data, f, ctx.Locale) } + +type ReferenceDatasetForm struct { + DatasetID []int64 `binding:"Required"` +} + +func (f *ReferenceDatasetForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/modules/dataset/dataset.go b/modules/dataset/dataset.go index a180af184..f79f0d1c2 100644 --- a/modules/dataset/dataset.go +++ b/modules/dataset/dataset.go @@ -1,5 +1,7 @@ package dataset +import "code.gitea.io/gitea/models" + func GetResourceType(cloudbrainType int) string { if cloudbrainType == 0 { return "CPU/GPU" @@ -15,3 +17,19 @@ func GetStatusText(isPrivate bool) string { return "dataset.public" } } + +func IsShowDataSetOfCurrentRepo(repoID int64) bool { + repo := models.Repository{ + ID: repoID, + } + + dataset, _ := models.GetDatasetByRepo(&repo) + if dataset != nil { + return true + } + if len(models.GetDatasetIdsByRepoID(repoID)) > 0 { + return false + } + return true + +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index ee4f8b379..1eb1875b2 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -193,8 +193,9 @@ var ( Wiki: []string{"never"}, }, } - RepoRootPath string - ScriptType = "bash" + RepoRootPath string + RepoMaxReferenceDatasetNum int + ScriptType = "bash" ) func newRepository() { @@ -210,6 +211,8 @@ func newRepository() { Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) RepoRootPath = sec.Key("ROOT").MustString(path.Join(homeDir, "gitea-repositories")) + RepoMaxReferenceDatasetNum = sec.Key("MAX_REF_DATASET_NUM").MustInt(20) + forcePathSeparator(RepoRootPath) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 857e365f8..8e7723e75 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -97,23 +97,24 @@ func NewFuncMap() []template.FuncMap { "AllowedReactions": func() []string { return setting.UI.Reactions }, - "AvatarLink": models.AvatarLink, - "Safe": Safe, - "SafeJS": SafeJS, - "Str2html": Str2html, - "subOne": subOne, - "TimeSince": timeutil.TimeSince, - "TimeSinceUnix": timeutil.TimeSinceUnix, - "TimeSinceUnix1": timeutil.TimeSinceUnix1, - "AttachmentResourceType": dataset.GetResourceType, - "AttachmentStatus": dataset.GetStatusText, - "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, - "RawTimeSince": timeutil.RawTimeSince, - "FileSize": base.FileSize, - "PrettyNumber": base.PrettyNumber, - "Subtract": base.Subtract, - "EntryIcon": base.EntryIcon, - "MigrationIcon": MigrationIcon, + "AvatarLink": models.AvatarLink, + "Safe": Safe, + "SafeJS": SafeJS, + "Str2html": Str2html, + "subOne": subOne, + "TimeSince": timeutil.TimeSince, + "TimeSinceUnix": timeutil.TimeSinceUnix, + "TimeSinceUnix1": timeutil.TimeSinceUnix1, + "AttachmentResourceType": dataset.GetResourceType, + "AttachmentStatus": dataset.GetStatusText, + "IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo, + "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, + "RawTimeSince": timeutil.RawTimeSince, + "FileSize": base.FileSize, + "PrettyNumber": base.PrettyNumber, + "Subtract": base.Subtract, + "EntryIcon": base.EntryIcon, + "MigrationIcon": MigrationIcon, "Add": func(a, b int) int { return a + b }, @@ -357,13 +358,15 @@ func NewTextFuncMap() []texttmpl.FuncMap { "AppDomain": func() string { return setting.Domain }, - "TimeSince": timeutil.TimeSince, - "TimeSinceUnix": timeutil.TimeSinceUnix, - "TimeSinceUnix1": timeutil.TimeSinceUnix1, - "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, - "RawTimeSince": timeutil.RawTimeSince, - "AttachmentResourceType": dataset.GetResourceType, - "AttachmentStatus": dataset.GetStatusText, + "TimeSince": timeutil.TimeSince, + "TimeSinceUnix": timeutil.TimeSinceUnix, + "TimeSinceUnix1": timeutil.TimeSinceUnix1, + "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, + "RawTimeSince": timeutil.RawTimeSince, + "AttachmentResourceType": dataset.GetResourceType, + "AttachmentStatus": dataset.GetStatusText, + "IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo, + "DateFmtLong": func(t time.Time) string { return t.Format(time.RFC1123Z) }, diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0fecf7de2..eda1af4ce 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -832,6 +832,8 @@ category = Category no_category = No Category task = Task no_task = No Task +reference_dataset_fail=Failed to reference dataset, please try again later. +cancel_reference_dataset_fail=Failed to cancel reference dataset, please try again later. license = License no_license = No License file = Dataset File diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2878627e2..ac38820b1 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -830,6 +830,8 @@ create_dataset=创建数据集 create_dataset_fail=创建数据集失败。 query_dataset_fail=查询数据集失败。 edit_attachment_fail=修改描述失败。 +reference_dataset_fail=关联数据集失败，请稍后再试。 +cancel_reference_dataset_fail=取消关联数据集失败，请稍后再试。 show_dataset=数据集 edit_dataset=编辑数据集 diff --git a/routers/home.go b/routers/home.go index 1a697946a..026491156 100755 --- a/routers/home.go +++ b/routers/home.go @@ -296,11 +296,10 @@ func ExploreDatasets(ctx *context.Context) { // ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled var ( - datasets []*models.Dataset - datasetsWithStar []*models.DatasetWithStar - count int64 - err error - orderBy models.SearchOrderBy + datasets []*models.Dataset + count int64 + err error + orderBy models.SearchOrderBy ) page := ctx.QueryInt("page") if page <= 0 { @@ -379,14 +378,6 @@ func ExploreDatasets(ctx *context.Context) { ctx.ServerError("SearchDatasets", err) return } - for _, dataset := range datasets { - if !ctx.IsSigned { - datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: false}) - } else { - datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: models.IsDatasetStaring(ctx.User.ID, dataset.ID)}) - } - - } pager := context.NewPagination(int(count), opts.PageSize, page, 5) ctx.Data["Keyword"] = opts.Keyword @@ -397,7 +388,7 @@ func ExploreDatasets(ctx *context.Context) { pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["Datasets"] = datasetsWithStar + ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets) ctx.Data["Total"] = count ctx.Data["PageIsDatasets"] = true ctx.HTML(200, tplExploreDataset) diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 4dde646a6..e987e4573 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -9,6 +9,8 @@ import ( "strings" "unicode/utf8" + "code.gitea.io/gitea/services/repository" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -22,6 +24,7 @@ const ( tplDatasetCreate base.TplName = "repo/datasets/create" tplDatasetEdit base.TplName = "repo/datasets/edit" taskstplIndex base.TplName = "repo/datasets/tasks/index" + tplReference base.TplName = "repo/datasets/reference" ) // MustEnableDataset check if repository enable internal dataset @@ -267,6 +270,59 @@ func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) { } } +func ReferenceDatasetDelete(ctx *context.Context) { + repoID := ctx.Repo.Repository.ID + datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) + + err := models.DeleteDatasetIdsByRepoID(repoID, []int64{datasetId}) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.cancel_reference_dataset_fail")) + } + ctx.JSON(http.StatusOK, models.BaseOKMessage) + +} + +func ReferenceDatasetPost(ctx *context.Context, form auth.ReferenceDatasetForm) { + repoID := ctx.Repo.Repository.ID + datasetsID := models.GetDatasetIdsByRepoID(repoID) + + var newDataset []int64 + var deletedDataset []int64 + + for _, id := range datasetsID { + if !contains(form.DatasetID, id) { + deletedDataset = append(deletedDataset, id) + } + } + + for _, id := range form.DatasetID { + if !contains(datasetsID, id) { + newDataset = append(newDataset, id) + } + } + + err := models.DeleteDatasetIdsByRepoID(repoID, deletedDataset) + + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.reference_dataset_fail"))) + } + err = models.NewDatasetIdsByRepoID(repoID, newDataset) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.reference_dataset_fail")) + } + + ctx.JSON(http.StatusOK, models.BaseOKMessage) + +} + +func contains(s []int64, e int64) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) { ctx.Data["PageIsDataset"] = true @@ -411,24 +467,13 @@ func MyDatasets(ctx *context.Context) { } func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { - page := ctx.QueryInt("page") - cloudbrainType := ctx.QueryInt("type") - keyword := strings.Trim(ctx.Query("q"), " ") - orderBy := models.SearchOrderByRecentUpdated - opts.Keyword = keyword - opts.SearchOrderBy = orderBy - opts.RecommendOnly = ctx.QueryBool("recommend") - opts.CloudBrainType = cloudbrainType - opts.ListOptions = models.ListOptions{ - Page: page, - PageSize: setting.UI.DatasetPagingNum, - } - opts.NeedAttachment = true - opts.JustNeedZipFile = true - opts.User = ctx.User + datasets, count, err := datasetMultipleGet(ctx, opts) - datasets, count, err := models.SearchDataset(opts) + datasetMultipleResult(ctx, err, datasets, count) + +} +func datasetMultipleResult(ctx *context.Context, err error, datasets models.DatasetList, count int64) { if err != nil { ctx.ServerError("datasets", err) return @@ -449,15 +494,53 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { "data": string(data), "count": strconv.FormatInt(count, 10), }) +} +func datasetMultipleGet(ctx *context.Context, opts *models.SearchDatasetOptions) (models.DatasetList, int64, error) { + page := ctx.QueryInt("page") + keyword := strings.Trim(ctx.Query("q"), " ") + orderBy := models.SearchOrderByRecentUpdated + opts.Keyword = keyword + opts.SearchOrderBy = orderBy + opts.RecommendOnly = ctx.QueryBool("recommend") + opts.ListOptions = models.ListOptions{ + Page: page, + PageSize: setting.UI.DatasetPagingNum, + } + opts.JustNeedZipFile = true + opts.User = ctx.User + + datasets, count, err := models.SearchDataset(opts) + return datasets, count, err } func CurrentRepoDatasetMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ - RepoID: ctx.Repo.Repository.ID, + RepoID: ctx.Repo.Repository.ID, + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), + DatasetIDs: models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID), + } + datasetList, count, err := datasetMultipleGet(ctx, opts) + + if len(datasetList) > 0 { + var convertDatasetList models.DatasetList + + for _, dataset := range datasetList { + if dataset.RepoID == ctx.Repo.Repository.ID && len(convertDatasetList) == 0 { + convertDatasetList = append(convertDatasetList, dataset) + } + } + for _, dataset := range datasetList { + if dataset.RepoID != ctx.Repo.Repository.ID { + convertDatasetList = append(convertDatasetList, dataset) + } + } + datasetMultipleResult(ctx, err, convertDatasetList, count) + } else { + datasetMultipleResult(ctx, err, datasetList, count) } - datasetMultiple(ctx, opts) } @@ -465,6 +548,19 @@ func MyDatasetsMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ UploadAttachmentByMe: true, + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), + } + datasetMultiple(ctx, opts) + +} + +func ReferenceDatasetAvailable(ctx *context.Context) { + + opts := &models.SearchDatasetOptions{ + PublicOnly: true, + NeedAttachment: false, + CloudBrainType: models.TypeCloudBrainAll, } datasetMultiple(ctx, opts) @@ -473,7 +569,9 @@ func MyDatasetsMultiple(ctx *context.Context) { func PublicDatasetMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ - PublicOnly: true, + PublicOnly: true, + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), } datasetMultiple(ctx, opts) @@ -482,11 +580,38 @@ func PublicDatasetMultiple(ctx *context.Context) { func MyFavoriteDatasetMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ - StarByMe: true, - DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID), + StarByMe: true, + DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID), + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), } datasetMultiple(ctx, opts) } +func ReferenceDataset(ctx *context.Context) { + MustEnableDataset(ctx) + opts := &models.SearchDatasetOptions{ + DatasetIDs: models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID), + NeedAttachment: false, + CloudBrainType: models.TypeCloudBrainAll, + ListOptions: models.ListOptions{ + Page: 1, + PageSize: setting.RepoMaxReferenceDatasetNum, + }, + SearchOrderBy: models.SearchOrderByRecentUpdated, + } + datasets, count, err := models.SearchDataset(opts) + + if err != nil { + ctx.ServerError("SearchDatasets", err) + return + } + + ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets) + ctx.Data["PageIsDataset"] = true + ctx.Data["Total"] = count + ctx.HTML(200, tplReference) + +} func PublicDataset(ctx *context.Context) { page := ctx.QueryInt("page") diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a4bc09472..d54606581 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1032,10 +1032,13 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/datasets", func() { m.Get("", reqRepoDatasetReader, repo.DatasetIndex) + m.Get("/reference_datasets", repo.ReferenceDataset) + m.Delete("/reference_datasets/:id", repo.ReferenceDatasetDelete) m.Put("/:id/:action", reqRepoDatasetReader, repo.DatasetAction) m.Get("/create", reqRepoDatasetWriter, repo.CreateDataset) m.Post("/create", reqRepoDatasetWriter, bindIgnErr(auth.CreateDatasetForm{}), repo.CreateDatasetPost) m.Get("/edit/:id", reqRepoDatasetWriter, repo.EditDataset) + m.Post("/reference_datasets", reqRepoDatasetWriter, bindIgnErr(auth.ReferenceDatasetForm{}), repo.ReferenceDatasetPost) m.Post("/edit", reqRepoDatasetWriter, bindIgnErr(auth.EditDatasetForm{}), repo.EditDatasetPost) m.Get("/current_repo", repo.CurrentRepoDataset) m.Get("/my_datasets", repo.MyDatasets) @@ -1045,6 +1048,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/current_repo_m", repo.CurrentRepoDatasetMultiple) m.Get("/my_datasets_m", repo.MyDatasetsMultiple) m.Get("/public_datasets_m", repo.PublicDatasetMultiple) + + m.Get("/reference_datasets_available", repo.ReferenceDatasetAvailable) m.Get("/my_favorite_m", repo.MyFavoriteDatasetMultiple) m.Group("/status", func() { diff --git a/services/repository/dataset.go b/services/repository/dataset.go new file mode 100644 index 000000000..ffe3c5466 --- /dev/null +++ b/services/repository/dataset.go @@ -0,0 +1,19 @@ +package repository + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" +) + +func ConvertToDatasetWithStar(ctx *context.Context, datasets []*models.Dataset) []*models.DatasetWithStar { + var datasetsWithStar []*models.DatasetWithStar + for _, dataset := range datasets { + if !ctx.IsSigned { + datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: false}) + } else { + datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: models.IsDatasetStaring(ctx.User.ID, dataset.ID)}) + } + + } + return datasetsWithStar +} From 46e97062a2b99ceb75aeb9069a3310314f682884 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 17 Aug 2022 14:56:36 +0800 Subject: [PATCH 02/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/dataset.go | 20 ++++--- models/dataset_reference.go | 81 ++++++++++++++++++++++------- models/models.go | 2 +- modules/dataset/dataset.go | 2 +- routers/repo/dataset.go | 124 ++++++++++++++++---------------------------- 5 files changed, 123 insertions(+), 106 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 45dfad735..04591c64e 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -6,6 +6,8 @@ import ( "sort" "strings" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" @@ -88,7 +90,7 @@ func (datasets DatasetList) loadAttributes(e Engine) error { if err := e. Where("id > 0"). In("id", keysInt64(set)). - Cols("id", "owner_id", "owner_name", "lower_name", "name", "description", "alias", "lower_alias","is_private"). + Cols("id", "owner_id", "owner_name", "lower_name", "name", "description", "alias", "lower_alias", "is_private"). Find(&repos); err != nil { return fmt.Errorf("find repos: %v", err) } @@ -126,7 +128,7 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) permission = false datasets[i].Repo.GetOwner() - if datasets[i].Repo.Owner.IsOrganization() { + if datasets[i].Repo.Owner.IsOrganization() { if datasets[i].Repo.Owner.IsUserPartOfOrg(opts.User.ID) { log.Info("user is member of org.") permission = true @@ -140,10 +142,10 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) } } - permissionMap[datasets[i].ID]=permission + permissionMap[datasets[i].ID] = permission } - if permission{ + if permission { datasets[i].Attachments = append(datasets[i].Attachments, attachment) } else if !attachment.IsPrivate { datasets[i].Attachments = append(datasets[i].Attachments, attachment) @@ -329,13 +331,15 @@ func SearchDatasetByCondition(opts *SearchDatasetOptions, cond builder.Cond) (Da return nil, 0, fmt.Errorf("Count: %v", err) } - sess.Select(selectColumnsSql).Join("INNER", "repository", "repository.id = dataset.repo_id"). + builderQuery := builder.Dialect(setting.Database.Type).Select("id", "title", "status", "category", "description", "download_times", "license", "task", "release_id", "user_id", "repo_id", "created_unix", "updated_unix", "num_stars", "recommend", "use_count").From(builder.Dialect(setting.Database.Type).Select(selectColumnsSql).From("dataset").Join("INNER", "repository", "repository.id = dataset.repo_id"). Join("INNER", "attachment", "attachment.dataset_id=dataset.id"). - Where(cond).OrderBy(opts.SearchOrderBy.String()) + Where(cond), "d").OrderBy(opts.SearchOrderBy.String()) + if opts.PageSize > 0 { - sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) + builderQuery.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) } - if err = sess.Find(&datasets); err != nil { + + if err = sess.SQL(builderQuery).Find(&datasets); err != nil { return nil, 0, fmt.Errorf("Dataset: %v", err) } diff --git a/models/dataset_reference.go b/models/dataset_reference.go index f3083ca32..90707e2be 100644 --- a/models/dataset_reference.go +++ b/models/dataset_reference.go @@ -1,42 +1,87 @@ package models -import "code.gitea.io/gitea/modules/timeutil" +import ( + "strconv" + "strings" + + "code.gitea.io/gitea/modules/timeutil" +) type DatasetReference struct { ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"INDEX unique(repo_dataset)"` - DatasetID int64 `xorm:"INDEX unique(repo_dataset)"` + RepoID int64 `xorm:"INDEX unique"` + DatasetID string `xorm:"TEXT"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } func GetDatasetIdsByRepoID(repoID int64) []int64 { var datasets []int64 + var datasetIds []string _ = x.Table("dataset_reference").Where("repo_id=?", repoID). - Cols("dataset_reference.dataset_id").Find(&datasets) + Cols("dataset_reference.dataset_id").Find(&datasetIds) + if len(datasetIds) > 0 { + for _, datasetIdStr := range strings.Split(datasetIds[0], ",") { + datasetId, err := strconv.ParseInt(datasetIdStr, 10, 64) + if err != nil { + continue + } + datasets = append(datasets, datasetId) + } + } + return datasets } -func DeleteDatasetIdsByRepoID(repoID int64, datasetIds []int64) error { - if len(datasetIds) == 0 { - return nil +func HasReferenceDataset(repoID int64) bool { + + var datasetIds []string + _ = x.Table("dataset_reference").Where("repo_id=?", repoID). + Cols("dataset_reference.dataset_id").Find(&datasetIds) + return len(datasetIds) > 0 +} + +func getReferenceDatasetStr(repoID int64) string { + + var datasetIds []string + _ = x.Table("dataset_reference").Where("repo_id=?", repoID). + Cols("dataset_reference.dataset_id").Find(&datasetIds) + if len(datasetIds) > 0 { + return datasetIds[0] } - _, err := x.In("dataset_id", datasetIds).And("repo_id", repoID).Delete(new(DatasetReference)) + return "" +} + +func deleteDatasetIdsByRepoID(repoID int64) error { + + _, err := x.Where("repo_id", repoID).Delete(new(DatasetReference)) return err } func NewDatasetIdsByRepoID(repoID int64, datasetIds []int64) error { - var datasetReference []DatasetReference + if len(datasetIds) == 0 { //关联数据集数组为空，不需要处理 + return nil + } + var datasetsStrArray []string for _, datasetId := range datasetIds { - datasetReference = append(datasetReference, DatasetReference{ - DatasetID: datasetId, - RepoID: repoID, - }) + datasetsStrArray = append(datasetsStrArray, strconv.FormatInt(datasetId, 10)) } - _, err := x.Insert(datasetReference) - return err -} -func DeleteReferenceDatasetByDatasetID(datasetId int64) error { - _, err := x.Delete(&DatasetReference{DatasetID: datasetId}) + newDatasetStr := strings.Join(datasetsStrArray, ",") + oldDatasetStr := getReferenceDatasetStr(repoID) + if newDatasetStr == oldDatasetStr { //关联数据集无变化，不需要处理 + return nil + } + if oldDatasetStr != "" { //已经存在关联数据集 + err := deleteDatasetIdsByRepoID(repoID) + if err != nil { + return err + } + } + datasetReference := DatasetReference{ + DatasetID: newDatasetStr, + RepoID: repoID, + } + + _, err := x.Insert(datasetReference) return err } diff --git a/models/models.go b/models/models.go index 8898955a7..25ddf5a0e 100755 --- a/models/models.go +++ b/models/models.go @@ -145,7 +145,7 @@ func init() { new(OrgStatistic), new(SearchRecord), new(AiModelConvert), - new(CloudbrainTemp), + new(DatasetReference), ) tablesStatistic = append(tablesStatistic, diff --git a/modules/dataset/dataset.go b/modules/dataset/dataset.go index f79f0d1c2..297ed460d 100644 --- a/modules/dataset/dataset.go +++ b/modules/dataset/dataset.go @@ -27,7 +27,7 @@ func IsShowDataSetOfCurrentRepo(repoID int64) bool { if dataset != nil { return true } - if len(models.GetDatasetIdsByRepoID(repoID)) > 0 { + if models.HasReferenceDataset(repoID) { return false } return true diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index e987e4573..c058c4e9c 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -274,7 +274,16 @@ func ReferenceDatasetDelete(ctx *context.Context) { repoID := ctx.Repo.Repository.ID datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) - err := models.DeleteDatasetIdsByRepoID(repoID, []int64{datasetId}) + oldDatasetIds := models.GetDatasetIdsByRepoID(repoID) + + var newDatasetIds []int64 + + for _, tempDatasetId := range oldDatasetIds { + if datasetId != tempDatasetId { + newDatasetIds = append(newDatasetIds, datasetId) + } + } + err := models.NewDatasetIdsByRepoID(repoID, newDatasetIds) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.cancel_reference_dataset_fail")) } @@ -284,29 +293,7 @@ func ReferenceDatasetDelete(ctx *context.Context) { func ReferenceDatasetPost(ctx *context.Context, form auth.ReferenceDatasetForm) { repoID := ctx.Repo.Repository.ID - datasetsID := models.GetDatasetIdsByRepoID(repoID) - - var newDataset []int64 - var deletedDataset []int64 - - for _, id := range datasetsID { - if !contains(form.DatasetID, id) { - deletedDataset = append(deletedDataset, id) - } - } - - for _, id := range form.DatasetID { - if !contains(datasetsID, id) { - newDataset = append(newDataset, id) - } - } - - err := models.DeleteDatasetIdsByRepoID(repoID, deletedDataset) - - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.reference_dataset_fail"))) - } - err = models.NewDatasetIdsByRepoID(repoID, newDataset) + err := models.NewDatasetIdsByRepoID(repoID, form.DatasetID) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.reference_dataset_fail")) } @@ -315,15 +302,6 @@ func ReferenceDatasetPost(ctx *context.Context, form auth.ReferenceDatasetForm) } -func contains(s []int64, e int64) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) { ctx.Data["PageIsDataset"] = true @@ -467,13 +445,23 @@ func MyDatasets(ctx *context.Context) { } func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { - datasets, count, err := datasetMultipleGet(ctx, opts) + page := ctx.QueryInt("page") + keyword := strings.Trim(ctx.Query("q"), " ") + opts.Keyword = keyword + if opts.SearchOrderBy.String() == "" { + opts.SearchOrderBy = models.SearchOrderByRecentUpdated + } - datasetMultipleResult(ctx, err, datasets, count) + opts.RecommendOnly = ctx.QueryBool("recommend") + opts.ListOptions = models.ListOptions{ + Page: page, + PageSize: setting.UI.DatasetPagingNum, + } + opts.JustNeedZipFile = true + opts.User = ctx.User -} + datasets, count, err := models.SearchDataset(opts) -func datasetMultipleResult(ctx *context.Context, err error, datasets models.DatasetList, count int64) { if err != nil { ctx.ServerError("datasets", err) return @@ -496,52 +484,31 @@ func datasetMultipleResult(ctx *context.Context, err error, datasets models.Data }) } -func datasetMultipleGet(ctx *context.Context, opts *models.SearchDatasetOptions) (models.DatasetList, int64, error) { - page := ctx.QueryInt("page") - keyword := strings.Trim(ctx.Query("q"), " ") - orderBy := models.SearchOrderByRecentUpdated - opts.Keyword = keyword - opts.SearchOrderBy = orderBy - opts.RecommendOnly = ctx.QueryBool("recommend") - opts.ListOptions = models.ListOptions{ - Page: page, - PageSize: setting.UI.DatasetPagingNum, - } - opts.JustNeedZipFile = true - opts.User = ctx.User - - datasets, count, err := models.SearchDataset(opts) - return datasets, count, err -} - func CurrentRepoDatasetMultiple(ctx *context.Context) { - + datasetIds := models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID) + searchOrderBy := getSearchOrderByInValues(datasetIds) opts := &models.SearchDatasetOptions{ RepoID: ctx.Repo.Repository.ID, NeedAttachment: true, CloudBrainType: ctx.QueryInt("type"), - DatasetIDs: models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID), + DatasetIDs: datasetIds, + SearchOrderBy: searchOrderBy, } - datasetList, count, err := datasetMultipleGet(ctx, opts) - if len(datasetList) > 0 { - var convertDatasetList models.DatasetList + datasetMultiple(ctx, opts) - for _, dataset := range datasetList { - if dataset.RepoID == ctx.Repo.Repository.ID && len(convertDatasetList) == 0 { - convertDatasetList = append(convertDatasetList, dataset) - } - } - for _, dataset := range datasetList { - if dataset.RepoID != ctx.Repo.Repository.ID { - convertDatasetList = append(convertDatasetList, dataset) - } - } - datasetMultipleResult(ctx, err, convertDatasetList, count) - } else { - datasetMultipleResult(ctx, err, datasetList, count) - } +} +func getSearchOrderByInValues(datasetIds []int64) models.SearchOrderBy { + if len(datasetIds) == 0 { + return "" + } + searchOrderBy := "CASE id " + for i, id := range datasetIds { + searchOrderBy += fmt.Sprintf(" WHEN %d THEN %d", id, i+1) + } + searchOrderBy += " ELSE 0 END" + return models.SearchOrderBy(searchOrderBy) } func MyDatasetsMultiple(ctx *context.Context) { @@ -589,17 +556,18 @@ func MyFavoriteDatasetMultiple(ctx *context.Context) { } func ReferenceDataset(ctx *context.Context) { MustEnableDataset(ctx) + datasetIds := models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID) opts := &models.SearchDatasetOptions{ - DatasetIDs: models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID), + DatasetIDs: datasetIds, NeedAttachment: false, CloudBrainType: models.TypeCloudBrainAll, ListOptions: models.ListOptions{ Page: 1, PageSize: setting.RepoMaxReferenceDatasetNum, }, - SearchOrderBy: models.SearchOrderByRecentUpdated, + SearchOrderBy: getSearchOrderByInValues(datasetIds), } - datasets, count, err := models.SearchDataset(opts) + datasets, _, err := models.SearchDataset(opts) if err != nil { ctx.ServerError("SearchDatasets", err) @@ -608,7 +576,7 @@ func ReferenceDataset(ctx *context.Context) { ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets) ctx.Data["PageIsDataset"] = true - ctx.Data["Total"] = count + ctx.Data["MaxReferenceDatasetNum"] = setting.RepoMaxReferenceDatasetNum ctx.HTML(200, tplReference) } From 2a8c5f901bb4544e08d78a1cc66b5affdd1e21a7 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 18 Aug 2022 10:21:36 +0800 Subject: [PATCH 03/29] fix-2430 --- models/cloudbrain.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 62bae29e2..046137fa5 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -8,13 +8,14 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/util" + "xorm.io/builder" "xorm.io/xorm" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) type CloudbrainStatus string @@ -142,7 +143,7 @@ type Cloudbrain struct { VersionID int64 //版本id VersionName string `xorm:"INDEX"` //当前版本 Uuid string //数据集id - DatasetName string + DatasetName string `xorm:"varchar(2000)"` VersionCount int //任务的当前版本数量，不包括删除的 IsLatestVersion string //是否是最新版本，1是，0否 CommitID string //提交的仓库代码id From 36cb70f51979f2c2c65053aea087aace5d969227 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 18 Aug 2022 15:31:36 +0800 Subject: [PATCH 04/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/dataset.go | 13 +++++++++---- routers/routes/routes.go | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index c058c4e9c..51cb4893f 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -556,6 +556,14 @@ func MyFavoriteDatasetMultiple(ctx *context.Context) { } func ReferenceDataset(ctx *context.Context) { MustEnableDataset(ctx) + ctx.Data["PageIsDataset"] = true + ctx.Data["MaxReferenceDatasetNum"] = setting.RepoMaxReferenceDatasetNum + ctx.HTML(200, tplReference) + +} + +func ReferenceDatasetData(ctx *context.Context) { + MustEnableDataset(ctx) datasetIds := models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID) opts := &models.SearchDatasetOptions{ DatasetIDs: datasetIds, @@ -574,10 +582,7 @@ func ReferenceDataset(ctx *context.Context) { return } - ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets) - ctx.Data["PageIsDataset"] = true - ctx.Data["MaxReferenceDatasetNum"] = setting.RepoMaxReferenceDatasetNum - ctx.HTML(200, tplReference) + ctx.JSON(http.StatusOK, repository.ConvertToDatasetWithStar(ctx, datasets)) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index d54606581..70bf1f181 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1032,7 +1032,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/datasets", func() { m.Get("", reqRepoDatasetReader, repo.DatasetIndex) - m.Get("/reference_datasets", repo.ReferenceDataset) + m.Get("/reference_datasets", reqRepoDatasetReader, repo.ReferenceDataset) + m.Get("/reference_datasets_data", reqRepoDatasetReader, repo.ReferenceDatasetData) m.Delete("/reference_datasets/:id", repo.ReferenceDatasetDelete) m.Put("/:id/:action", reqRepoDatasetReader, repo.DatasetAction) m.Get("/create", reqRepoDatasetWriter, repo.CreateDataset) From 548ffc82969ba15d14298d02ce5e0c7ef6bda8b6 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Fri, 19 Aug 2022 11:01:06 +0800 Subject: [PATCH 05/29] fix issue --- templates/repo/datasets/index.tmpl | 144 ++++---- templates/repo/datasets/reference.tmpl | 8 + templates/repo/header.tmpl | 2 +- web_src/js/components/dataset/referenceDataset.vue | 368 +++++++++++++++++++++ web_src/js/index.js | 13 + 5 files changed, 473 insertions(+), 62 deletions(-) create mode 100644 templates/repo/datasets/reference.tmpl create mode 100644 web_src/js/components/dataset/referenceDataset.vue diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index 22ee1f4c6..3dc12cf12 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -20,7 +20,6 @@ .wrapper { display: flex; overflow: hidden; - padding: 0 1rem; } .exp { @@ -40,7 +39,8 @@ } .exp:checked+.text .btn::after { - content:'{{$.i18n.Tr "org.fold"}}' + content:'{{$.i18n.Tr "org.fold"}}'; + color: #3291f8; } .wrapper>.text { @@ -80,7 +80,7 @@ margin-left: 20px; font-size: 14px; padding: 0 8px; - background: #3F51B5; + background-color: transparent; line-height: 20px; border-radius: 4px; color: #fff; @@ -89,7 +89,8 @@ } .btn::after { - content:'{{$.i18n.Tr "org.unfold"}}' + content:'{{$.i18n.Tr "org.unfold"}}'; + color: #3291f8; } .btn::before { @@ -132,6 +133,15 @@ border: 5px solid transparent; border-top-color: #c0c4cc; } + .dataset-flavor-button{ + display: flex; + align-items: center; + padding: 0.5rem; + border: 1px solid rgba(34,36,38,0.15); + border-top-right-radius: 0; + border-bottom-right-radius: 0; + box-shadow: none + }
{{template "repo/header" .}} @@ -142,44 +152,59 @@
{{end}}
-
-
-

{{.dataset.Title}}

+
+ 当前数据集 + 关联数据集
-
- - {{if $.IsSigned}} -
- - - - -
- ${num_stars} - {{else}} -
- - - - -
- ${num_stars} - {{end}} - {{.i18n.Tr "repo.modelarts.modify"}} - +
+
+

{{.dataset.Title}}

+ +
+ {{if $.IsSigned}} + + + {{else}} + + + {{end}}
+ + ${num_stars} + + {{.i18n.Tr "repo.modelarts.modify"}} +
+ {{if .dataset.Description}} +
+
+ +
+ + {{.dataset.Description}} +
+
+
+ {{end}} +
{{if or (.dataset.Category) (.dataset.Task) (.dataset.License)}} -
+
{{if .dataset.Category}} {{$category := .dataset.Category}} {{end}} -
- {{if .dataset.Description}} -
-
- -
- - {{.dataset.Description}} -
-
-
- {{end}} -
-
-
+
@@ -222,7 +233,7 @@ @click="gotoUpload('{{.RepoLink}}',{{.dataset.ID}})">{{$.i18n.Tr "dataset.dataset_upload"}}
-
+
@@ -387,16 +398,27 @@
{{else}} -
-
-
{{.i18n.Tr "dataset.dataset_no_create"}}
- {{if $.CanWrite}} - {{.i18n.Tr "dataset.create_new_dataset"}} - {{end}} -
-
{{.i18n.Tr "dataset.dataset_explain"}}
-
{{.i18n.Tr "dataset.dataset_instructions_for_use"}}{{.i18n.Tr "dataset.dataset_camp_course"}}
+
+
+
+
+ 当前数据集 + 关联数据集 +
+ {{if $.CanWrite}} + 创建数据集 + {{end}} +
+
+ +
+
+
{{.i18n.Tr "dataset.dataset_no_create"}}
+
+
{{.i18n.Tr "dataset.dataset_explain"}}
+
{{.i18n.Tr "dataset.dataset_instructions_for_use"}}{{.i18n.Tr "dataset.dataset_camp_course"}}
+
{{end}} diff --git a/templates/repo/datasets/reference.tmpl b/templates/repo/datasets/reference.tmpl new file mode 100644 index 000000000..e51989228 --- /dev/null +++ b/templates/repo/datasets/reference.tmpl @@ -0,0 +1,8 @@ +{{template "base/head" .}} + +
+ {{template "repo/header" .}} +
+
+
+{{template "base/footer" .}} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index b36e89d4d..9a4a65eac 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -138,7 +138,7 @@ {{end}} {{if .Permission.CanRead $.UnitTypeDatasets}} - + {{.i18n.Tr "datasets"}} diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue new file mode 100644 index 000000000..19f08ff0e --- /dev/null +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -0,0 +1,368 @@ + + + + + diff --git a/web_src/js/index.js b/web_src/js/index.js index b6872eaa7..2dd815715 100755 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -46,6 +46,7 @@ import initCloudrain from "./features/cloudrbanin.js"; import initCloudrainSow from "./features/cloudbrainShow.js"; import initImage from "./features/images.js"; import selectDataset from "./components/dataset/selectDataset.vue"; +import referenceDataset from "./components/dataset/referenceDataset.vue"; // import $ from 'jquery.js' import router from "./router/index.js"; import { Message } from "element-ui"; @@ -2900,6 +2901,7 @@ $(document).ready(async () => { initVueDataAnalysis(); initVueWxAutorize(); initVueselectDataset(); + initVuereferenceDataset(); initTeamSettings(); initCtrlEnterSubmit(); initNavbarContentToggle(); @@ -4532,6 +4534,17 @@ function initVueselectDataset() { render: (h) => h(selectDataset), }); } +function initVuereferenceDataset() { + const el = document.getElementById("reference-dataset"); + console.log("-=-==", el); + if (!el) { + return; + } + new Vue({ + el: el, + render: (h) => h(referenceDataset), + }); +} window.timeAddManual = function () { $(".mini.modal") .modal({ From 5464921cc104f5d6592ab34ede6ef2fbe2731697 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Fri, 19 Aug 2022 18:06:07 +0800 Subject: [PATCH 06/29] fix issue --- web_src/js/components/dataset/referenceDataset.vue | 139 ++++++++++++++++++--- 1 file changed, 125 insertions(+), 14 deletions(-) diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue index 19f08ff0e..45f0e5c0e 100644 --- a/web_src/js/components/dataset/referenceDataset.vue +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -13,7 +13,7 @@ >
+
@@ -219,20 +224,80 @@ - - - - - - + +
+
+ + {{ item.Title }} + + asdasdasdasd +
+
asdasdsa233333
+
+ +
+ 已选数据文件 +
+
+ + + +
+
+ 确定 +
+
@@ -255,6 +320,7 @@ export default { publicDatasetList: [], showFlag: true, search: "", + selectDatasetArray: [], }; }, methods: { @@ -263,6 +329,32 @@ export default { this.dialogVisible = true; }, searchName() {}, + cancelReferData(id) { + let url = `${this.repoLink}/datasets/reference_datasets/${id}`; + this.$axios.delete(url).then((res) => { + console.log(res); + if (res.data.Code === 0) { + console.log("delete success"); + let index = this.datasetList.find((item) => { + return item.ID === id; + }); + this.datasetList.splice(index, 1); + } + }); + }, + confirmDataset() {}, + changeCheckbox(checked, item) { + console.log(checked, item, this.checkList); + if (checked) { + this.selectDatasetArray.push({ ID: item.ID, Title: item.Title }); + } else { + let index = this.selectDatasetArray.findIndex((element) => { + return element.ID === item.ID; + }); + this.selectDatasetArray.splice(index, 1); + } + console.log(this.selectDatasetArray); + }, getDatasetList() { let url = `${this.repoLink}/datasets/reference_datasets_data`; this.$axios.get(url).then((res) => { @@ -317,9 +409,6 @@ export default { height: 30px; top: 60px; } -.checkbox-list { - display: flex; -} .refer-dataset-card { cursor: pointer; box-shadow: 0px 4px 4px 0px rgba(232, 232, 232, 0.6); @@ -365,4 +454,26 @@ export default { font-size: 14px; margin-top: 10px; } +.select-data-wrap { + padding: 1rem 0; + border-bottom: 1px solid rgba(0, 0, 0, 0.05); +} +.select-data-title { + flex: 1; + overflow: hidden; +} +.select-data-title .ref-data-title { + font-size: 18px; + color: #454545; + font-weight: 700; + overflow: hidden; + text-overflow: ellipsis; +} +.select-data-href { + text-align: right; + text-overflow: ellipsis; + max-width: 35%; + word-break: initial; + margin-left: 1rem; +} From 1ce1c1f10c4e1197e67b059ed3b49d9a3db8dc4d Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 22 Aug 2022 10:16:38 +0800 Subject: [PATCH 07/29] fix issue --- web_src/js/components/dataset/referenceDataset.vue | 44 ++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue index 45f0e5c0e..a871bb9aa 100644 --- a/web_src/js/components/dataset/referenceDataset.vue +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -237,6 +237,7 @@ {{ item.Title }} @@ -281,8 +282,11 @@ v-for="(item, index) in selectDatasetArray" :key="index" :label="item.ID" - :title="item.Name" - > + :title="item.Title" + @change="(checked) => changeCheckSelected(checked, item)" + style="display: flex; margin: 0.5rem 0" + >{{ item.Title }}
@@ -342,7 +346,10 @@ export default { } }); }, - confirmDataset() {}, + confirmDataset() { + this.submitReferDataset(); + this.dialogVisible = false; + }, changeCheckbox(checked, item) { console.log(checked, item, this.checkList); if (checked) { @@ -355,6 +362,16 @@ export default { } console.log(this.selectDatasetArray); }, + changeCheckSelected(checked, item) { + console.log(checked, item); + if (!checked) { + let index = this.selectDatasetArray.findIndex((element) => { + return element.ID === item.ID; + }); + this.selectDatasetArray.splice(index, 1); + } + console.log(this.checkList, this.selectDatasetArray); + }, getDatasetList() { let url = `${this.repoLink}/datasets/reference_datasets_data`; this.$axios.get(url).then((res) => { @@ -373,7 +390,18 @@ export default { console.log("this.publicDatasetList", this.publicDatasetList); }); }, + submitReferDataset() { + let url = `${this.repoLink}/datasets/reference_datasets_available`; + let data = this.qs.stringify({ + _csrf: csrf, + dataset_id: this.checkList, + }); + this.$axios.post(url, data).then((res) => { + console.log("post success"); + }); + }, }, + filters: { transformTimestamp(timestamp) { const date = new Date(parseInt(timestamp) * 1000); @@ -476,4 +504,14 @@ export default { word-break: initial; margin-left: 1rem; } +/deep/ .el-checkbox-group .el-checkbox .el-checkbox__label { + max-width: 100%; +} +.select-data-right { + overflow: hidden; + vertical-align: middle; + text-overflow: ellipsis; + max-width: 100%; + display: inline-block; +} From 6af10b01fabcd3bbf7ddd52f9c8f894508d8ec1e Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 22 Aug 2022 11:04:59 +0800 Subject: [PATCH 08/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/dataset.go | 3 ++- routers/repo/dataset.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 04591c64e..6e34ee093 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -190,6 +190,7 @@ type SearchDatasetOptions struct { JustNeedZipFile bool NeedAttachment bool UploadAttachmentByMe bool + QueryReference bool } func CreateDataset(dataset *Dataset) (err error) { @@ -260,7 +261,7 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { } } if len(opts.DatasetIDs) > 0 { - if opts.StarByMe { + if opts.StarByMe || (opts.RepoID == 0 && opts.QueryReference) { cond = cond.And(builder.In("dataset.id", opts.DatasetIDs)) } else { subCon := builder.NewCond() diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 51cb4893f..cbfed4666 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -573,7 +573,8 @@ func ReferenceDatasetData(ctx *context.Context) { Page: 1, PageSize: setting.RepoMaxReferenceDatasetNum, }, - SearchOrderBy: getSearchOrderByInValues(datasetIds), + SearchOrderBy: getSearchOrderByInValues(datasetIds), + QueryReference: true, } datasets, _, err := models.SearchDataset(opts) From 89df1f1ec20f20fac1aa34b1a481219979babe72 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 22 Aug 2022 15:16:44 +0800 Subject: [PATCH 09/29] fix issue --- web_src/js/components/dataset/referenceDataset.vue | 179 +++++++++++++++++---- web_src/js/features/datsetCate.js | 99 ++++++++++++ web_src/js/index.js | 3 + 3 files changed, 249 insertions(+), 32 deletions(-) create mode 100644 web_src/js/features/datsetCate.js diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue index a871bb9aa..8fa1548df 100644 --- a/web_src/js/components/dataset/referenceDataset.vue +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -5,11 +5,13 @@
- 当前数据集 + {{ + i18n.current_dataset + }} 关联数据集{{ i18n.linked_datasets }}
@@ -26,6 +28,7 @@ class="ui card refer-dataset-card" v-for="(item, index) in datasetList" :key="index" + @click="gotoDataset(item)" >
@@ -45,9 +48,12 @@ @@ -164,15 +173,17 @@
- 当前数据集 + {{ + i18n.current_dataset + }} 关联数据集{{ i18n.linked_datasets }}
@@ -195,7 +206,7 @@ @@ -207,7 +218,7 @@ > @@ -222,7 +233,7 @@ " > - + - asdasdasdasd{{ item.Repo.OwnerName }}/{{ item.Repo.Alias }}
-
asdasdsa233333
+
+ {{ item.Description }} +
+
+ + +
@@ -274,7 +305,7 @@ line-height: 40px; " > - 已选数据文件 + {{ i18n.selected_data_file }}
@@ -285,7 +316,9 @@ :title="item.Title" @change="(checked) => changeCheckSelected(checked, item)" style="display: flex; margin: 0.5rem 0" - >{{ item.Title }}{{ + item.Title + }}
@@ -298,7 +331,7 @@ color: #fff; border: 1px solid #389e0d; " - >确定{{ i18n.sure }}
@@ -321,10 +354,17 @@ export default { datasetList: [], test: false, checkList: [], + tableTotalList: [], + filterDatasetList: [], publicDatasetList: [], showFlag: true, search: "", selectDatasetArray: [], + + i18n: {}, + + totalNum: 0, + currentPage: 1, }; }, methods: { @@ -332,7 +372,27 @@ export default { console.log("open"); this.dialogVisible = true; }, - searchName() {}, + gotoDataset(item) { + console.log(item); + location.href = `/${item.Repo.OwnerName}/${item.Repo.Name}/datasets`; + }, + currentChange(page) { + this.currentPage = page; + this.publicDatasetList = this.filterDatasetList.slice( + (this.currentPage - 1) * 5, + this.currentPage * 5 + ); + }, + searchName() { + this.filterDatasetList = this.tableTotalList.filter((item) => { + let title = item.Title.toLowerCase(); + let search = this.search.toLowerCase(); + let desc = item.Description.toLowerCase(); + return title.indexOf(search) !== -1 || desc.indexOf(search) !== -1; + }); + this.totalNum = this.filterDatasetList.length; + this.currentChange(1); + }, cancelReferData(id) { let url = `${this.repoLink}/datasets/reference_datasets/${id}`; this.$axios.delete(url).then((res) => { @@ -372,32 +432,63 @@ export default { } console.log(this.checkList, this.selectDatasetArray); }, - getDatasetList() { - let url = `${this.repoLink}/datasets/reference_datasets_data`; + postStar(item) { + console.log(item); + if (item.IsStaring) { + let url = `${this.repoLink}/datasets/${item.ID}/unstar`; + this.$axios.put(url).then((res) => { + if (res.data.Code === 0) { + this.datasetList.forEach((element, i) => { + if (element.ID === item.ID) { + this.datasetList[i].NumStars -= 1; + this.datasetList[i].IsStaring = !this.datasetList[i].IsStaring; + } + }); + } + }); + } else { + let url = `${this.repoLink}/datasets/${item.ID}/star`; + this.$axios.put(url).then((res) => { + if (res.data.Code === 0) { + this.datasetList.forEach((element, i) => { + if (element.ID === item.ID) { + this.datasetList[i].NumStars += 1; + this.datasetList[i].IsStaring = !this.datasetList[i].IsStaring; + } + }); + } + }); + } + }, + + getSelectDatasetList() { + let url = `${this.repoLink}/datasets/reference_datasets_available`; this.$axios.get(url).then((res) => { - this.datasetList = res.data; + this.datasetList = JSON.parse(res.data.data); this.datasetList.length ? (this.showFlag = true) : (this.showFlag = false); console.log("this.getDatasetList:", this.datasetList); }); }, - getSelectDatasetList() { - let url = `${this.repoLink}/datasets/reference_datasets_available`; + getDatasetList() { + let url = `${this.repoLink}/datasets/reference_datasets_data`; this.$axios.get(url).then((res) => { - this.publicDatasetList = JSON.parse(res.data.data); - - console.log("this.publicDatasetList", this.publicDatasetList); + this.tableTotalList = res.data; + this.search = ""; + this.searchName(); }); }, submitReferDataset() { - let url = `${this.repoLink}/datasets/reference_datasets_available`; + let url = `${this.repoLink}/datasets/reference_datasets`; let data = this.qs.stringify({ _csrf: csrf, dataset_id: this.checkList, }); this.$axios.post(url, data).then((res) => { - console.log("post success"); + if (res.data.Code === 0) { + this.getSelectDatasetList(); + } }); }, }, @@ -417,13 +508,24 @@ export default { return dateString; }, }, - watch: {}, + watch: { + search(val) { + if (!val) { + this.searchName(); + } + }, + }, mounted() { this.getDatasetList(); this.getSelectDatasetList(); }, created() { this.repoLink = $(".reference-dataset").data("repolink") || ""; + if (document.documentElement.attributes["lang"].nodeValue == "en-US") { + this.i18n = this.$locale.US; + } else { + this.i18n = this.$locale.CN; + } }, beforeDestroy() {}, }; @@ -472,6 +574,7 @@ export default { border: #888888; border-top-left-radius: 0 !important; border-bottom-left-radius: 0 !important; + cursor: default !important; } .card-flavor-desc { -webkit-box-orient: vertical; @@ -514,4 +617,16 @@ export default { max-width: 100%; display: inline-block; } +.data-multiple-wrap { + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; + display: -webkit-box; + max-width: 100%; + overflow: hidden; + padding-top: 1rem; + color: #888888; + font: 12px; + line-height: 20px; + margin-left: 2rem; +} diff --git a/web_src/js/features/datsetCate.js b/web_src/js/features/datsetCate.js new file mode 100644 index 000000000..d27d10f79 --- /dev/null +++ b/web_src/js/features/datsetCate.js @@ -0,0 +1,99 @@ +export const i18nVue = { + CN: { + computer_vision: "计算机视觉", + natural_language_processing: "自然语言处理", + speech_processing: "语音处理", + computer_vision_natural_language_processing: "计算机视觉、自然语言处理", + machine_translation: "机器翻译", + question_answering_system: "问答系统", + information_retrieval: "信息检索", + knowledge_graph: "知识图谱", + text_annotation: "文本标注", + text_categorization: "文本分类", + emotion_analysis: "情感分析", + language_modeling: "语言建模", + speech_recognition: "语音识别", + automatic_digest: "自动文摘", + information_extraction: "信息抽取", + description_generation: "说明生成", + image_classification: "图像分类", + face_recognition: "人脸识别", + image_search: "图像搜索", + target_detection: "目标检测", + image_description_generation: "图像描述生成", + vehicle_license_plate_recognition: "车辆车牌识别", + medical_image_analysis: "医学图像分析", + unmanned: "无人驾驶", + unmanned_security: "无人安防", + drone: "无人机", + vr_ar: "VR/AR", + "2_d_vision": "2-D视觉", + "2_5_d_vision": "2.5-D视觉", + "3_d_reconstruction": "3D重构", + image_processing: "图像处理", + video_processing: "视频处理", + visual_input_system: "视觉输入系统", + speech_coding: "语音编码", + speech_enhancement: "语音增强", + speech_recognition: "语音识别", + speech_synthesis: "语音合成", + current_dataset: "当前数据集", + linked_datasets: "关联数据集", + unfavorite: "取消收藏", + favorite: "收藏", + disassociate: "取消关联", + public_dataset: "公开数据集", + selected_data_file: "已选数据文件", + sure: "确定", + search_dataset: "搜数据集名称/描述...", + }, + US: { + computer_vision: "computer vision", + natural_language_processing: "natural language processing", + speech_processing: "speech processing", + computer_vision_natural_language_processing: + "computer vision and natural language processing", + machine_translation: "machine translation", + question_answering_system: "question answering system", + information_retrieval: "information retrieval", + knowledge_graph: "knowledge graph", + text_annotation: "text annotation", + text_categorization: "text categorization", + emotion_analysis: "emotion analysis", + language_modeling: "language modeling", + speech_recognition: "speech recognition", + automatic_digest: "automatic digest", + information_extraction: "information extraction", + description_generation: "description generation", + image_classification: "image classification", + face_recognition: "face recognition", + image_search: "image search", + target_detection: "target detection", + image_description_generation: "image description generation", + vehicle_license_plate_recognition: "vehicle license plate recognition", + medical_image_analysis: "medical image analysis", + unmanned: "unmanned", + unmanned_security: "unmanned security", + drone: "drone", + vr_ar: "VR/AR", + "2_d_vision": "2.D vision", + "2.5_d_vision": "2.5D vision", + "3_d_reconstruction": "3Dreconstruction", + image_processing: "image processing", + video_processing: "video processing", + visual_input_system: "visual input system", + speech_coding: "speech coding", + speech_enhancement: "speech enhancement", + speech_recognition: "speech recognition", + speech_synthesis: "speech synthesis", + current_dataset: "Current Dataset", + linked_datasets: "Linked Datasets", + unfavorite: "Unfavorite", + favorite: "Favorite", + disassociate: "Disassociate", + public_dataset: "Public Dataset", + selected_data_file: "Selected Data File", + sure: "Sure", + search_dataset: "Search dataset name/description ...", + }, +}; diff --git a/web_src/js/index.js b/web_src/js/index.js index 2dd815715..0d7457788 100755 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -51,11 +51,14 @@ import referenceDataset from "./components/dataset/referenceDataset.vue"; import router from "./router/index.js"; import { Message } from "element-ui"; +import { i18nVue } from "./features/datsetCate.js"; + Vue.use(ElementUI); Vue.prototype.$axios = axios; Vue.prototype.$Cookies = Cookies; Vue.prototype.qs = qs; Vue.prototype.$message = Message; +Vue.prototype.$locale = i18nVue; const { AppSubUrl, StaticUrlPrefix, csrf } = window.config; Object.defineProperty(Vue.prototype, "$echarts", { From a36449e3fe46c98be59e2731c500351bf98c3436 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 22 Aug 2022 15:29:52 +0800 Subject: [PATCH 10/29] fix issue --- web_src/js/components/dataset/referenceDataset.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue index 8fa1548df..d6ffbfaba 100644 --- a/web_src/js/components/dataset/referenceDataset.vue +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -371,6 +371,7 @@ export default { openDataset() { console.log("open"); this.dialogVisible = true; + this.getDatasetList(); }, gotoDataset(item) { console.log(item); @@ -516,7 +517,6 @@ export default { }, }, mounted() { - this.getDatasetList(); this.getSelectDatasetList(); }, created() { From 595d242ebbc6855fca1219e3ae19ebf31cbd504c Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 22 Aug 2022 15:52:58 +0800 Subject: [PATCH 11/29] fix issue --- web_src/js/components/dataset/referenceDataset.vue | 42 ++++++++++------------ 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue index d6ffbfaba..9cb356272 100644 --- a/web_src/js/components/dataset/referenceDataset.vue +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -354,12 +354,11 @@ export default { datasetList: [], test: false, checkList: [], - tableTotalList: [], - filterDatasetList: [], publicDatasetList: [], showFlag: true, search: "", selectDatasetArray: [], + paramsPublics: { page: 1, q: "" }, i18n: {}, @@ -378,21 +377,13 @@ export default { location.href = `/${item.Repo.OwnerName}/${item.Repo.Name}/datasets`; }, currentChange(page) { - this.currentPage = page; - this.publicDatasetList = this.filterDatasetList.slice( - (this.currentPage - 1) * 5, - this.currentPage * 5 - ); + this.paramsPublics.page = page; + this.getDatasetList(); }, searchName() { - this.filterDatasetList = this.tableTotalList.filter((item) => { - let title = item.Title.toLowerCase(); - let search = this.search.toLowerCase(); - let desc = item.Description.toLowerCase(); - return title.indexOf(search) !== -1 || desc.indexOf(search) !== -1; - }); - this.totalNum = this.filterDatasetList.length; - this.currentChange(1); + this.paramsPublics.q = this.search; + this.paramsPublics.page = 1; + this.getDatasetList(); }, cancelReferData(id) { let url = `${this.repoLink}/datasets/reference_datasets/${id}`; @@ -463,9 +454,9 @@ export default { }, getSelectDatasetList() { - let url = `${this.repoLink}/datasets/reference_datasets_available`; + let url = `${this.repoLink}/datasets/reference_datasets_data`; this.$axios.get(url).then((res) => { - this.datasetList = JSON.parse(res.data.data); + this.datasetList = res.data; this.datasetList.length ? (this.showFlag = true) : (this.showFlag = false); @@ -473,12 +464,17 @@ export default { }); }, getDatasetList() { - let url = `${this.repoLink}/datasets/reference_datasets_data`; - this.$axios.get(url).then((res) => { - this.tableTotalList = res.data; - this.search = ""; - this.searchName(); - }); + let url = `${this.repoLink}/datasets/reference_datasets_available`; + this.$axios + .get(url, { + params: this.paramsPublics, + }) + .then((res) => { + this.publicDatasetList = JSON.parse(res.data.data); + this.totalNum = parseInt(res.data.count); + // this.search = ""; + // this.searchName(); + }); }, submitReferDataset() { let url = `${this.repoLink}/datasets/reference_datasets`; From c6986b507c836cb5261667b356f664e0163fc161 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 22 Aug 2022 16:04:04 +0800 Subject: [PATCH 12/29] fix issue --- web_src/js/components/dataset/referenceDataset.vue | 305 +++++++++++---------- 1 file changed, 158 insertions(+), 147 deletions(-) diff --git a/web_src/js/components/dataset/referenceDataset.vue b/web_src/js/components/dataset/referenceDataset.vue index 9cb356272..2c533db95 100644 --- a/web_src/js/components/dataset/referenceDataset.vue +++ b/web_src/js/components/dataset/referenceDataset.vue @@ -1,165 +1,170 @@