You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

dataset.go 22 kB

3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
2 years ago
3 years ago
3 years ago
3 years ago
2 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
2 years ago
3 years ago
2 years ago
2 years ago
2 years ago
3 years ago
2 years ago
3 years ago
2 years ago
3 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
3 years ago
2 years ago
3 years ago
2 years ago
3 years ago
2 years ago
3 years ago
2 years ago
3 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843
  1. package repo
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "net/http"
  6. "sort"
  7. "strconv"
  8. "strings"
  9. "unicode/utf8"
  10. "code.gitea.io/gitea/services/repository"
  11. "code.gitea.io/gitea/models"
  12. "code.gitea.io/gitea/modules/auth"
  13. "code.gitea.io/gitea/modules/base"
  14. "code.gitea.io/gitea/modules/context"
  15. "code.gitea.io/gitea/modules/log"
  16. "code.gitea.io/gitea/modules/setting"
  17. )
  18. const (
  19. tplIndex base.TplName = "repo/datasets/index"
  20. tplDatasetCreate base.TplName = "repo/datasets/create"
  21. tplDatasetEdit base.TplName = "repo/datasets/edit"
  22. taskstplIndex base.TplName = "repo/datasets/tasks/index"
  23. tplReference base.TplName = "repo/datasets/reference"
  24. )
  25. // MustEnableDataset check if repository enable internal dataset
  26. func MustEnableDataset(ctx *context.Context) {
  27. if !ctx.Repo.CanRead(models.UnitTypeDatasets) {
  28. ctx.NotFound("MustEnableDataset", nil)
  29. return
  30. }
  31. }
  32. func newFilterPrivateAttachments(ctx *context.Context, list []*models.Attachment, repo *models.Repository) []*models.Attachment {
  33. if ctx.Repo.CanWrite(models.UnitTypeDatasets) {
  34. log.Info("can write.")
  35. return list
  36. } else {
  37. if repo.Owner == nil {
  38. repo.GetOwner()
  39. }
  40. permission := false
  41. if repo.Owner.IsOrganization() && ctx.User != nil {
  42. if repo.Owner.IsUserPartOfOrg(ctx.User.ID) {
  43. log.Info("user is member of org.")
  44. permission = true
  45. }
  46. }
  47. if !permission && ctx.User != nil {
  48. isCollaborator, _ := repo.IsCollaborator(ctx.User.ID)
  49. if isCollaborator {
  50. log.Info("Collaborator user may visit the attach.")
  51. permission = true
  52. }
  53. }
  54. var publicList []*models.Attachment
  55. for _, attach := range list {
  56. if !attach.IsPrivate {
  57. publicList = append(publicList, attach)
  58. } else {
  59. if permission {
  60. publicList = append(publicList, attach)
  61. }
  62. }
  63. }
  64. return publicList
  65. }
  66. }
  67. func QueryDataSet(ctx *context.Context) []*models.Attachment {
  68. repo := ctx.Repo.Repository
  69. dataset, err := models.GetDatasetByRepo(repo)
  70. if err != nil {
  71. log.Error("zou not found dataset 1")
  72. ctx.NotFound("GetDatasetByRepo", err)
  73. return nil
  74. }
  75. if ctx.Query("type") == "" {
  76. log.Error("zou not found type 2")
  77. ctx.NotFound("type error", nil)
  78. return nil
  79. }
  80. err = models.GetDatasetAttachments(ctx.QueryInt("type"), ctx.IsSigned, ctx.User, dataset)
  81. if err != nil {
  82. ctx.ServerError("GetDatasetAttachments", err)
  83. return nil
  84. }
  85. attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo)
  86. ctx.Data["SortType"] = ctx.Query("sort")
  87. sort.Slice(attachments, func(i, j int) bool {
  88. return attachments[i].CreatedUnix > attachments[j].CreatedUnix
  89. })
  90. return attachments
  91. }
  92. func DatasetIndex(ctx *context.Context) {
  93. log.Info("dataset index 1")
  94. MustEnableDataset(ctx)
  95. ctx.Data["PageIsDataset"] = true
  96. ctx.Data["SortType"] = ctx.Query("sort")
  97. repo := ctx.Repo.Repository
  98. dataset, err := models.GetDatasetByRepo(repo)
  99. ctx.Data["CanWrite"] = ctx.Repo.CanWrite(models.UnitTypeDatasets)
  100. if err != nil {
  101. log.Warn("query dataset, not found.")
  102. ctx.HTML(200, tplIndex)
  103. return
  104. }
  105. cloudbrainType := -1
  106. if ctx.Query("type") != "" {
  107. cloudbrainType = ctx.QueryInt("type")
  108. }
  109. err = models.GetDatasetAttachments(cloudbrainType, ctx.IsSigned, ctx.User, dataset)
  110. if err != nil {
  111. ctx.ServerError("GetDatasetAttachments", err)
  112. return
  113. }
  114. attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo)
  115. if ctx.Data["SortType"] == "nameAsc" {
  116. sort.Slice(attachments, func(i, j int) bool {
  117. return strings.ToLower(attachments[i].Name) < strings.ToLower(attachments[j].Name)
  118. })
  119. } else if ctx.Data["SortType"] == "nameDesc" {
  120. sort.Slice(attachments, func(i, j int) bool {
  121. return strings.ToLower(attachments[i].Name) > strings.ToLower(attachments[j].Name)
  122. })
  123. } else if ctx.Data["SortType"] == "sizeAsc" {
  124. sort.Slice(attachments, func(i, j int) bool {
  125. return attachments[i].Size < attachments[j].Size
  126. })
  127. } else if ctx.Data["SortType"] == "sizeDesc" {
  128. sort.Slice(attachments, func(i, j int) bool {
  129. return attachments[i].Size > attachments[j].Size
  130. })
  131. } else if ctx.Data["SortType"] == "timeAsc" {
  132. sort.Slice(attachments, func(i, j int) bool {
  133. return attachments[i].CreatedUnix < attachments[j].CreatedUnix
  134. })
  135. } else {
  136. sort.Slice(attachments, func(i, j int) bool {
  137. return attachments[i].CreatedUnix > attachments[j].CreatedUnix
  138. })
  139. }
  140. page := ctx.QueryInt("page")
  141. if page <= 0 {
  142. page = 1
  143. }
  144. pagesize := ctx.QueryInt("pagesize")
  145. if pagesize <= 0 {
  146. pagesize = 10
  147. }
  148. pager := context.NewPagination(len(attachments), pagesize, page, 5)
  149. pageAttachments := getPageAttachments(attachments, page, pagesize)
  150. //load attachment creator
  151. for _, attachment := range pageAttachments {
  152. uploader, _ := models.GetUserByID(attachment.UploaderID)
  153. attachment.Uploader = uploader
  154. if !strings.HasSuffix(attachment.Name, ".zip") && !strings.HasSuffix(attachment.Name, ".tar.gz") {
  155. attachment.DecompressState = -1 //非压缩文件
  156. }
  157. }
  158. ctx.Data["Page"] = pager
  159. ctx.Data["Title"] = ctx.Tr("dataset.show_dataset")
  160. ctx.Data["Link"] = ctx.Repo.RepoLink + "/datasets"
  161. ctx.Data["dataset"] = dataset
  162. ctx.Data["Attachments"] = pageAttachments
  163. ctx.Data["IsOwner"] = true
  164. ctx.Data["StoreType"] = setting.Attachment.StoreType
  165. ctx.Data["Type"] = cloudbrainType
  166. renderAttachmentSettings(ctx)
  167. ctx.HTML(200, tplIndex)
  168. }
  169. func getPageAttachments(attachments []*models.Attachment, page int, pagesize int) []*models.Attachment {
  170. begin := (page - 1) * pagesize
  171. end := (page) * pagesize
  172. if begin > len(attachments)-1 {
  173. return nil
  174. }
  175. if end > len(attachments)-1 {
  176. return attachments[begin:]
  177. } else {
  178. return attachments[begin:end]
  179. }
  180. }
  181. func CreateDataset(ctx *context.Context) {
  182. MustEnableDataset(ctx)
  183. ctx.Data["PageIsDataset"] = true
  184. ctx.HTML(200, tplDatasetCreate)
  185. }
  186. func EditDataset(ctx *context.Context) {
  187. MustEnableDataset(ctx)
  188. ctx.Data["PageIsDataset"] = true
  189. datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64)
  190. dataset, _ := models.GetDatasetByID(datasetId)
  191. if dataset == nil {
  192. ctx.Error(http.StatusNotFound, "")
  193. return
  194. }
  195. ctx.Data["Dataset"] = dataset
  196. ctx.HTML(200, tplDatasetEdit)
  197. }
  198. func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) {
  199. dataset := &models.Dataset{}
  200. if !NamePattern.MatchString(form.Title) {
  201. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
  202. return
  203. }
  204. if utf8.RuneCountInString(form.Description) > 1024 {
  205. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 1024)))
  206. return
  207. }
  208. dataset.RepoID = ctx.Repo.Repository.ID
  209. dataset.UserID = ctx.User.ID
  210. dataset.Category = form.Category
  211. dataset.Task = form.Task
  212. dataset.Title = form.Title
  213. dataset.License = form.License
  214. dataset.Description = form.Description
  215. dataset.DownloadTimes = 0
  216. if ctx.Repo.Repository.IsPrivate {
  217. dataset.Status = 0
  218. } else {
  219. dataset.Status = 1
  220. }
  221. err := models.CreateDataset(dataset)
  222. if err != nil {
  223. log.Error("fail to create dataset", err)
  224. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.create_dataset_fail")))
  225. } else {
  226. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  227. }
  228. }
  229. func ReferenceDatasetDelete(ctx *context.Context) {
  230. repoID := ctx.Repo.Repository.ID
  231. datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64)
  232. oldDatasetIds := models.GetDatasetIdsByRepoID(repoID)
  233. var newDatasetIds []int64
  234. for _, tempDatasetId := range oldDatasetIds {
  235. if datasetId != tempDatasetId {
  236. newDatasetIds = append(newDatasetIds, tempDatasetId)
  237. }
  238. }
  239. err := models.NewDatasetIdsByRepoID(repoID, newDatasetIds)
  240. if err != nil {
  241. ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.cancel_reference_dataset_fail"))
  242. }
  243. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  244. }
  245. func ReferenceDatasetPost(ctx *context.Context, form auth.ReferenceDatasetForm) {
  246. repoID := ctx.Repo.Repository.ID
  247. err := models.NewDatasetIdsByRepoID(repoID, form.DatasetID)
  248. if err != nil {
  249. ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.reference_dataset_fail"))
  250. }
  251. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  252. }
  253. func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) {
  254. ctx.Data["PageIsDataset"] = true
  255. ctx.Data["Title"] = ctx.Tr("dataset.edit_dataset")
  256. if !NamePattern.MatchString(form.Title) {
  257. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
  258. return
  259. }
  260. if utf8.RuneCountInString(form.Description) > 1024 {
  261. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 1024)))
  262. return
  263. }
  264. rel, err := models.GetDatasetByID(form.ID)
  265. ctx.Data["dataset"] = rel
  266. if err != nil {
  267. log.Error("failed to query dataset", err)
  268. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail")))
  269. return
  270. }
  271. rel.Title = form.Title
  272. rel.Description = form.Description
  273. rel.Category = form.Category
  274. rel.Task = form.Task
  275. rel.License = form.License
  276. if err = models.UpdateDataset(models.DefaultDBContext(), rel); err != nil {
  277. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail")))
  278. }
  279. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  280. }
  281. func DatasetAction(ctx *context.Context) {
  282. var err error
  283. datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64)
  284. switch ctx.Params(":action") {
  285. case "star":
  286. err = models.StarDataset(ctx.User.ID, datasetId, true)
  287. case "unstar":
  288. err = models.StarDataset(ctx.User.ID, datasetId, false)
  289. }
  290. if err != nil {
  291. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action"))))
  292. } else {
  293. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  294. }
  295. }
  296. func CurrentRepoDataset(ctx *context.Context) {
  297. page := ctx.QueryInt("page")
  298. cloudbrainType := ctx.QueryInt("type")
  299. keyword := strings.Trim(ctx.Query("q"), " ")
  300. repo := ctx.Repo.Repository
  301. var datasetIDs []int64
  302. dataset, err := models.GetDatasetByRepo(repo)
  303. if err != nil {
  304. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetByRepo failed", err)))
  305. return
  306. }
  307. datasetIDs = append(datasetIDs, dataset.ID)
  308. datasets, count, err := models.Attachments(&models.AttachmentsOptions{
  309. ListOptions: models.ListOptions{
  310. Page: page,
  311. PageSize: setting.UI.DatasetPagingNum,
  312. },
  313. Keyword: keyword,
  314. NeedDatasetIDs: true,
  315. DatasetIDs: datasetIDs,
  316. Type: cloudbrainType,
  317. NeedIsPrivate: false,
  318. JustNeedZipFile: true,
  319. NeedRepoInfo: true,
  320. })
  321. if err != nil {
  322. ctx.ServerError("datasets", err)
  323. return
  324. }
  325. data, err := json.Marshal(datasets)
  326. if err != nil {
  327. log.Error("json.Marshal failed:", err.Error())
  328. ctx.JSON(200, map[string]string{
  329. "result_code": "-1",
  330. "error_msg": err.Error(),
  331. "data": "",
  332. })
  333. return
  334. }
  335. ctx.JSON(200, map[string]string{
  336. "result_code": "0",
  337. "data": string(data),
  338. "count": strconv.FormatInt(count, 10),
  339. })
  340. }
  341. func MyDatasets(ctx *context.Context) {
  342. page := ctx.QueryInt("page")
  343. cloudbrainType := ctx.QueryInt("type")
  344. keyword := strings.Trim(ctx.Query("q"), " ")
  345. uploaderID := ctx.User.ID
  346. datasets, count, err := models.Attachments(&models.AttachmentsOptions{
  347. ListOptions: models.ListOptions{
  348. Page: page,
  349. PageSize: setting.UI.DatasetPagingNum,
  350. },
  351. Keyword: keyword,
  352. NeedDatasetIDs: false,
  353. UploaderID: uploaderID,
  354. Type: cloudbrainType,
  355. NeedIsPrivate: false,
  356. JustNeedZipFile: true,
  357. NeedRepoInfo: true,
  358. RecommendOnly: ctx.QueryBool("recommend"),
  359. })
  360. if err != nil {
  361. ctx.ServerError("datasets", err)
  362. return
  363. }
  364. data, err := json.Marshal(datasets)
  365. if err != nil {
  366. log.Error("json.Marshal failed:", err.Error())
  367. ctx.JSON(200, map[string]string{
  368. "result_code": "-1",
  369. "error_msg": err.Error(),
  370. "data": "",
  371. })
  372. return
  373. }
  374. ctx.JSON(200, map[string]string{
  375. "result_code": "0",
  376. "data": string(data),
  377. "count": strconv.FormatInt(count, 10),
  378. })
  379. }
  380. func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
  381. page := ctx.QueryInt("page")
  382. keyword := strings.Trim(ctx.Query("q"), " ")
  383. opts.Keyword = keyword
  384. if opts.SearchOrderBy.String() == "" {
  385. opts.SearchOrderBy = models.SearchOrderByRecentUpdated
  386. }
  387. opts.RecommendOnly = ctx.QueryBool("recommend")
  388. opts.ListOptions = models.ListOptions{
  389. Page: page,
  390. PageSize: setting.UI.DatasetPagingNum,
  391. }
  392. opts.JustNeedZipFile = true
  393. opts.User = ctx.User
  394. datasets, count, err := models.SearchDataset(opts)
  395. if err != nil {
  396. ctx.ServerError("datasets", err)
  397. return
  398. }
  399. data, err := json.Marshal(datasets)
  400. if err != nil {
  401. log.Error("json.Marshal failed:", err.Error())
  402. ctx.JSON(200, map[string]string{
  403. "result_code": "-1",
  404. "error_msg": err.Error(),
  405. "data": "",
  406. })
  407. return
  408. }
  409. ctx.JSON(200, map[string]string{
  410. "result_code": "0",
  411. "data": string(data),
  412. "count": strconv.FormatInt(count, 10),
  413. })
  414. }
  415. func CurrentRepoDatasetMultiple(ctx *context.Context) {
  416. datasetIds := models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID)
  417. searchOrderBy := getSearchOrderByInValues(datasetIds)
  418. opts := &models.SearchDatasetOptions{
  419. RepoID: ctx.Repo.Repository.ID,
  420. NeedAttachment: true,
  421. CloudBrainType: ctx.QueryInt("type"),
  422. DatasetIDs: datasetIds,
  423. SearchOrderBy: searchOrderBy,
  424. }
  425. datasetMultiple(ctx, opts)
  426. }
  427. func getSearchOrderByInValues(datasetIds []int64) models.SearchOrderBy {
  428. if len(datasetIds) == 0 {
  429. return ""
  430. }
  431. searchOrderBy := "CASE id "
  432. for i, id := range datasetIds {
  433. searchOrderBy += fmt.Sprintf(" WHEN %d THEN %d", id, i+1)
  434. }
  435. searchOrderBy += " ELSE 0 END"
  436. return models.SearchOrderBy(searchOrderBy)
  437. }
  438. func MyDatasetsMultiple(ctx *context.Context) {
  439. opts := &models.SearchDatasetOptions{
  440. UploadAttachmentByMe: true,
  441. NeedAttachment: true,
  442. CloudBrainType: ctx.QueryInt("type"),
  443. }
  444. datasetMultiple(ctx, opts)
  445. }
  446. func ReferenceDatasetAvailable(ctx *context.Context) {
  447. opts := &models.SearchDatasetOptions{
  448. PublicOnly: true,
  449. NeedAttachment: false,
  450. CloudBrainType: models.TypeCloudBrainAll,
  451. }
  452. datasetMultiple(ctx, opts)
  453. }
  454. func PublicDatasetMultiple(ctx *context.Context) {
  455. opts := &models.SearchDatasetOptions{
  456. PublicOnly: true,
  457. NeedAttachment: true,
  458. CloudBrainType: ctx.QueryInt("type"),
  459. }
  460. datasetMultiple(ctx, opts)
  461. }
  462. func MyFavoriteDatasetMultiple(ctx *context.Context) {
  463. opts := &models.SearchDatasetOptions{
  464. StarByMe: true,
  465. DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID),
  466. NeedAttachment: true,
  467. CloudBrainType: ctx.QueryInt("type"),
  468. }
  469. datasetMultiple(ctx, opts)
  470. }
  471. func ReferenceDataset(ctx *context.Context) {
  472. MustEnableDataset(ctx)
  473. ctx.Data["PageIsDataset"] = true
  474. ctx.Data["MaxReferenceDatasetNum"] = setting.RepoMaxReferenceDatasetNum
  475. ctx.HTML(200, tplReference)
  476. }
  477. func ReferenceDatasetData(ctx *context.Context) {
  478. MustEnableDataset(ctx)
  479. datasetIds := models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID)
  480. var datasets models.DatasetList
  481. var err error
  482. if len(datasetIds) > 0 {
  483. opts := &models.SearchDatasetOptions{
  484. DatasetIDs: datasetIds,
  485. NeedAttachment: false,
  486. CloudBrainType: models.TypeCloudBrainAll,
  487. ListOptions: models.ListOptions{
  488. Page: 1,
  489. PageSize: setting.RepoMaxReferenceDatasetNum,
  490. },
  491. SearchOrderBy: getSearchOrderByInValues(datasetIds),
  492. QueryReference: true,
  493. }
  494. datasets, _, err = models.SearchDataset(opts)
  495. if err != nil {
  496. ctx.ServerError("SearchDatasets", err)
  497. return
  498. }
  499. }
  500. ctx.JSON(http.StatusOK, repository.ConvertToDatasetWithStar(ctx, datasets))
  501. }
  502. func PublicDataset(ctx *context.Context) {
  503. page := ctx.QueryInt("page")
  504. cloudbrainType := ctx.QueryInt("type")
  505. keyword := strings.Trim(ctx.Query("q"), " ")
  506. datasets, count, err := models.Attachments(&models.AttachmentsOptions{
  507. ListOptions: models.ListOptions{
  508. Page: page,
  509. PageSize: setting.UI.DatasetPagingNum,
  510. },
  511. Keyword: keyword,
  512. NeedDatasetIDs: false,
  513. NeedIsPrivate: true,
  514. IsPrivate: false,
  515. Type: cloudbrainType,
  516. JustNeedZipFile: true,
  517. NeedRepoInfo: true,
  518. RecommendOnly: ctx.QueryBool("recommend"),
  519. })
  520. if err != nil {
  521. ctx.ServerError("datasets", err)
  522. return
  523. }
  524. data, err := json.Marshal(datasets)
  525. if err != nil {
  526. log.Error("json.Marshal failed:", err.Error())
  527. ctx.JSON(200, map[string]string{
  528. "result_code": "-1",
  529. "error_msg": err.Error(),
  530. "data": "",
  531. })
  532. return
  533. }
  534. ctx.JSON(200, map[string]string{
  535. "result_code": "0",
  536. "data": string(data),
  537. "count": strconv.FormatInt(count, 10),
  538. })
  539. }
  540. func MyFavoriteDataset(ctx *context.Context) {
  541. UserId := ctx.User.ID
  542. cloudbrainType := ctx.QueryInt("type")
  543. keyword := strings.Trim(ctx.Query("q"), " ")
  544. var NotColDatasetIDs []int64
  545. var IsColDatasetIDs []int64
  546. datasetStars, err := models.GetDatasetStarByUser(ctx.User)
  547. if err != nil {
  548. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetStarByUser failed", err)))
  549. log.Error("GetDatasetStarByUser failed:", err.Error())
  550. ctx.JSON(200, map[string]string{
  551. "result_code": "-1",
  552. "error_msg": err.Error(),
  553. "data": "",
  554. })
  555. return
  556. }
  557. //If the dataset has been deleted, it will not be counted
  558. for _, datasetStar := range datasetStars {
  559. IsExist, repo, dataset, err := IsDatasetStarExist(datasetStar)
  560. if err != nil {
  561. log.Error("IsDatasetStarExist error:", err.Error())
  562. }
  563. if IsExist {
  564. DatasetIsCollaborator := DatasetIsCollaborator(ctx, dataset)
  565. if repo.OwnerID == ctx.User.ID || DatasetIsCollaborator {
  566. IsColDatasetIDs = append(IsColDatasetIDs, datasetStar.DatasetID)
  567. } else {
  568. NotColDatasetIDs = append(NotColDatasetIDs, datasetStar.DatasetID)
  569. }
  570. }
  571. }
  572. NotColDatasets, NotColcount, err := models.Attachments(&models.AttachmentsOptions{
  573. Keyword: keyword,
  574. NeedDatasetIDs: true,
  575. DatasetIDs: NotColDatasetIDs,
  576. NeedIsPrivate: true,
  577. IsPrivate: false,
  578. Type: cloudbrainType,
  579. JustNeedZipFile: true,
  580. NeedRepoInfo: true,
  581. RecommendOnly: ctx.QueryBool("recommend"),
  582. UserId: UserId,
  583. })
  584. if err != nil {
  585. ctx.ServerError("datasets", err)
  586. return
  587. }
  588. //If is collaborator, there is no need to determine whether the dataset is private or public
  589. IsColDatasets, IsColcount, err := models.Attachments(&models.AttachmentsOptions{
  590. Keyword: keyword,
  591. NeedDatasetIDs: true,
  592. DatasetIDs: IsColDatasetIDs,
  593. NeedIsPrivate: false,
  594. Type: cloudbrainType,
  595. JustNeedZipFile: true,
  596. NeedRepoInfo: true,
  597. RecommendOnly: ctx.QueryBool("recommend"),
  598. UserId: UserId,
  599. })
  600. if err != nil {
  601. ctx.ServerError("datasets", err)
  602. return
  603. }
  604. for _, NotColDataset := range NotColDatasets {
  605. IsColDatasets = append(IsColDatasets, NotColDataset)
  606. }
  607. datasets := IsColDatasets
  608. count := NotColcount + IsColcount
  609. sort.Slice(datasets, func(i, j int) bool {
  610. return datasets[i].Attachment.CreatedUnix > datasets[j].Attachment.CreatedUnix
  611. })
  612. page := ctx.QueryInt("page")
  613. if page <= 0 {
  614. page = 1
  615. }
  616. pagesize := ctx.QueryInt("pagesize")
  617. if pagesize <= 0 {
  618. pagesize = 5
  619. }
  620. pageDatasetsInfo := getPageDatasets(datasets, page, pagesize)
  621. if pageDatasetsInfo == nil {
  622. ctx.JSON(200, map[string]string{
  623. "result_code": "0",
  624. "data": "[]",
  625. "count": strconv.FormatInt(count, 10),
  626. })
  627. return
  628. }
  629. data, err := json.Marshal(pageDatasetsInfo)
  630. log.Info("data:", data)
  631. if err != nil {
  632. log.Error("json.Marshal failed:", err.Error())
  633. ctx.JSON(200, map[string]string{
  634. "result_code": "-1",
  635. "error_msg": err.Error(),
  636. "data": "",
  637. })
  638. return
  639. }
  640. ctx.JSON(200, map[string]string{
  641. "result_code": "0",
  642. "data": string(data),
  643. "count": strconv.FormatInt(count, 10),
  644. })
  645. }
  646. func getPageDatasets(AttachmentInfos []*models.AttachmentInfo, page int, pagesize int) []*models.AttachmentInfo {
  647. begin := (page - 1) * pagesize
  648. end := (page) * pagesize
  649. if begin > len(AttachmentInfos)-1 {
  650. return nil
  651. }
  652. if end > len(AttachmentInfos)-1 {
  653. return AttachmentInfos[begin:]
  654. } else {
  655. return AttachmentInfos[begin:end]
  656. }
  657. }
  658. func getTotalPage(total int64, pageSize int) int {
  659. another := 0
  660. if int(total)%pageSize != 0 {
  661. another = 1
  662. }
  663. return int(total)/pageSize + another
  664. }
  665. func GetDatasetStatus(ctx *context.Context) {
  666. var (
  667. err error
  668. )
  669. UUID := ctx.Params(":uuid")
  670. attachment, err := models.GetAttachmentByUUID(UUID)
  671. if err != nil {
  672. log.Error("GetDatasetStarByUser failed:", err.Error())
  673. ctx.JSON(200, map[string]string{
  674. "result_code": "-1",
  675. "error_msg": err.Error(),
  676. "data": "",
  677. })
  678. return
  679. }
  680. ctx.JSON(200, map[string]string{
  681. "result_code": "0",
  682. "UUID": UUID,
  683. "AttachmentStatus": fmt.Sprint(attachment.DecompressState),
  684. })
  685. }
  686. func DatasetIsCollaborator(ctx *context.Context, dataset *models.Dataset) bool {
  687. repo, err := models.GetRepositoryByID(dataset.RepoID)
  688. if err != nil {
  689. log.Error("query repo error:", err.Error())
  690. } else {
  691. repo.GetOwner()
  692. if ctx.User != nil {
  693. if repo.Owner.IsOrganization() {
  694. org := repo.Owner
  695. org.Teams, err = org.GetUserTeams(ctx.User.ID)
  696. if err != nil {
  697. log.Error("GetUserTeams error:", err.Error())
  698. return false
  699. }
  700. if org.IsUserPartOfOrg(ctx.User.ID) {
  701. for _, t := range org.Teams {
  702. if t.IsMember(ctx.User.ID) && t.HasRepository(repo.ID) {
  703. return true
  704. }
  705. }
  706. isOwner, _ := models.IsOrganizationOwner(repo.OwnerID, ctx.User.ID)
  707. if isOwner {
  708. return isOwner
  709. }
  710. return false
  711. }
  712. }
  713. isCollaborator, _ := repo.IsCollaborator(ctx.User.ID)
  714. if isCollaborator {
  715. return true
  716. }
  717. }
  718. }
  719. return false
  720. }
  721. func IsDatasetStarExist(datasetStar *models.DatasetStar) (bool, *models.Repository, *models.Dataset, error) {
  722. dataset, err := models.GetDatasetByID(datasetStar.DatasetID)
  723. if err != nil {
  724. log.Error("query dataset error:", err.Error())
  725. return false, nil, nil, err
  726. } else {
  727. repo, err := models.GetRepositoryByID(dataset.RepoID)
  728. if err != nil {
  729. log.Error("GetRepositoryByID error:", err.Error())
  730. return false, nil, nil, err
  731. }
  732. return true, repo, dataset, nil
  733. }
  734. }