aiforge/routers/repo/dataset.go

package repo

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"unicode/utf8"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
)

const (
	tplIndex         base.TplName = "repo/datasets/index"
	tplDatasetCreate base.TplName = "repo/datasets/create"
	tplDatasetEdit   base.TplName = "repo/datasets/edit"
	taskstplIndex    base.TplName = "repo/datasets/tasks/index"
)

// MustEnableDataset check if repository enable internal dataset
func MustEnableDataset(ctx *context.Context) {
	if !ctx.Repo.CanRead(models.UnitTypeDatasets) {
		ctx.NotFound("MustEnableDataset", nil)
		return
	}
}

func newFilterPrivateAttachments(ctx *context.Context, list []*models.Attachment, repo *models.Repository) []*models.Attachment {

	if ctx.Repo.CanWrite(models.UnitTypeDatasets) {
		log.Info("can write.")
		return list
	} else {
		if repo.Owner == nil {
			repo.GetOwner()
		}
		permission := false
		if repo.Owner.IsOrganization() && ctx.User != nil {
			if repo.Owner.IsUserPartOfOrg(ctx.User.ID) {
				log.Info("user is member of org.")
				permission = true
			}
		}
		if !permission && ctx.User != nil {
			isCollaborator, _ := repo.IsCollaborator(ctx.User.ID)
			if isCollaborator {
				log.Info("Collaborator user may visit the attach.")
				permission = true
			}
		}

		var publicList []*models.Attachment
		for _, attach := range list {
			if !attach.IsPrivate {
				publicList = append(publicList, attach)
			} else {
				if permission {
					publicList = append(publicList, attach)
				}
			}
		}
		return publicList
	}
}

func QueryDataSet(ctx *context.Context) []*models.Attachment {
	repo := ctx.Repo.Repository

	dataset, err := models.GetDatasetByRepo(repo)
	if err != nil {
		log.Error("zou  not found dataset 1")
		ctx.NotFound("GetDatasetByRepo", err)
		return nil
	}

	if ctx.Query("type") == "" {
		log.Error("zou  not found  type 2")
		ctx.NotFound("type error", nil)
		return nil
	}
	err = models.GetDatasetAttachments(ctx.QueryInt("type"), ctx.IsSigned, ctx.User, dataset)
	if err != nil {
		ctx.ServerError("GetDatasetAttachments", err)
		return nil
	}
	attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo)

	ctx.Data["SortType"] = ctx.Query("sort")

	sort.Slice(attachments, func(i, j int) bool {
		return attachments[i].CreatedUnix > attachments[j].CreatedUnix
	})

	return attachments
}

func DatasetIndex(ctx *context.Context) {
	log.Info("dataset index 1")
	MustEnableDataset(ctx)
	ctx.Data["PageIsDataset"] = true

	ctx.Data["SortType"] = ctx.Query("sort")

	repo := ctx.Repo.Repository

	dataset, err := models.GetDatasetByRepo(repo)
	ctx.Data["CanWrite"] = ctx.Repo.CanWrite(models.UnitTypeDatasets)
	if err != nil {
		log.Warn("query dataset, not found.")
		ctx.HTML(200, tplIndex)
		return
	}
	cloudbrainType := -1
	if ctx.Query("type") != "" {

		cloudbrainType = ctx.QueryInt("type")
	}
	err = models.GetDatasetAttachments(cloudbrainType, ctx.IsSigned, ctx.User, dataset)
	if err != nil {
		ctx.ServerError("GetDatasetAttachments", err)
		return
	}

	attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo)

	if ctx.Data["SortType"] == "nameAsc" {
		sort.Slice(attachments, func(i, j int) bool {
			return strings.ToLower(attachments[i].Name) < strings.ToLower(attachments[j].Name)
		})
	} else if ctx.Data["SortType"] == "nameDesc" {
		sort.Slice(attachments, func(i, j int) bool {
			return strings.ToLower(attachments[i].Name) > strings.ToLower(attachments[j].Name)
		})
	} else if ctx.Data["SortType"] == "sizeAsc" {
		sort.Slice(attachments, func(i, j int) bool {
			return attachments[i].Size < attachments[j].Size
		})
	} else if ctx.Data["SortType"] == "sizeDesc" {
		sort.Slice(attachments, func(i, j int) bool {
			return attachments[i].Size > attachments[j].Size
		})
	} else if ctx.Data["SortType"] == "timeAsc" {
		sort.Slice(attachments, func(i, j int) bool {
			return attachments[i].CreatedUnix < attachments[j].CreatedUnix
		})
	} else {
		sort.Slice(attachments, func(i, j int) bool {
			return attachments[i].CreatedUnix > attachments[j].CreatedUnix
		})
	}

	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pagesize := ctx.QueryInt("pagesize")
	if pagesize <= 0 {
		pagesize = 10
	}
	pager := context.NewPagination(len(attachments), pagesize, page, 5)

	pageAttachments := getPageAttachments(attachments, page, pagesize)

	//load attachment creator
	for _, attachment := range pageAttachments {
		uploader, _ := models.GetUserByID(attachment.UploaderID)
		attachment.Uploader = uploader
		if !strings.HasSuffix(attachment.Name, ".zip") && !strings.HasSuffix(attachment.Name, ".tar.gz") {
			attachment.DecompressState = -1 //非压缩文件
		}

	}

	ctx.Data["Page"] = pager

	ctx.Data["Title"] = ctx.Tr("dataset.show_dataset")
	ctx.Data["Link"] = ctx.Repo.RepoLink + "/datasets"
	ctx.Data["dataset"] = dataset
	ctx.Data["Attachments"] = pageAttachments
	ctx.Data["IsOwner"] = true
	ctx.Data["StoreType"] = setting.Attachment.StoreType
	ctx.Data["Type"] = cloudbrainType

	renderAttachmentSettings(ctx)

	ctx.HTML(200, tplIndex)
}

func getPageAttachments(attachments []*models.Attachment, page int, pagesize int) []*models.Attachment {
	begin := (page - 1) * pagesize
	end := (page) * pagesize

	if begin > len(attachments)-1 {
		return nil
	}
	if end > len(attachments)-1 {
		return attachments[begin:]
	} else {
		return attachments[begin:end]
	}

}

func CreateDataset(ctx *context.Context) {

	MustEnableDataset(ctx)
	ctx.Data["PageIsDataset"] = true

	ctx.HTML(200, tplDatasetCreate)
}

func EditDataset(ctx *context.Context) {

	MustEnableDataset(ctx)
	ctx.Data["PageIsDataset"] = true
	datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64)

	dataset, _ := models.GetDatasetByID(datasetId)
	if dataset == nil {
		ctx.Error(http.StatusNotFound, "")
		return
	}
	ctx.Data["Dataset"] = dataset

	ctx.HTML(200, tplDatasetEdit)
}

func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) {

	dataset := &models.Dataset{}

	if !NamePattern.MatchString(form.Title) {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
		return
	}
	if utf8.RuneCountInString(form.Description) > 1024 {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 1024)))
		return
	}

	dataset.RepoID = ctx.Repo.Repository.ID
	dataset.UserID = ctx.User.ID
	dataset.Category = form.Category
	dataset.Task = form.Task
	dataset.Title = form.Title
	dataset.License = form.License
	dataset.Description = form.Description
	dataset.DownloadTimes = 0
	if ctx.Repo.Repository.IsPrivate {
		dataset.Status = 0
	} else {
		dataset.Status = 1
	}
	err := models.CreateDataset(dataset)
	if err != nil {
		log.Error("fail to create dataset", err)
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.create_dataset_fail")))
	} else {
		ctx.JSON(http.StatusOK, models.BaseOKMessage)
	}

}

func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) {
	ctx.Data["PageIsDataset"] = true

	ctx.Data["Title"] = ctx.Tr("dataset.edit_dataset")

	if !NamePattern.MatchString(form.Title) {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
		return
	}
	if utf8.RuneCountInString(form.Description) > 1024 {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 1024)))
		return
	}

	rel, err := models.GetDatasetByID(form.ID)
	ctx.Data["dataset"] = rel

	if err != nil {
		log.Error("failed to query dataset", err)
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail")))
		return
	}

	rel.Title = form.Title
	rel.Description = form.Description
	rel.Category = form.Category
	rel.Task = form.Task
	rel.License = form.License
	if err = models.UpdateDataset(models.DefaultDBContext(), rel); err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail")))
	}
	ctx.JSON(http.StatusOK, models.BaseOKMessage)
}

func DatasetAction(ctx *context.Context) {
	var err error
	datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64)
	switch ctx.Params(":action") {
	case "star":
		err = models.StarDataset(ctx.User.ID, datasetId, true)
	case "unstar":
		err = models.StarDataset(ctx.User.ID, datasetId, false)

	}
	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action"))))
	} else {
		ctx.JSON(http.StatusOK, models.BaseOKMessage)
	}

}

func CurrentRepoDataset(ctx *context.Context) {
	page := ctx.QueryInt("page")
	cloudbrainType := ctx.QueryInt("type")
	keyword := strings.Trim(ctx.Query("q"), " ")

	repo := ctx.Repo.Repository
	var datasetIDs []int64
	dataset, err := models.GetDatasetByRepo(repo)
	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetByRepo failed", err)))
		return
	}
	datasetIDs = append(datasetIDs, dataset.ID)
	datasets, count, err := models.Attachments(&models.AttachmentsOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: setting.UI.DatasetPagingNum,
		},
		Keyword:         keyword,
		NeedDatasetIDs:  true,
		DatasetIDs:      datasetIDs,
		Type:            cloudbrainType,
		NeedIsPrivate:   false,
		JustNeedZipFile: true,
		NeedRepoInfo:    true,
	})
	if err != nil {
		ctx.ServerError("datasets", err)
		return
	}

	data, err := json.Marshal(datasets)
	if err != nil {
		log.Error("json.Marshal failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}
	ctx.JSON(200, map[string]string{
		"result_code": "0",
		"data":        string(data),
		"count":       strconv.FormatInt(count, 10),
	})
}

func MyDatasets(ctx *context.Context) {
	page := ctx.QueryInt("page")
	cloudbrainType := ctx.QueryInt("type")
	keyword := strings.Trim(ctx.Query("q"), " ")

	uploaderID := ctx.User.ID
	datasets, count, err := models.Attachments(&models.AttachmentsOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: setting.UI.DatasetPagingNum,
		},
		Keyword:         keyword,
		NeedDatasetIDs:  false,
		UploaderID:      uploaderID,
		Type:            cloudbrainType,
		NeedIsPrivate:   false,
		JustNeedZipFile: true,
		NeedRepoInfo:    true,
		RecommendOnly:   ctx.QueryBool("recommend"),
	})
	if err != nil {
		ctx.ServerError("datasets", err)
		return
	}

	data, err := json.Marshal(datasets)
	if err != nil {
		log.Error("json.Marshal failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}
	ctx.JSON(200, map[string]string{
		"result_code": "0",
		"data":        string(data),
		"count":       strconv.FormatInt(count, 10),
	})
}

func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
	page := ctx.QueryInt("page")
	cloudbrainType := ctx.QueryInt("type")
	keyword := strings.Trim(ctx.Query("q"), " ")
	orderBy := models.SearchOrderByRecentUpdated
	var ownerID int64
	if ctx.User != nil {
		ownerID = ctx.User.ID
	}

	opts.Keyword = keyword
	opts.SearchOrderBy = orderBy
	opts.OwnerID = ownerID
	opts.RecommendOnly = ctx.QueryBool("recommend")
	opts.CloudBrainType = cloudbrainType
	opts.ListOptions = models.ListOptions{
		Page:     page,
		PageSize: setting.UI.DatasetPagingNum,
	}
	opts.NeedAttachment = true
	opts.JustNeedZipFile = true
	opts.User = ctx.User

	datasets, count, err := models.SearchDataset(opts)

	if err != nil {
		ctx.ServerError("datasets", err)
		return
	}

	data, err := json.Marshal(datasets)
	if err != nil {
		log.Error("json.Marshal failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}
	ctx.JSON(200, map[string]string{
		"result_code": "0",
		"data":        string(data),
		"count":       strconv.FormatInt(count, 10),
	})

}

func CurrentRepoDatasetMultiple(ctx *context.Context) {

	opts := &models.SearchDatasetOptions{
		RepoID: ctx.Repo.Repository.ID,
	}
	datasetMultiple(ctx, opts)

}

func MyDatasetsMultiple(ctx *context.Context) {

	opts := &models.SearchDatasetOptions{
		UploadAttachmentByMe: true,
	}
	datasetMultiple(ctx, opts)

}

func PublicDatasetMultiple(ctx *context.Context) {

	opts := &models.SearchDatasetOptions{
		PublicOnly: true,
	}
	datasetMultiple(ctx, opts)

}

func MyFavoriteDatasetMultiple(ctx *context.Context) {

	opts := &models.SearchDatasetOptions{
		StarByMe:   true,
		DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID),
	}
	datasetMultiple(ctx, opts)
}

func PublicDataset(ctx *context.Context) {
	page := ctx.QueryInt("page")
	cloudbrainType := ctx.QueryInt("type")
	keyword := strings.Trim(ctx.Query("q"), " ")

	datasets, count, err := models.Attachments(&models.AttachmentsOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: setting.UI.DatasetPagingNum,
		},
		Keyword:         keyword,
		NeedDatasetIDs:  false,
		NeedIsPrivate:   true,
		IsPrivate:       false,
		Type:            cloudbrainType,
		JustNeedZipFile: true,
		NeedRepoInfo:    true,
		RecommendOnly:   ctx.QueryBool("recommend"),
	})
	if err != nil {
		ctx.ServerError("datasets", err)
		return
	}

	data, err := json.Marshal(datasets)
	if err != nil {
		log.Error("json.Marshal failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}
	ctx.JSON(200, map[string]string{
		"result_code": "0",
		"data":        string(data),
		"count":       strconv.FormatInt(count, 10),
	})
}

func MyFavoriteDataset(ctx *context.Context) {
	UserId := ctx.User.ID
	cloudbrainType := ctx.QueryInt("type")
	keyword := strings.Trim(ctx.Query("q"), " ")
	var NotColDatasetIDs []int64
	var IsColDatasetIDs []int64
	datasetStars, err := models.GetDatasetStarByUser(ctx.User)
	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetStarByUser failed", err)))
		log.Error("GetDatasetStarByUser failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}
	//If the dataset has been deleted, it will not be counted
	for _, datasetStar := range datasetStars {
		IsExist, repo, dataset, err := IsDatasetStarExist(datasetStar)
		if err != nil {
			log.Error("IsDatasetStarExist error:", err.Error())
		}
		if IsExist {
			DatasetIsCollaborator := DatasetIsCollaborator(ctx, dataset)
			if repo.OwnerID == ctx.User.ID || DatasetIsCollaborator {
				IsColDatasetIDs = append(IsColDatasetIDs, datasetStar.DatasetID)
			} else {
				NotColDatasetIDs = append(NotColDatasetIDs, datasetStar.DatasetID)
			}
		}
	}

	NotColDatasets, NotColcount, err := models.Attachments(&models.AttachmentsOptions{
		Keyword:         keyword,
		NeedDatasetIDs:  true,
		DatasetIDs:      NotColDatasetIDs,
		NeedIsPrivate:   true,
		IsPrivate:       false,
		Type:            cloudbrainType,
		JustNeedZipFile: true,
		NeedRepoInfo:    true,
		RecommendOnly:   ctx.QueryBool("recommend"),
		UserId:          UserId,
	})
	if err != nil {
		ctx.ServerError("datasets", err)
		return
	}
	//If is collaborator, there is no need to determine whether the dataset is private or public
	IsColDatasets, IsColcount, err := models.Attachments(&models.AttachmentsOptions{
		Keyword:         keyword,
		NeedDatasetIDs:  true,
		DatasetIDs:      IsColDatasetIDs,
		NeedIsPrivate:   false,
		Type:            cloudbrainType,
		JustNeedZipFile: true,
		NeedRepoInfo:    true,
		RecommendOnly:   ctx.QueryBool("recommend"),
		UserId:          UserId,
	})
	if err != nil {
		ctx.ServerError("datasets", err)
		return
	}
	for _, NotColDataset := range NotColDatasets {
		IsColDatasets = append(IsColDatasets, NotColDataset)
	}
	datasets := IsColDatasets
	count := NotColcount + IsColcount
	sort.Slice(datasets, func(i, j int) bool {
		return datasets[i].Attachment.CreatedUnix > datasets[j].Attachment.CreatedUnix
	})

	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pagesize := ctx.QueryInt("pagesize")
	if pagesize <= 0 {
		pagesize = 5
	}
	pageDatasetsInfo := getPageDatasets(datasets, page, pagesize)
	if pageDatasetsInfo == nil {
		ctx.JSON(200, map[string]string{
			"result_code": "0",
			"data":        "[]",
			"count":       strconv.FormatInt(count, 10),
		})
		return
	}
	data, err := json.Marshal(pageDatasetsInfo)
	log.Info("data:", data)
	if err != nil {
		log.Error("json.Marshal failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}
	ctx.JSON(200, map[string]string{
		"result_code": "0",
		"data":        string(data),
		"count":       strconv.FormatInt(count, 10),
	})

}
func getPageDatasets(AttachmentInfos []*models.AttachmentInfo, page int, pagesize int) []*models.AttachmentInfo {
	begin := (page - 1) * pagesize
	end := (page) * pagesize

	if begin > len(AttachmentInfos)-1 {
		return nil
	}
	if end > len(AttachmentInfos)-1 {
		return AttachmentInfos[begin:]
	} else {
		return AttachmentInfos[begin:end]
	}

}
func getTotalPage(total int64, pageSize int) int {

	another := 0
	if int(total)%pageSize != 0 {
		another = 1
	}
	return int(total)/pageSize + another

}

func GetDatasetStatus(ctx *context.Context) {

	var (
		err error
	)

	UUID := ctx.Params(":uuid")
	attachment, err := models.GetAttachmentByUUID(UUID)
	if err != nil {
		log.Error("GetDatasetStarByUser failed:", err.Error())
		ctx.JSON(200, map[string]string{
			"result_code": "-1",
			"error_msg":   err.Error(),
			"data":        "",
		})
		return
	}

	ctx.JSON(200, map[string]string{
		"result_code":      "0",
		"UUID":             UUID,
		"AttachmentStatus": fmt.Sprint(attachment.DecompressState),
	})
}
func DatasetIsCollaborator(ctx *context.Context, dataset *models.Dataset) bool {
	repo, err := models.GetRepositoryByID(dataset.RepoID)
	if err != nil {
		log.Error("query repo error:", err.Error())
	} else {
		repo.GetOwner()
		if ctx.User != nil {
			if repo.Owner.IsOrganization() {
				org := repo.Owner
				org.Teams, err = org.GetUserTeams(ctx.User.ID)
				if err != nil {
					log.Error("GetUserTeams error:", err.Error())
					return false
				}
				if org.IsUserPartOfOrg(ctx.User.ID) {
					for _, t := range org.Teams {
						if t.IsMember(ctx.User.ID) && t.HasRepository(repo.ID) {
							return true
						}
					}
					isOwner, _ := models.IsOrganizationOwner(repo.OwnerID, ctx.User.ID)
					if isOwner {
						return isOwner
					}
					return false
				}
			}

			isCollaborator, _ := repo.IsCollaborator(ctx.User.ID)
			if isCollaborator {
				return true
			}
		}
	}

	return false
}
func IsDatasetStarExist(datasetStar *models.DatasetStar) (bool, *models.Repository, *models.Dataset, error) {
	dataset, err := models.GetDatasetByID(datasetStar.DatasetID)
	if err != nil {
		log.Error("query dataset error:", err.Error())
		return false, nil, nil, err
	} else {
		repo, err := models.GetRepositoryByID(dataset.RepoID)
		if err != nil {
			log.Error("GetRepositoryByID error:", err.Error())
			return false, nil, nil, err
		}
		return true, repo, dataset, nil
	}

}