diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 8aeb24bce..5b06d2581 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1127,3 +1127,20 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Limit(100). Find(&cloudbrains) } + +func GetCloudbrainCountByUserID(userID int64) (int, error) { + count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + return int(count), err +} + +func GetCloudbrainNotebookCountByUserID(userID int64) (int, error) { + count, err := x.In("status", ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsRestarting). + And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) + return int(count), err +} + +func GetCloudbrainTrainJobCountByUserID(userID int64) (int, error) { + count, err := x.In("status", ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobWaiting, ModelArtsTrainJobRunning, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted). + And("job_type = ? and user_id = ? and type = ?", JobTypeTrain, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) + return int(count), err +} diff --git a/modules/context/repo.go b/modules/context/repo.go index c6e5e8edd..9f8a178fc 100755 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -524,7 +524,7 @@ func RepoAssignment() macaron.Handler { } ctx.Data["Tags"] = tags - brs, err := ctx.Repo.GitRepo.GetBranches() + brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches", err) return @@ -712,7 +712,7 @@ func RepoRefByType(refType RepoRefType) macaron.Handler { refName = ctx.Repo.Repository.DefaultBranch ctx.Repo.BranchName = refName if !ctx.Repo.GitRepo.IsBranchExist(refName) { - brs, err := ctx.Repo.GitRepo.GetBranches() + brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches", err) return diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go old mode 100644 new mode 100755 index 8f9c802e0..ad5acfdbf --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -6,7 +6,9 @@ package git import ( + "bufio" "fmt" + "io" "strings" "github.com/go-git/go-git/v5/plumbing" @@ -74,25 +76,6 @@ func (repo *Repository) SetDefaultBranch(name string) error { return err } -// GetBranches returns all branches of the repository. -func (repo *Repository) GetBranches() ([]string, error) { - var branchNames []string - - branches, err := repo.gogitRepo.Branches() - if err != nil { - return nil, err - } - - _ = branches.ForEach(func(branch *plumbing.Reference) error { - branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) - return nil - }) - - // TODO: Sort? - - return branchNames, nil -} - // GetBranch returns a branch by it's name func (repo *Repository) GetBranch(branch string) (*Branch, error) { if !repo.IsBranchExist(branch) { @@ -106,16 +89,16 @@ func (repo *Repository) GetBranch(branch string) (*Branch, error) { } // GetBranchesByPath returns a branch by it's path -func GetBranchesByPath(path string) ([]*Branch, error) { +func GetBranchesByPath(path string, skip, limit int) ([]*Branch, int, error) { gitRepo, err := OpenRepository(path) if err != nil { - return nil, err + return nil, 0, err } defer gitRepo.Close() - brs, err := gitRepo.GetBranches() + brs, countAll, err := gitRepo.GetBranches(skip, limit) if err != nil { - return nil, err + return nil, 0, err } branches := make([]*Branch, len(brs)) @@ -127,7 +110,7 @@ func GetBranchesByPath(path string) ([]*Branch, error) { } } - return branches, nil + return branches, countAll, nil } // DeleteBranchOptions Option(s) for delete branch @@ -183,3 +166,91 @@ func (repo *Repository) RemoveRemote(name string) error { func (branch *Branch) GetCommit() (*Commit, error) { return branch.gitRepo.GetBranchCommit(branch.Name) } + +// GetBranches returns branches from the repository, skipping skip initial branches and +// returning at most limit branches, or all branches if limit is 0. +func (repo *Repository) GetBranches(skip, limit int) ([]string, int, error) { + return callShowRef(repo.Path, BranchPrefix, "--heads", skip, limit) +} + +// callShowRef return refs, if limit = 0 it will not limit +func callShowRef(repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + if err != nil { + if stderrBuilder.Len() == 0 { + _ = stdoutWriter.Close() + return + } + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + i := 0 + bufReader := bufio.NewReader(stdoutReader) + for i < skip { + _, isPrefix, err := bufReader.ReadLine() + if err == io.EOF { + return branchNames, i, nil + } + if err != nil { + return nil, 0, err + } + if !isPrefix { + i++ + } + } + for limit == 0 || i < skip+limit { + // The output of show-ref is simply a list: + // SP LF + _, err := bufReader.ReadSlice(' ') + for err == bufio.ErrBufferFull { + // This shouldn't happen but we'll tolerate it for the sake of peace + _, err = bufReader.ReadSlice(' ') + } + if err == io.EOF { + return branchNames, i, nil + } + if err != nil { + return nil, 0, err + } + + branchName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This shouldn't happen... but we'll tolerate it for the sake of peace + return branchNames, i, nil + } + if err != nil { + return nil, i, err + } + branchName = strings.TrimPrefix(branchName, prefix) + if len(branchName) > 0 { + branchName = branchName[:len(branchName)-1] + } + branchNames = append(branchNames, branchName) + i++ + } + // count all refs + for limit != 0 { + _, isPrefix, err := bufReader.ReadLine() + if err == io.EOF { + return branchNames, i, nil + } + if err != nil { + return nil, 0, err + } + if !isPrefix { + i++ + } + } + return branchNames, i, nil +} diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go old mode 100644 new mode 100755 index 7780e3477..08f6f5ec1 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/go-git/go-git/v5/plumbing" - "github.com/mcuadros/go-version" ) // TagPrefix tags prefix path on the repository @@ -225,29 +224,35 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) { return tags, nil } -// GetTags returns all tags of the repository. -func (repo *Repository) GetTags() ([]string, error) { - var tagNames []string - - tags, err := repo.gogitRepo.Tags() - if err != nil { - return nil, err - } +//// GetTags returns all tags of the repository. +//func (repo *Repository) GetTags() ([]string, error) { +// var tagNames []string +// +// tags, err := repo.gogitRepo.Tags() +// if err != nil { +// return nil, err +// } +// +// _ = tags.ForEach(func(tag *plumbing.Reference) error { +// tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) +// return nil +// }) +// +// version.Sort(tagNames) +// +// // Reverse order +// for i := 0; i < len(tagNames)/2; i++ { +// j := len(tagNames) - i - 1 +// tagNames[i], tagNames[j] = tagNames[j], tagNames[i] +// } +// +// return tagNames, nil +//} - _ = tags.ForEach(func(tag *plumbing.Reference) error { - tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) - return nil - }) - - version.Sort(tagNames) - - // Reverse order - for i := 0; i < len(tagNames)/2; i++ { - j := len(tagNames) - i - 1 - tagNames[i], tagNames[j] = tagNames[j], tagNames[i] - } - - return tagNames, nil +// GetTags returns all tags of the repository. +func (repo *Repository) GetTags() (tags []string, err error) { + tags, _, err = callShowRef(repo.Path, TagPrefix, "--tags", 0, 0) + return } // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) diff --git a/modules/git/utils.go b/modules/git/utils.go old mode 100644 new mode 100755 index 83209924c..0d044feda --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -140,3 +140,11 @@ func ParseBool(value string) (result bool, valid bool) { } return intValue != 0, true } + +// ConcatenateError concatenats an error with stderr string +func ConcatenateError(err error, stderr string) error { + if len(stderr) == 0 { + return err + } + return fmt.Errorf("%w - %s", err, stderr) +} diff --git a/modules/repository/branch.go b/modules/repository/branch.go old mode 100644 new mode 100755 index 418ba25c8..7b203dd91 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -23,9 +23,10 @@ func GetBranch(repo *models.Repository, branch string) (*git.Branch, error) { return gitRepo.GetBranch(branch) } -// GetBranches returns all the branches of a repository -func GetBranches(repo *models.Repository) ([]*git.Branch, error) { - return git.GetBranchesByPath(repo.RepoPath()) +// GetBranches returns branches from the repository, skipping skip initial branches and +// returning at most limit branches, or all branches if limit is 0. +func GetBranches(repo *models.Repository, skip, limit int) ([]*git.Branch, int, error) { + return git.GetBranchesByPath(repo.RepoPath(), skip, limit) } // checkBranchName validates branch name with existing repository branches @@ -36,7 +37,7 @@ func checkBranchName(repo *models.Repository, name string) error { } defer gitRepo.Close() - branches, err := GetBranches(repo) + branches, _, err := GetBranches(repo, 0, 0) if err != nil { return err } diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go old mode 100644 new mode 100755 index 57c74d7da..f4d1c924c --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -204,7 +204,7 @@ func ListBranches(ctx *context.APIContext) { // "200": // "$ref": "#/responses/BranchList" - branches, err := repo_module.GetBranches(ctx.Repo.Repository) + branches, _, err := repo_module.GetBranches(ctx.Repo.Repository,0,0) if err != nil { ctx.Error(http.StatusInternalServerError, "GetBranches", err) return diff --git a/routers/repo/branch.go b/routers/repo/branch.go old mode 100644 new mode 100755 index e7eac04bc..c8e492373 --- a/routers/repo/branch.go +++ b/routers/repo/branch.go @@ -181,7 +181,7 @@ func deleteBranch(ctx *context.Context, branchName string) error { } func loadBranches(ctx *context.Context) []*Branch { - rawBranches, err := repo_module.GetBranches(ctx.Repo.Repository) + rawBranches, _, err := repo_module.GetBranches(ctx.Repo.Repository, 0, 0) if err != nil { ctx.ServerError("GetBranches", err) return nil diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index e94892a93..c74d618d5 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -216,7 +216,22 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - _, err := models.GetCloudbrainByName(jobName) + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainNew, &form) + return + } + } + + _, err = models.GetCloudbrainByName(jobName) if err == nil { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) diff --git a/routers/repo/compare.go b/routers/repo/compare.go old mode 100644 new mode 100755 index 97bb5e6b1..babe416a7 --- a/routers/repo/compare.go +++ b/routers/repo/compare.go @@ -507,7 +507,7 @@ func getBranchesForRepo(user *models.User, repo *models.Repository) (bool, []str } defer gitRepo.Close() - branches, err := gitRepo.GetBranches() + branches, _, err := gitRepo.GetBranches(0, 0) if err != nil { return false, nil, err } @@ -528,7 +528,7 @@ func CompareDiff(ctx *context.Context) { } if ctx.Data["PageIsComparePull"] == true { - headBranches, err := headGitRepo.GetBranches() + headBranches, _, err := headGitRepo.GetBranches(0,0) if err != nil { ctx.ServerError("GetBranches", err) return diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 9483814d6..77ed0251d 100755 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -424,7 +424,7 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository, isPull boo return nil } - brs, err := ctx.Repo.GitRepo.GetBranches() + brs, _, err := ctx.Repo.GitRepo.GetBranches(0,0) if err != nil { ctx.ServerError("GetBranches", err) return nil diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 7324bbbee..229e029b4 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -115,7 +115,22 @@ func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) description := form.Description flavor := form.Flavor - err := modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) + count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplModelArtsNotebookNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsNotebookNew, &form) + return + } + } + + err = modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) if err != nil { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return @@ -360,18 +375,6 @@ func trainJobNewDataPrepare(ctx *context.Context) error { outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - - Branches, err := ctx.Repo.GitRepo.GetBranches() - if err != nil { - log.Error("GetBranches failed:%v", err) - ctx.ServerError("GetBranches error:", err) - return err - } - if Branches != nil { - ctx.Data["Branches"] = Branches - } - - ctx.Data["BranchesCount"] = len(Branches) ctx.Data["params"] = "" ctx.Data["branchName"] = ctx.Repo.BranchName @@ -442,14 +445,6 @@ func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArts outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - Branches, err := ctx.Repo.GitRepo.GetBranches() - if err != nil { - ctx.ServerError("GetBranches error:", err) - return err - } - ctx.Data["Branches"] = Branches - ctx.Data["BranchesCount"] = len(Branches) - configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { ctx.ServerError("getConfigList failed:", err) @@ -545,13 +540,13 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - Branches, err := ctx.Repo.GitRepo.GetBranches() + branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches error:", err) return err } - ctx.Data["branches"] = Branches + ctx.Data["branches"] = branches ctx.Data["branch_name"] = task.BranchName ctx.Data["description"] = task.Description ctx.Data["boot_file"] = task.BootFile @@ -634,12 +629,12 @@ func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrai outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - Branches, err := ctx.Repo.GitRepo.GetBranches() + branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches error:", err) return err } - ctx.Data["branches"] = Branches + ctx.Data["branches"] = branches ctx.Data["description"] = form.Description ctx.Data["dataset_name"] = task.DatasetName ctx.Data["work_server_number"] = form.WorkServerNumber @@ -687,6 +682,21 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCount EngineName := form.EngineName + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobNew, &form) + return + } + } + if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) trainJobErrorNewDataPrepare(ctx, form) @@ -839,7 +849,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - err := modelarts.GenerateTrainJob(ctx, req) + err = modelarts.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) trainJobErrorNewDataPrepare(ctx, form) @@ -853,6 +863,21 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.Data["PageIsTrainJob"] = true var jobID = ctx.Params(":jobid") + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("system error", tplModelArtsTrainJobVersionNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobVersionNew, &form) + return + } + } + latestTask, err := models.GetCloudbrainByJobIDAndIsLatestVersion(jobID, modelarts.IsLatestVersion) if err != nil { ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go old mode 100644 new mode 100755 index 165e7cd35..924574471 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -252,7 +252,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { } } - branches, err := repo_module.GetBranches(m.Repo) + branches, _, err := repo_module.GetBranches(m.Repo,0,0) if err != nil { log.Error("GetBranches: %v", err) return nil, false diff --git a/services/pull/pull.go b/services/pull/pull.go old mode 100644 new mode 100755 index fb4af0637..230a9e389 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -452,7 +452,7 @@ func CloseBranchPulls(doer *models.User, repoID int64, branch string) error { // CloseRepoBranchesPulls close all pull requests which head branches are in the given repository func CloseRepoBranchesPulls(doer *models.User, repo *models.Repository) error { - branches, err := git.GetBranchesByPath(repo.RepoPath()) + branches, _, err := git.GetBranchesByPath(repo.RepoPath(), 0, 0) if err != nil { return err } diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index a3fdddf38..cd85057de 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -474,6 +474,7 @@ td, th {