Browse Source

Merge pull request 'V20211213.patch合入V20211213' (#1021) from merge-patch into V20211213

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/1021
pull/1023/head
lewis 3 years ago
parent
commit
e608c2e764
16 changed files with 235 additions and 90 deletions
  1. +17
    -0
      models/cloudbrain.go
  2. +2
    -2
      modules/context/repo.go
  3. +95
    -24
      modules/git/repo_branch.go
  4. +28
    -23
      modules/git/repo_tag.go
  5. +8
    -0
      modules/git/utils.go
  6. +5
    -4
      modules/repository/branch.go
  7. +1
    -1
      routers/api/v1/repo/branch.go
  8. +1
    -1
      routers/repo/branch.go
  9. +16
    -1
      routers/repo/cloudbrain.go
  10. +2
    -2
      routers/repo/compare.go
  11. +1
    -1
      routers/repo/issue.go
  12. +51
    -26
      routers/repo/modelarts.go
  13. +1
    -1
      services/mirror/mirror.go
  14. +1
    -1
      services/pull/pull.go
  15. +5
    -2
      templates/repo/modelarts/trainjob/show.tmpl
  16. +1
    -1
      templates/repo/modelarts/trainjob/version_new.tmpl

+ 17
- 0
models/cloudbrain.go View File

@@ -1127,3 +1127,20 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) {
Limit(100).
Find(&cloudbrains)
}

func GetCloudbrainCountByUserID(userID int64) (int, error) {
count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainOne).Count(new(Cloudbrain))
return int(count), err
}

func GetCloudbrainNotebookCountByUserID(userID int64) (int, error) {
count, err := x.In("status", ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsRestarting).
And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainTwo).Count(new(Cloudbrain))
return int(count), err
}

func GetCloudbrainTrainJobCountByUserID(userID int64) (int, error) {
count, err := x.In("status", ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobWaiting, ModelArtsTrainJobRunning, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted).
And("job_type = ? and user_id = ? and type = ?", JobTypeTrain, userID, TypeCloudBrainTwo).Count(new(Cloudbrain))
return int(count), err
}

+ 2
- 2
modules/context/repo.go View File

@@ -524,7 +524,7 @@ func RepoAssignment() macaron.Handler {
}
ctx.Data["Tags"] = tags

brs, err := ctx.Repo.GitRepo.GetBranches()
brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return
@@ -712,7 +712,7 @@ func RepoRefByType(refType RepoRefType) macaron.Handler {
refName = ctx.Repo.Repository.DefaultBranch
ctx.Repo.BranchName = refName
if !ctx.Repo.GitRepo.IsBranchExist(refName) {
brs, err := ctx.Repo.GitRepo.GetBranches()
brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return


+ 95
- 24
modules/git/repo_branch.go View File

@@ -6,7 +6,9 @@
package git

import (
"bufio"
"fmt"
"io"
"strings"

"github.com/go-git/go-git/v5/plumbing"
@@ -74,25 +76,6 @@ func (repo *Repository) SetDefaultBranch(name string) error {
return err
}

// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
var branchNames []string

branches, err := repo.gogitRepo.Branches()
if err != nil {
return nil, err
}

_ = branches.ForEach(func(branch *plumbing.Reference) error {
branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
return nil
})

// TODO: Sort?

return branchNames, nil
}

// GetBranch returns a branch by it's name
func (repo *Repository) GetBranch(branch string) (*Branch, error) {
if !repo.IsBranchExist(branch) {
@@ -106,16 +89,16 @@ func (repo *Repository) GetBranch(branch string) (*Branch, error) {
}

// GetBranchesByPath returns a branch by it's path
func GetBranchesByPath(path string) ([]*Branch, error) {
func GetBranchesByPath(path string, skip, limit int) ([]*Branch, int, error) {
gitRepo, err := OpenRepository(path)
if err != nil {
return nil, err
return nil, 0, err
}
defer gitRepo.Close()

brs, err := gitRepo.GetBranches()
brs, countAll, err := gitRepo.GetBranches(skip, limit)
if err != nil {
return nil, err
return nil, 0, err
}

branches := make([]*Branch, len(brs))
@@ -127,7 +110,7 @@ func GetBranchesByPath(path string) ([]*Branch, error) {
}
}

return branches, nil
return branches, countAll, nil
}

// DeleteBranchOptions Option(s) for delete branch
@@ -183,3 +166,91 @@ func (repo *Repository) RemoveRemote(name string) error {
func (branch *Branch) GetCommit() (*Commit, error) {
return branch.gitRepo.GetBranchCommit(branch.Name)
}

// GetBranches returns branches from the repository, skipping skip initial branches and
// returning at most limit branches, or all branches if limit is 0.
func (repo *Repository) GetBranches(skip, limit int) ([]string, int, error) {
return callShowRef(repo.Path, BranchPrefix, "--heads", skip, limit)
}

// callShowRef return refs, if limit = 0 it will not limit
func callShowRef(repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()

go func() {
stderrBuilder := &strings.Builder{}
err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder)
if err != nil {
if stderrBuilder.Len() == 0 {
_ = stdoutWriter.Close()
return
}
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
} else {
_ = stdoutWriter.Close()
}
}()

i := 0
bufReader := bufio.NewReader(stdoutReader)
for i < skip {
_, isPrefix, err := bufReader.ReadLine()
if err == io.EOF {
return branchNames, i, nil
}
if err != nil {
return nil, 0, err
}
if !isPrefix {
i++
}
}
for limit == 0 || i < skip+limit {
// The output of show-ref is simply a list:
// <sha> SP <ref> LF
_, err := bufReader.ReadSlice(' ')
for err == bufio.ErrBufferFull {
// This shouldn't happen but we'll tolerate it for the sake of peace
_, err = bufReader.ReadSlice(' ')
}
if err == io.EOF {
return branchNames, i, nil
}
if err != nil {
return nil, 0, err
}

branchName, err := bufReader.ReadString('\n')
if err == io.EOF {
// This shouldn't happen... but we'll tolerate it for the sake of peace
return branchNames, i, nil
}
if err != nil {
return nil, i, err
}
branchName = strings.TrimPrefix(branchName, prefix)
if len(branchName) > 0 {
branchName = branchName[:len(branchName)-1]
}
branchNames = append(branchNames, branchName)
i++
}
// count all refs
for limit != 0 {
_, isPrefix, err := bufReader.ReadLine()
if err == io.EOF {
return branchNames, i, nil
}
if err != nil {
return nil, 0, err
}
if !isPrefix {
i++
}
}
return branchNames, i, nil
}

+ 28
- 23
modules/git/repo_tag.go View File

@@ -10,7 +10,6 @@ import (
"strings"

"github.com/go-git/go-git/v5/plumbing"
"github.com/mcuadros/go-version"
)

// TagPrefix tags prefix path on the repository
@@ -225,29 +224,35 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) {
return tags, nil
}

// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
var tagNames []string

tags, err := repo.gogitRepo.Tags()
if err != nil {
return nil, err
}
//// GetTags returns all tags of the repository.
//func (repo *Repository) GetTags() ([]string, error) {
// var tagNames []string
//
// tags, err := repo.gogitRepo.Tags()
// if err != nil {
// return nil, err
// }
//
// _ = tags.ForEach(func(tag *plumbing.Reference) error {
// tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
// return nil
// })
//
// version.Sort(tagNames)
//
// // Reverse order
// for i := 0; i < len(tagNames)/2; i++ {
// j := len(tagNames) - i - 1
// tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
// }
//
// return tagNames, nil
//}

_ = tags.ForEach(func(tag *plumbing.Reference) error {
tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
return nil
})

version.Sort(tagNames)

// Reverse order
for i := 0; i < len(tagNames)/2; i++ {
j := len(tagNames) - i - 1
tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
}

return tagNames, nil
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() (tags []string, err error) {
tags, _, err = callShowRef(repo.Path, TagPrefix, "--tags", 0, 0)
return
}

// GetTagType gets the type of the tag, either commit (simple) or tag (annotated)


+ 8
- 0
modules/git/utils.go View File

@@ -140,3 +140,11 @@ func ParseBool(value string) (result bool, valid bool) {
}
return intValue != 0, true
}

// ConcatenateError concatenats an error with stderr string
func ConcatenateError(err error, stderr string) error {
if len(stderr) == 0 {
return err
}
return fmt.Errorf("%w - %s", err, stderr)
}

+ 5
- 4
modules/repository/branch.go View File

@@ -23,9 +23,10 @@ func GetBranch(repo *models.Repository, branch string) (*git.Branch, error) {
return gitRepo.GetBranch(branch)
}

// GetBranches returns all the branches of a repository
func GetBranches(repo *models.Repository) ([]*git.Branch, error) {
return git.GetBranchesByPath(repo.RepoPath())
// GetBranches returns branches from the repository, skipping skip initial branches and
// returning at most limit branches, or all branches if limit is 0.
func GetBranches(repo *models.Repository, skip, limit int) ([]*git.Branch, int, error) {
return git.GetBranchesByPath(repo.RepoPath(), skip, limit)
}

// checkBranchName validates branch name with existing repository branches
@@ -36,7 +37,7 @@ func checkBranchName(repo *models.Repository, name string) error {
}
defer gitRepo.Close()

branches, err := GetBranches(repo)
branches, _, err := GetBranches(repo, 0, 0)
if err != nil {
return err
}


+ 1
- 1
routers/api/v1/repo/branch.go View File

@@ -204,7 +204,7 @@ func ListBranches(ctx *context.APIContext) {
// "200":
// "$ref": "#/responses/BranchList"

branches, err := repo_module.GetBranches(ctx.Repo.Repository)
branches, _, err := repo_module.GetBranches(ctx.Repo.Repository,0,0)
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetBranches", err)
return


+ 1
- 1
routers/repo/branch.go View File

@@ -181,7 +181,7 @@ func deleteBranch(ctx *context.Context, branchName string) error {
}

func loadBranches(ctx *context.Context) []*Branch {
rawBranches, err := repo_module.GetBranches(ctx.Repo.Repository)
rawBranches, _, err := repo_module.GetBranches(ctx.Repo.Repository, 0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return nil


+ 16
- 1
routers/repo/cloudbrain.go View File

@@ -216,7 +216,22 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
return
}

_, err := models.GetCloudbrainByName(jobName)
count, err := models.GetCloudbrainCountByUserID(ctx.User.ID)
if err != nil {
log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr("system error", tplCloudBrainNew, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainNew, &form)
return
}
}

_, err = models.GetCloudbrainByName(jobName)
if err == nil {
log.Error("the job name did already exist", ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)


+ 2
- 2
routers/repo/compare.go View File

@@ -507,7 +507,7 @@ func getBranchesForRepo(user *models.User, repo *models.Repository) (bool, []str
}
defer gitRepo.Close()

branches, err := gitRepo.GetBranches()
branches, _, err := gitRepo.GetBranches(0, 0)
if err != nil {
return false, nil, err
}
@@ -528,7 +528,7 @@ func CompareDiff(ctx *context.Context) {
}

if ctx.Data["PageIsComparePull"] == true {
headBranches, err := headGitRepo.GetBranches()
headBranches, _, err := headGitRepo.GetBranches(0,0)
if err != nil {
ctx.ServerError("GetBranches", err)
return


+ 1
- 1
routers/repo/issue.go View File

@@ -424,7 +424,7 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository, isPull boo
return nil
}

brs, err := ctx.Repo.GitRepo.GetBranches()
brs, _, err := ctx.Repo.GitRepo.GetBranches(0,0)
if err != nil {
ctx.ServerError("GetBranches", err)
return nil


+ 51
- 26
routers/repo/modelarts.go View File

@@ -115,7 +115,22 @@ func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm)
description := form.Description
flavor := form.Flavor

err := modelarts.GenerateTask(ctx, jobName, uuid, description, flavor)
count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID)
if err != nil {
log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr("system error", tplModelArtsNotebookNew, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsNotebookNew, &form)
return
}
}

err = modelarts.GenerateTask(ctx, jobName, uuid, description, flavor)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form)
return
@@ -360,18 +375,6 @@ func trainJobNewDataPrepare(ctx *context.Context) error {

outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
ctx.Data["train_url"] = outputObsPath

Branches, err := ctx.Repo.GitRepo.GetBranches()
if err != nil {
log.Error("GetBranches failed:%v", err)
ctx.ServerError("GetBranches error:", err)
return err
}
if Branches != nil {
ctx.Data["Branches"] = Branches
}

ctx.Data["BranchesCount"] = len(Branches)
ctx.Data["params"] = ""
ctx.Data["branchName"] = ctx.Repo.BranchName

@@ -442,14 +445,6 @@ func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArts
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
ctx.Data["train_url"] = outputObsPath

Branches, err := ctx.Repo.GitRepo.GetBranches()
if err != nil {
ctx.ServerError("GetBranches error:", err)
return err
}
ctx.Data["Branches"] = Branches
ctx.Data["BranchesCount"] = len(Branches)

configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
if err != nil {
ctx.ServerError("getConfigList failed:", err)
@@ -545,13 +540,13 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
ctx.Data["train_url"] = outputObsPath

Branches, err := ctx.Repo.GitRepo.GetBranches()
branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches error:", err)
return err
}

ctx.Data["branches"] = Branches
ctx.Data["branches"] = branches
ctx.Data["branch_name"] = task.BranchName
ctx.Data["description"] = task.Description
ctx.Data["boot_file"] = task.BootFile
@@ -634,12 +629,12 @@ func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrai
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
ctx.Data["train_url"] = outputObsPath

Branches, err := ctx.Repo.GitRepo.GetBranches()
branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches error:", err)
return err
}
ctx.Data["branches"] = Branches
ctx.Data["branches"] = branches
ctx.Data["description"] = form.Description
ctx.Data["dataset_name"] = task.DatasetName
ctx.Data["work_server_number"] = form.WorkServerNumber
@@ -687,6 +682,21 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
VersionCount := modelarts.VersionCount
EngineName := form.EngineName

count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID)
if err != nil {
log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"])
trainJobErrorNewDataPrepare(ctx, form)
ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
trainJobErrorNewDataPrepare(ctx, form)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobNew, &form)
return
}
}

if err := paramCheckCreateTrainJob(form); err != nil {
log.Error("paramCheckCreateTrainJob failed:(%v)", err)
trainJobErrorNewDataPrepare(ctx, form)
@@ -839,7 +849,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
return
}

err := modelarts.GenerateTrainJob(ctx, req)
err = modelarts.GenerateTrainJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error())
trainJobErrorNewDataPrepare(ctx, form)
@@ -853,6 +863,21 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
ctx.Data["PageIsTrainJob"] = true
var jobID = ctx.Params(":jobid")

count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID)
if err != nil {
log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"])
versionErrorDataPrepare(ctx, form)
ctx.RenderWithErr("system error", tplModelArtsTrainJobVersionNew, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
versionErrorDataPrepare(ctx, form)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobVersionNew, &form)
return
}
}

latestTask, err := models.GetCloudbrainByJobIDAndIsLatestVersion(jobID, modelarts.IsLatestVersion)
if err != nil {
ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err)


+ 1
- 1
services/mirror/mirror.go View File

@@ -252,7 +252,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
}
}

branches, err := repo_module.GetBranches(m.Repo)
branches, _, err := repo_module.GetBranches(m.Repo,0,0)
if err != nil {
log.Error("GetBranches: %v", err)
return nil, false


+ 1
- 1
services/pull/pull.go View File

@@ -452,7 +452,7 @@ func CloseBranchPulls(doer *models.User, repoID int64, branch string) error {

// CloseRepoBranchesPulls close all pull requests which head branches are in the given repository
func CloseRepoBranchesPulls(doer *models.User, repo *models.Repository) error {
branches, err := git.GetBranchesByPath(repo.RepoPath())
branches, _, err := git.GetBranchesByPath(repo.RepoPath(), 0, 0)
if err != nil {
return err
}


+ 5
- 2
templates/repo/modelarts/trainjob/show.tmpl View File

@@ -474,6 +474,7 @@ td, th {

<script>
console.log({{.version_list_task}})
console.log({{.}})
$('.menu .item').tab()
// $('.ui.style.accordion').accordion();

@@ -709,11 +710,13 @@ td, th {
}
function logScroll(version_name) {
let container = document.querySelector(`#log${version_name}`)
let scrollTop = container.scrollTop
let scrollHeight = container.scrollHeight
let clientHeight = container.clientHeight
if(parseInt(scrollTop) + clientHeight == scrollHeight || parseInt(scrollTop) + clientHeight +1 == scrollHeight || parseInt(scrollTop) + clientHeight - 1 == scrollHeight){
let scrollLeft = container.scrollLeft
if((parseInt(scrollTop) + clientHeight == scrollHeight || parseInt(scrollTop) + clientHeight +1 == scrollHeight || parseInt(scrollTop) + clientHeight - 1 == scrollHeight) && (scrollLeft===0)){
let end_line = $(`#log${version_name} input[name=end_line]`).val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?version_name=${version_name}&base_line=${end_line}&lines=50&order=desc`, (data) => {
if (data.Lines == 0){
@@ -736,7 +739,7 @@ td, th {
console.log(err);
});
}
if(scrollTop == 0){
if(scrollTop == 0 && scrollLeft==0){
let start_line = $(`#log${version_name} input[name=start_line]`).val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?version_name=${version_name}&base_line=${start_line}&lines=50&order=asc`, (data) => {
if (data.Lines == 0){


+ 1
- 1
templates/repo/modelarts/trainjob/version_new.tmpl View File

@@ -346,7 +346,7 @@
{{template "base/footer" .}}

<script>
let url_href = location.pathname.split('create_version')[0]
let url_href = location.pathname.split('/create_version')[0]
let url_post = location.pathname
let version_name = location.search.split('?version_name=')[1]
$("#parents_version").val(version_name)


Loading…
Cancel
Save