Browse Source

Merge branch 'V20221228' into fix-3342

fix-3342
liuzx 2 years ago
parent
commit
f267d8dc5c
55 changed files with 1527 additions and 939 deletions
  1. +11
    -0
      models/attachment.go
  2. +28
    -13
      models/cloudbrain.go
  3. +6
    -1
      models/user.go
  4. +1
    -0
      modules/auth/modelarts.go
  5. +7
    -19
      modules/cloudbrain/cloudbrain.go
  6. +7
    -5
      modules/convert/cloudbrain.go
  7. +1
    -83
      modules/modelarts/modelarts.go
  8. +1
    -0
      modules/modelarts_cd/modelarts.go
  9. +198
    -0
      modules/notebook/contentManager.go
  10. +3
    -0
      modules/repository/repo.go
  11. +15
    -0
      modules/setting/setting.go
  12. +28
    -27
      modules/structs/cloudbrain.go
  13. +8
    -2
      modules/templates/helper.go
  14. +6
    -2
      options/locale/locale_en-US.ini
  15. +8
    -4
      options/locale/locale_zh-CN.ini
  16. +34
    -65
      package-lock.json
  17. +1
    -1
      routers/admin/cloudbrains.go
  18. +5
    -1
      routers/api/v1/api.go
  19. +3
    -0
      routers/api/v1/repo/cloudbrain.go
  20. +2
    -2
      routers/api/v1/repo/cloudbrain_dashboard.go
  21. +32
    -0
      routers/api/v1/repo/modelmanage.go
  22. +20
    -0
      routers/api/v1/user/repo.go
  23. +20
    -13
      routers/repo/ai_model_manage.go
  24. +105
    -33
      routers/repo/cloudbrain.go
  25. +1
    -0
      routers/repo/dataset.go
  26. +6
    -1
      routers/repo/grampus.go
  27. +66
    -5
      routers/repo/modelarts.go
  28. +7
    -3
      routers/routes/routes.go
  29. +1
    -1
      routers/user/home.go
  30. +14
    -14
      services/cloudbrain/clear.go
  31. +2
    -2
      services/cloudbrain/cloudbrainTask/count.go
  32. +240
    -28
      services/cloudbrain/cloudbrainTask/notebook.go
  33. +2
    -2
      templates/custom/task_wait_count.tmpl
  34. +123
    -257
      templates/repo/cloudbrain/benchmark/new.tmpl
  35. +30
    -21
      templates/repo/cloudbrain/benchmark/show.tmpl
  36. +1
    -1
      templates/repo/cloudbrain/inference/new.tmpl
  37. +12
    -4
      templates/repo/cloudbrain/inference/show.tmpl
  38. +2
    -1
      templates/repo/cloudbrain/new.tmpl
  39. +1
    -1
      templates/repo/cloudbrain/trainjob/new.tmpl
  40. +22
    -3
      templates/repo/cloudbrain/trainjob/show.tmpl
  41. +2
    -3
      templates/repo/grampus/notebook/gpu/new.tmpl
  42. +16
    -3
      templates/repo/grampus/notebook/npu/new.tmpl
  43. +23
    -4
      templates/repo/grampus/trainjob/show.tmpl
  44. +12
    -4
      templates/repo/modelarts/inferencejob/show.tmpl
  45. +16
    -3
      templates/repo/modelarts/notebook/new.tmpl
  46. +24
    -4
      templates/repo/modelarts/trainjob/show.tmpl
  47. +2
    -35
      templates/repo/modelsafety/new.tmpl
  48. +29
    -0
      web_src/js/components/dataset/selectDataset.vue
  49. +1
    -196
      web_src/js/features/cloudbrainShow.js
  50. +3
    -40
      web_src/js/features/cloudrbanin.js
  51. +2
    -0
      web_src/js/features/i18nVue.js
  52. +244
    -1
      web_src/js/standalone/cloudbrainNew.js
  53. +2
    -0
      web_src/less/openi.less
  54. +11
    -2
      web_src/vuepages/apis/modules/notobook.js
  55. +60
    -29
      web_src/vuepages/pages/notebook/debug/index.vue

+ 11
- 0
models/attachment.go View File

@@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) {
return getAttachmentsByCommentID(x, commentID) return getAttachmentsByCommentID(x, commentID)
} }


func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) {
attach := &Attachment{DatasetID: datasetId, Name: fileName}
has, err := x.Get(attach)
if err != nil {
return nil, err
} else if !has {
return nil, err
}
return attach, nil
}

func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) {
attachments := make([]*Attachment, 0, 10) attachments := make([]*Attachment, 0, 10)
return attachments, e.Where("comment_id=?", commentID).Find(&attachments) return attachments, e.Where("comment_id=?", commentID).Find(&attachments)


+ 28
- 13
models/cloudbrain.go View File

@@ -60,6 +60,7 @@ const (
JobTypeModelSafety JobType = "MODELSAFETY" JobTypeModelSafety JobType = "MODELSAFETY"
JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeSnn4imagenet JobType = "SNN4IMAGENET"
JobTypeBrainScore JobType = "BRAINSCORE" JobTypeBrainScore JobType = "BRAINSCORE"
JobTypeSnn4Ecoset JobType = "SNN4ECOSET"
JobTypeTrain JobType = "TRAIN" JobTypeTrain JobType = "TRAIN"
JobTypeInference JobType = "INFERENCE" JobTypeInference JobType = "INFERENCE"


@@ -205,7 +206,7 @@ type Cloudbrain struct {
BenchmarkTypeRankLink string `xorm:"-"` BenchmarkTypeRankLink string `xorm:"-"`
StartTime timeutil.TimeStamp StartTime timeutil.TimeStamp
EndTime timeutil.TimeStamp EndTime timeutil.TimeStamp
Cleared bool `xorm:"DEFAULT false"`
Cleared bool `xorm:"DEFAULT false"`
Spec *Specification `xorm:"-"` Spec *Specification `xorm:"-"`
} }


@@ -335,6 +336,9 @@ func IsModelArtsDebugJobTerminal(status string) bool {
func IsCloudBrainOneDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool {
return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded)
} }
func IsModelBenchMarkJobType(jobType string) bool {
return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset)
}


func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) {
isActivated := result.JobStatus.CreatedTime > 0 isActivated := result.JobStatus.CreatedTime > 0
@@ -1823,7 +1827,7 @@ func QueryModelTrainJobVersionList(jobId string) ([]*Cloudbrain, int, error) {
return cloudbrains, int(len(cloudbrains)), nil return cloudbrains, int(len(cloudbrains)), nil
} }


func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
func QueryModelTrainJobList(repoId int64) ([]*Cloudbrain, int, error) {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()
var cond = builder.NewCond() var cond = builder.NewCond()
@@ -1840,14 +1844,14 @@ func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
// builder.In("type", 0, 1), // builder.In("type", 0, 1),
// ) // )


cloudbrains := make([]*CloudbrainInfo, 0)
cloudbrains := make([]*Cloudbrain, 0)
if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC"). if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC").
Find(&cloudbrains); err != nil { Find(&cloudbrains); err != nil {
return nil, 0, fmt.Errorf("Find: %v", err) return nil, 0, fmt.Errorf("Find: %v", err)
} }


keys := make(map[string]string) keys := make(map[string]string)
uniqueElements := make([]*CloudbrainInfo, 0)
uniqueElements := make([]*Cloudbrain, 0)
for _, entry := range cloudbrains { for _, entry := range cloudbrains {
if _, value := keys[entry.JobID]; !value { if _, value := keys[entry.JobID]; !value {
keys[entry.JobID] = entry.DisplayJobName keys[entry.JobID] = entry.DisplayJobName
@@ -1988,7 +1992,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) {
return getRepoCloudBrain(cb) return getRepoCloudBrain(cb)
} }


func IsCloudbrainExistByJobName(jobName string)(bool,error){
func IsCloudbrainExistByJobName(jobName string) (bool, error) {
return x.Unscoped().Exist(&Cloudbrain{ return x.Unscoped().Exist(&Cloudbrain{
JobName: jobName, JobName: jobName,
}) })
@@ -2126,6 +2130,15 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
cb := &Cloudbrain{JobName: jobName} cb := &Cloudbrain{JobName: jobName}
return getRepoCloudBrain(cb) return getRepoCloudBrain(cb)
} }
func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) {
cloudBrain := new(Cloudbrain)
has, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting,
ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain)
if has {
return cloudBrain, err
}
return nil, err
}


func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) {
@@ -2159,7 +2172,7 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) {
Find(&cloudbrains) Find(&cloudbrains)
} }


func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
func GetGPUStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0, 10) cloudbrains := make([]*Cloudbrain, 0, 10)
endTimeBefore := time.Now().Unix() - int64(days)*24*3600 endTimeBefore := time.Now().Unix() - int64(days)*24*3600
missEndTimeBefore := endTimeBefore - 24*3600 missEndTimeBefore := endTimeBefore - 24*3600
@@ -2168,29 +2181,29 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra
JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted,
ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed,
ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed).
Where("(((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false and type=0 and job_type != 'DEBUG'", missEndTimeBefore, endTimeBefore).
Where("(((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false and (type=0 or (type =2 and compute_resource='CPU/GPU')) and job_type != 'DEBUG'", missEndTimeBefore, endTimeBefore).
Limit(limit). Limit(limit).
Find(&cloudbrains) Find(&cloudbrains)
} }

/** /**
本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间 本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间
*/
func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
*/
func GetGPUStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0, 10) cloudbrains := make([]*Cloudbrain, 0, 10)
endTimeBefore := time.Now().Unix() - int64(days)*24*3600 endTimeBefore := time.Now().Unix() - int64(days)*24*3600
missEndTimeBefore := endTimeBefore - 24*3600 missEndTimeBefore := endTimeBefore - 24*3600
sql:=`SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
id, job_name, job_id,status,end_time,updated_unix,cleared id, job_name, job_id,status,end_time,updated_unix,cleared
FROM cloudbrain FROM cloudbrain
where type=0 and job_type='DEBUG'
where (type=0 or (type =2 and compute_resource='CPU/GPU')) and job_type='DEBUG'
ORDER BY job_name, updated_unix DESC) a ORDER BY job_name, updated_unix DESC) a
where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false` where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false`


return cloudbrains, x.Unscoped().SQL(sql,missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains)
return cloudbrains, x.Unscoped().SQL(sql, missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains)


} }



func UpdateCloudBrainRecordsCleared(ids []int64) error { func UpdateCloudBrainRecordsCleared(ids []int64) error {
pageSize := 150 pageSize := 150
n := len(ids) / pageSize n := len(ids) / pageSize
@@ -2609,6 +2622,7 @@ type DatasetInfo struct {
DataLocalPath string DataLocalPath string
Name string Name string
FullName string FullName string
Size int
} }


func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) {
@@ -2664,6 +2678,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn
DataLocalPath: dataLocalPath, DataLocalPath: dataLocalPath,
Name: fileName, Name: fileName,
FullName: attach.Name, FullName: attach.Name,
Size: int(attach.Size),
} }
if i == 0 { if i == 0 {
datasetNames = attach.Name datasetNames = attach.Name


+ 6
- 1
models/user.go View File

@@ -16,6 +16,7 @@ import (
"fmt" "fmt"
_ "image/jpeg" // Needed for jpeg support _ "image/jpeg" // Needed for jpeg support
"image/png" "image/png"
"math/rand"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
@@ -495,7 +496,11 @@ func (u *User) RealSizedAvatarLink(size int) string {
// may either be a sub-URL to this site, or a full URL to an external avatar // may either be a sub-URL to this site, or a full URL to an external avatar
// service. // service.
func (u *User) RelAvatarLink() string { func (u *User) RelAvatarLink() string {
return u.SizedRelAvatarLink(base.DefaultAvatarSize)
append := ""
if u.UseCustomAvatar {
append = "?" + fmt.Sprint(rand.Intn(100))
}
return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append
} }


// AvatarLink returns user avatar absolute link. // AvatarLink returns user avatar absolute link.


+ 1
- 0
modules/auth/modelarts.go View File

@@ -28,6 +28,7 @@ type CreateModelArtsNotebookForm struct {
LabelName string `form:"label_names"` LabelName string `form:"label_names"`
PreTrainModelUrl string `form:"pre_train_model_url"` PreTrainModelUrl string `form:"pre_train_model_url"`
SpecId int64 `form:"spec_id" binding:"Required"` SpecId int64 `form:"spec_id" binding:"Required"`
DatasetName string `form:"dataset_name"`
} }


func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {


+ 7
- 19
modules/cloudbrain/cloudbrain.go View File

@@ -32,10 +32,10 @@ const (
Snn4imagenetMountPath = "/snn4imagenet" Snn4imagenetMountPath = "/snn4imagenet"
BrainScoreMountPath = "/brainscore" BrainScoreMountPath = "/brainscore"
TaskInfoName = "/taskInfo" TaskInfoName = "/taskInfo"
Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt`
BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt`
SubTaskName = "task1"
Snn4imagenetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt`
BrainScoreCommand = `bash /benchmark/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt`
Snn4EcosetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt`
SubTaskName = "task1"


Success = "S000" Success = "S000"


@@ -259,20 +259,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) {
}, },
{ {
HostPath: models.StHostPath{ HostPath: models.StHostPath{
Path: req.Snn4ImageNetPath,
MountPath: Snn4imagenetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: req.BrainScorePath,
MountPath: BrainScoreMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: req.ResultPath, Path: req.ResultPath,
MountPath: ResultPath, MountPath: ResultPath,
ReadOnly: false, ReadOnly: false,
@@ -406,7 +392,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) {
} }


func IsBenchmarkJob(jobType string) bool { func IsBenchmarkJob(jobType string) bool {
return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType
return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType
} }


func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 {
@@ -675,6 +661,7 @@ func IsElementExist(s []string, str string) bool {
return false return false
} }



func GetCloudBrainByIdOrJobId(id string, initialQuery string) (*models.Cloudbrain, error) { func GetCloudBrainByIdOrJobId(id string, initialQuery string) (*models.Cloudbrain, error) {
_, err := strconv.ParseInt(id, 10, 64) _, err := strconv.ParseInt(id, 10, 64)
var job *models.Cloudbrain var job *models.Cloudbrain
@@ -709,6 +696,7 @@ type GenerateModelArtsNotebookReq struct {


ImageId string ImageId string
AutoStopDurationMs int64 AutoStopDurationMs int64
BranchName string


Spec *models.Specification Spec *models.Specification
ModelName string ModelName string


+ 7
- 5
modules/convert/cloudbrain.go View File

@@ -28,14 +28,13 @@ func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain {
BootFile: task.BootFile, BootFile: task.BootFile,
Description: task.Description, Description: task.Description,
ModelName: task.ModelName, ModelName: task.ModelName,
ModelVersion: task.ModelVersion,
CkptName: task.CkptName,
VersionName: task.VersionName,
ModelVersion: task.ModelVersion,
CkptName: task.CkptName,


StartTime: int64(task.StartTime), StartTime: int64(task.StartTime),
EndTime: int64(task.EndTime), EndTime: int64(task.EndTime),

Spec: ToSpecification(task.Spec),
Spec: ToSpecification(task.Spec),
} }
} }
func ToAttachment(attachment *models.Attachment) *api.AttachmentShow { func ToAttachment(attachment *models.Attachment) *api.AttachmentShow {
@@ -89,6 +88,9 @@ func ToDataset(dataset *models.Dataset) *api.Dataset {
} }


func ToSpecification(s *models.Specification) *api.SpecificationShow { func ToSpecification(s *models.Specification) *api.SpecificationShow {
if s == nil {
return nil
}
return &api.SpecificationShow{ return &api.SpecificationShow{
ID: s.ID, ID: s.ID,
AccCardsNum: s.AccCardsNum, AccCardsNum: s.AccCardsNum,


+ 1
- 83
modules/modelarts/modelarts.go View File

@@ -1,13 +1,9 @@
package modelarts package modelarts


import ( import (
"encoding/base64"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net/http"
"path"
"strconv" "strconv"
"strings" "strings"


@@ -239,6 +235,7 @@ func GenerateNotebook2(ctx *context.Context, req cloudbrain.GenerateModelArtsNot
ComputeResource: models.NPUResource, ComputeResource: models.NPUResource,
Image: imageName, Image: imageName,
BootFile: req.BootFile, BootFile: req.BootFile,
BranchName: req.BranchName,
Description: req.Description, Description: req.Description,
CreatedUnix: createTime, CreatedUnix: createTime,
UpdatedUnix: createTime, UpdatedUnix: createTime,
@@ -830,10 +827,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error {
task.FlavorCode = result.Flavor task.FlavorCode = result.Flavor
} }


if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" {
uploadNoteBookFile(task, result)

}
err = models.UpdateJob(task) err = models.UpdateJob(task)
if err != nil { if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
@@ -844,81 +837,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error {
return nil return nil
} }


func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) {
jupyterUrl := result.Url + "?token=" + result.Token

cookies, xsrf := getCookiesAndCsrf(jupyterUrl)
if xsrf == "" {
log.Error("browser jupyterUrl failed:%v", task.DisplayJobName)
} else {

codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath
fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile)
if err != nil {
log.Error("read jupyter file failed:%v", task.DisplayJobName, err)
}

base64Content := base64.StdEncoding.EncodeToString(fileContents)
client := getRestyClient()
uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile)
res, err := client.R().
SetCookies(cookies).
SetHeader("X-XSRFToken", xsrf).
SetBody(map[string]interface{}{
"type": "file",
"format": "base64",
"name": path.Base(task.BootFile),
"path": path.Base(task.BootFile),
"content": base64Content}).
Put(uploadUrl)
if err != nil {
log.Error("upload jupyter file failed:%v", task.DisplayJobName, err)
} else if res.StatusCode() != http.StatusCreated {
log.Error("upload jupyter file failed:%v", task.DisplayJobName, err)
}

}

}

func getJupyterBaseUrl(url string) string {
jupyterUrlLength := len(url)
baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))]
return baseUrl
}

func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) {
log.Info("jupyter url:" + jupyterUrl)
var cookies []*http.Cookie
const retryTimes = 10
for i := 0; i < retryTimes; i++ {
res, err := http.Get(jupyterUrl)
if err != nil {
log.Error("browser jupyterUrl failed.", err)
if i == retryTimes-1 {
return cookies, ""
}

} else {
cookies = res.Cookies()
xsrf := ""
for _, cookie := range cookies {
if cookie.Name == "_xsrf" {
xsrf = cookie.Value
break
}

}
if xsrf != "" {
return cookies, xsrf
}

}
}
return cookies, ""

}

func SyncTempStatusJob() { func SyncTempStatusJob() {
jobs, err := models.GetCloudBrainTempJobs() jobs, err := models.GetCloudBrainTempJobs()
if err != nil { if err != nil {


+ 1
- 0
modules/modelarts_cd/modelarts.go View File

@@ -148,6 +148,7 @@ func GenerateNotebook(ctx *context.Context, req cloudbrain.GenerateModelArtsNote
UpdatedUnix: createTime, UpdatedUnix: createTime,
Spec: req.Spec, Spec: req.Spec,
BootFile: req.BootFile, BootFile: req.BootFile,
BranchName: req.BranchName,
ModelName: req.ModelName, ModelName: req.ModelName,
ModelVersion: req.ModelVersion, ModelVersion: req.ModelVersion,
LabelName: req.LabelName, LabelName: req.LabelName,


+ 198
- 0
modules/notebook/contentManager.go View File

@@ -0,0 +1,198 @@
package notebook

import (
"crypto/tls"
"encoding/base64"
"fmt"
"io/ioutil"
"net/http"
"path"
"strings"

"github.com/go-resty/resty/v2"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/setting"

"code.gitea.io/gitea/modules/log"
)

var restyClient *resty.Client

type NotebookApiResponse struct {
Name string `json:"name"`
Path string `json:"path"`
}

type NotebookContent struct {
Url string
Path string
Cookies []*http.Cookie
Xsrf string
PathType string //file directory
Token string
}

func (c *NotebookContent) IsNotebookFileCanBrowser() bool {
if c.Xsrf == "" {
c.SetCookiesAndCsrf()
}
if c.Xsrf == "" {
log.Warn("xsrf is empty, can not broswer url:" + c.Url)
return false
}
return c.IsNoteBookContentsExist()

}

func (c *NotebookContent) SetCookiesAndCsrf() {
log.Info("jupyter url:" + c.Url)
var cookies []*http.Cookie
const retryTimes = 10
url := c.Url
if c.Token != "" {
url = c.Url + "?token=" + c.Token
}
for i := 0; i < retryTimes; i++ {
res, err := http.Get(url)
if err != nil {
log.Error("browser jupyterUrl failed.", err)
if i == retryTimes-1 {
c.Cookies = cookies
}

} else {
cookies = res.Cookies()
xsrf := ""
for _, cookie := range cookies {
if cookie.Name == "_xsrf" {
xsrf = cookie.Value
if len(cookies) > 1 {
break
}

}

}
if xsrf != "" {
c.Cookies = cookies
c.Xsrf = xsrf
}

}
}
c.Cookies = cookies

}

func (c *NotebookContent) IsNoteBookContentsExist() bool {
client := getRestyClient()
uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + "?type=" + c.PathType
res, err := client.R().
SetCookies(c.Cookies).
SetHeader("X-XSRFToken", c.Xsrf).
Get(uploadUrl)
if err != nil {
log.Warn("browser url error:"+uploadUrl, err)
return false
}
return res.StatusCode() == http.StatusOK
}

func (c *NotebookContent) UploadNoteBookFile(task *models.Cloudbrain) error {

err := c.MakeNoteBookDir()
if err != nil {
return err
}

codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath
fileContents, err := ioutil.ReadFile(codePath + "/" + c.Path)
if err != nil {
log.Error("read jupyter file failed:%v", task.DisplayJobName, err)
}

base64Content := base64.StdEncoding.EncodeToString(fileContents)
client := getRestyClient()
uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path
res, err := client.R().
SetCookies(c.Cookies).
SetHeader("X-XSRFToken", c.Xsrf).
SetBody(map[string]interface{}{
"type": "file",
"format": "base64",
"name": path.Base(c.Path),
"path": c.Path,
"content": base64Content}).
Put(uploadUrl)
if err != nil {
log.Error("upload jupyter file failed:%v", task.DisplayJobName, err)
return err
} else if res.StatusCode() != http.StatusCreated {
log.Error("upload jupyter file failed:%v, status is %s", task.DisplayJobName, res.Status(), err)
return fmt.Errorf("status:", res.StatusCode())
}
return nil
}

/**
if c.Path is a/b/c.txt
makedir a/b
if c.Path is a/b/c
makedir a/b
*/
func (c *NotebookContent) MakeNoteBookDir() error {
filePaths := strings.Split(c.Path, "/")

for i := 0; i < len(filePaths)-1; i++ {
cTemp := &NotebookContent{
Url: c.Url,
Cookies: c.Cookies,
Path: path.Join(filePaths[0 : i+1]...),
PathType: "directory",
Xsrf: c.Xsrf,
}
if !cTemp.IsNoteBookContentsExist() {

createTempDirUrl := getJupyterBaseUrl(cTemp.Url) + "api/contents/" + cTemp.Path
client := getRestyClient()
var jobResult NotebookApiResponse
res, err := client.R().
SetCookies(c.Cookies).
SetHeader("X-XSRFToken", c.Xsrf).
SetBody(map[string]interface{}{
"type": cTemp.PathType,
"path": cTemp.Path,
}).SetResult(&jobResult).
Put(createTempDirUrl)
if err != nil {
return err
}
if res.StatusCode() != http.StatusCreated {
return fmt.Errorf("status code:" + res.Status())
}

}

}
return nil
}

func getJupyterBaseUrl(url string) string {
jupyterUrlLength := len(url)
baseUrl := url
if strings.HasSuffix(url, "lab") {
baseUrl = url[0 : jupyterUrlLength-len(path.Base(url))]
}

return baseUrl
}

func getRestyClient() *resty.Client {
if restyClient == nil {
restyClient = resty.New()
restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
return restyClient
}

+ 3
- 0
modules/repository/repo.go View File

@@ -139,6 +139,9 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt
} }


repo.IsMirror = true repo.IsMirror = true
if repo.Description == "" {
repo.Description = opts.Description
}
err = models.UpdateRepository(repo, false) err = models.UpdateRepository(repo, false)
} else { } else {
repo, err = CleanUpMigrateInfo(repo) repo, err = CleanUpMigrateInfo(repo)


+ 15
- 0
modules/setting/setting.go View File

@@ -518,6 +518,7 @@ var (
MaxDatasetNum int MaxDatasetNum int
CullIdleTimeout string CullIdleTimeout string
CullInterval string CullInterval string
DebugAttachSize int


//benchmark config //benchmark config
IsBenchmarkEnabled bool IsBenchmarkEnabled bool
@@ -543,6 +544,12 @@ var (
BrainScoreName string BrainScoreName string
BrainScoreServerHost string BrainScoreServerHost string


IsSnn4EcosetEnabled bool
Snn4EcosetOwner string
Snn4EcosetName string
Snn4EcosetServerHost string
Snn4AttachmentName string

//blockchain config //blockchain config
BlockChainHost string BlockChainHost string
CommitValidDate string CommitValidDate string
@@ -1497,6 +1504,7 @@ func NewContext() {
MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5)
CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900")
CullInterval = sec.Key("CULL_INTERVAL").MustString("60") CullInterval = sec.Key("CULL_INTERVAL").MustString("60")
DebugAttachSize = sec.Key("DEBUG_ATTACH_SIZE").MustInt(20)


sec = Cfg.Section("benchmark") sec = Cfg.Section("benchmark")
IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
@@ -1522,6 +1530,13 @@ func NewContext() {
BrainScoreName = sec.Key("NAME").MustString("") BrainScoreName = sec.Key("NAME").MustString("")
BrainScoreServerHost = sec.Key("HOST").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("")


sec = Cfg.Section("snn4ecoset")
IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false)
Snn4EcosetOwner = sec.Key("OWNER").MustString("")
Snn4EcosetName = sec.Key("NAME").MustString("")
Snn4EcosetServerHost = sec.Key("HOST").MustString("")
Snn4AttachmentName = sec.Key("DATASET").MustString("")

sec = Cfg.Section("blockchain") sec = Cfg.Section("blockchain")
BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/")
CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15")


+ 28
- 27
modules/structs/cloudbrain.go View File

@@ -47,36 +47,37 @@ type CreateFileNotebookJobOption struct {
BranchName string `json:"branch_name" binding:"Required"` BranchName string `json:"branch_name" binding:"Required"`
OwnerName string `json:"owner_name" binding:"Required"` OwnerName string `json:"owner_name" binding:"Required"`
ProjectName string `json:"project_name" binding:"Required"` ProjectName string `json:"project_name" binding:"Required"`
JobId string `json:"job_id"`
} }


type Cloudbrain struct { type Cloudbrain struct {
ID int64 `json:"id"`
JobID string `json:"job_id"`
JobType string `json:"job_type"`
Type int `json:"type"`
DisplayJobName string `json:"display_job_name"`
Status string `json:"status"`
CreatedUnix int64 `json:"created_unix"`
RepoID int64 `json:"repo_id"`
Duration int64 `json:"duration"` //运行时长 单位秒
TrainJobDuration string `json:"train_job_duration"`
ImageID string `json:"image_id"` //grampus image_id
Image string `json:"image"`
Uuid string `json:"uuid"` //数据集id
DatasetName string `json:"dataset_name"`
ComputeResource string `json:"compute_resource"` //计算资源,例如npu
AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name
BranchName string `json:"branch_name"` //分支名称
Parameters string `json:"parameters"` //传给modelarts的param参数
BootFile string `json:"boot_file"` //启动文件
Description string `json:"description"` //描述
ModelName string `json:"model_name"` //模型名称
ModelVersion string `json:"model_version"` //模型版本
CkptName string `json:"ckpt_name"` //权重文件名称
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
Spec *SpecificationShow `json:"spec"`
ID int64 `json:"id"`
JobID string `json:"job_id"`
JobType string `json:"job_type"`
Type int `json:"type"`
DisplayJobName string `json:"display_job_name"`
Status string `json:"status"`
CreatedUnix int64 `json:"created_unix"`
RepoID int64 `json:"repo_id"`
Duration int64 `json:"duration"` //运行时长 单位秒
TrainJobDuration string `json:"train_job_duration"`
ImageID string `json:"image_id"` //grampus image_id
Image string `json:"image"`
Uuid string `json:"uuid"` //数据集id
DatasetName string `json:"dataset_name"`
ComputeResource string `json:"compute_resource"` //计算资源,例如npu
AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name
BranchName string `json:"branch_name"` //分支名称
Parameters string `json:"parameters"` //传给modelarts的param参数
BootFile string `json:"boot_file"` //启动文件
Description string `json:"description"` //描述
ModelName string `json:"model_name"` //模型名称
ModelVersion string `json:"model_version"` //模型版本
CkptName string `json:"ckpt_name"` //权重文件名称
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
VersionName string `json:"version_name"`
Spec *SpecificationShow `json:"spec"`
} }


type SpecificationShow struct { type SpecificationShow struct {


+ 8
- 2
modules/templates/helper.go View File

@@ -98,11 +98,15 @@ func NewFuncMap() []template.FuncMap {
"AllowedReactions": func() []string { "AllowedReactions": func() []string {
return setting.UI.Reactions return setting.UI.Reactions
}, },
"DebugAttachSize": func() int {
return setting.DebugAttachSize * 1000 * 1000 * 1000
},
"AvatarLink": models.AvatarLink, "AvatarLink": models.AvatarLink,
"Safe": Safe, "Safe": Safe,
"SafeJS": SafeJS, "SafeJS": SafeJS,
"Str2html": Str2html, "Str2html": Str2html,
"subOne": subOne, "subOne": subOne,
"addOne": addOne,
"TimeSince": timeutil.TimeSince, "TimeSince": timeutil.TimeSince,
"TimeSinceUnix": timeutil.TimeSinceUnix, "TimeSinceUnix": timeutil.TimeSinceUnix,
"TimeSinceUnix1": timeutil.TimeSinceUnix1, "TimeSinceUnix1": timeutil.TimeSinceUnix1,
@@ -153,7 +157,7 @@ func NewFuncMap() []template.FuncMap {
"EscapePound": func(str string) string { "EscapePound": func(str string) string {
return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str) return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str)
}, },
"IpynbBool":func(str string) bool{
"IpynbBool": func(str string) bool {
return strings.Contains(str, ".ipynb") return strings.Contains(str, ".ipynb")
}, },
"nl2br": func(text string) template.HTML { "nl2br": func(text string) template.HTML {
@@ -470,7 +474,9 @@ func Str2html(raw string) template.HTML {
func subOne(length int) int { func subOne(length int) int {
return length - 1 return length - 1
} }

func addOne(length int64) int64 {
return length + 1
}
// Escape escapes a HTML string // Escape escapes a HTML string
func Escape(raw string) string { func Escape(raw string) string {
return html.EscapeString(raw) return html.EscapeString(raw)


+ 6
- 2
options/locale/locale_en-US.ini View File

@@ -577,6 +577,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count
static.CloudBrainRunTime=CloudBrain Run Time static.CloudBrainRunTime=CloudBrain Run Time
static.CommitDatasetNum=Commit Dataset Count static.CommitDatasetNum=Commit Dataset Count
static.CommitModelCount=Commit Model Count static.CommitModelCount=Commit Model Count
static.ModelConvertCount=Model Convert Count
static.UserIndex=Normalized user index static.UserIndex=Normalized user index
static.UserIndexPrimitive=User Index static.UserIndexPrimitive=User Index
static.countdate=Count Date static.countdate=Count Date
@@ -1061,7 +1062,9 @@ model_rename=Duplicate model name, please modify model name.
notebook_file_not_exist=Notebook file does not exist. notebook_file_not_exist=Notebook file does not exist.
notebook_select_wrong=Please select a Notebook(.ipynb) file first. notebook_select_wrong=Please select a Notebook(.ipynb) file first.
notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. notebook_file_no_right=You have no right to access the Notebook(.ipynb) file.
notebook_repo_conflict=The files in different branches of the same repository can not run together.
debug_again_fail=Fail to restart debug task, please try again later. debug_again_fail=Fail to restart debug task, please try again later.
debug_again_fail_forever=The task was scheduled failed last time, can not restart.


date=Date date=Date
repo_add=Project Increment repo_add=Project Increment
@@ -1079,6 +1082,7 @@ delete=Delete
more=More more=More
gpu_type_all=All gpu_type_all=All
model_download=Model Download model_download=Model Download
all_result_download=All result download
submit_image=Submit Image submit_image=Submit Image
modify_image=Modify Image modify_image=Modify Image
image_exist=Image name has been used, please use a new one. image_exist=Image name has been used, please use a new one.
@@ -1091,8 +1095,8 @@ image_delete_fail=Failed to delete image, please try again later.
image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image? image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image?
download=Download download=Download
score=Score score=Score
wait_count_start = There are currently
wait_count_end = tasks queued
wait_count_start = Your current queue position is
wait_count_end =
file_limit_100 = Display up to 100 files or folders in a single directory file_limit_100 = Display up to 100 files or folders in a single directory
images.name = Image Tag images.name = Image Tag
images.name_placerholder = Please enter the image name images.name_placerholder = Please enter the image name


+ 8
- 4
options/locale/locale_zh-CN.ini View File

@@ -581,6 +581,7 @@ static.CloudBrainTaskNum=云脑任务数
static.CloudBrainRunTime=云脑运行时间(小时) static.CloudBrainRunTime=云脑运行时间(小时)
static.CommitDatasetNum=上传(提交)数据集文件数 static.CommitDatasetNum=上传(提交)数据集文件数
static.CommitModelCount=提交模型数 static.CommitModelCount=提交模型数
static.ModelConvertCount=模型转换数
static.UserIndex=归一化用户指数 static.UserIndex=归一化用户指数
static.UserIndexPrimitive=用户指数 static.UserIndexPrimitive=用户指数
static.countdate=系统统计时间 static.countdate=系统统计时间
@@ -1060,7 +1061,9 @@ model_rename=模型名称重复,请修改模型名称
notebook_file_not_exist=Notebook文件不存在。 notebook_file_not_exist=Notebook文件不存在。
notebook_select_wrong=请先选择Notebook(.ipynb)文件。 notebook_select_wrong=请先选择Notebook(.ipynb)文件。
notebook_file_no_right=您没有这个Notebook文件的读权限。 notebook_file_no_right=您没有这个Notebook文件的读权限。
notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。
debug_again_fail=再次调试失败,请稍后再试。 debug_again_fail=再次调试失败,请稍后再试。
debug_again_fail_forever=这个任务之前没有调度成功,不能再次调试。


date=日期 date=日期
repo_add=新增项目 repo_add=新增项目
@@ -1078,6 +1081,7 @@ delete=删除
more=更多 more=更多
gpu_type_all=全部 gpu_type_all=全部
model_download=结果下载 model_download=结果下载
all_result_download=全部结果下载
submit_image=提交镜像 submit_image=提交镜像
modify_image=修改镜像 modify_image=修改镜像
image_exist=镜像Tag已被使用,请修改镜像Tag。 image_exist=镜像Tag已被使用,请修改镜像Tag。
@@ -1090,8 +1094,8 @@ image_delete_fail=删除镜像失败,请稍后再试。
image_overwrite=您已经提交过相同名称的镜像,您确定要覆盖原来提交的镜像吗? image_overwrite=您已经提交过相同名称的镜像,您确定要覆盖原来提交的镜像吗?
download=模型下载 download=模型下载
score=评分 score=评分
wait_count_start = 当前有
wait_count_end = 个任务正在排队
wait_count_start = 您当前排队位置是第
wait_count_end =
file_limit_100 = 单目录下最多显示100个文件或文件夹 file_limit_100 = 单目录下最多显示100个文件或文件夹
images.name = 镜像Tag images.name = 镜像Tag
images.name_placerholder = 请输入镜像Tag images.name_placerholder = 请输入镜像Tag
@@ -1672,7 +1676,7 @@ issues.action_assignee_no_select=未指派
issues.opened_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s创建 issues.opened_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s创建
pulls.merged_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s 合并 pulls.merged_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s 合并
pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并
issues.closed_by=按 <a href="%[2]s">%[3]s</a> 关闭%[1]s
issues.closed_by=由 <a href="%[2]s">%[3]s</a> 创建,被关闭于 %[1]s
issues.opened_by_fake=由 %[2]s 于 %[1]s创建 issues.opened_by_fake=由 %[2]s 于 %[1]s创建
issues.closed_by_fake=通过 %[2]s 关闭 %[1]s issues.closed_by_fake=通过 %[2]s 关闭 %[1]s
issues.previous=上一页 issues.previous=上一页
@@ -3325,7 +3329,7 @@ Stopped_failed=任务停止失败,请稍后再试。
Stopped_success_update_status_fail=任务停止成功,状态及运行时间更新失败。 Stopped_success_update_status_fail=任务停止成功,状态及运行时间更新失败。
load_code_failed=代码加载失败,请确认选择了正确的分支。 load_code_failed=代码加载失败,请确认选择了正确的分支。


error.debug_datasetsize = 数据集大小超过限制('%d'GB)
error.dataset_select = 数据集选择错误:数量超过限制或者有同名数据集 error.dataset_select = 数据集选择错误:数量超过限制或者有同名数据集
new_train_gpu_tooltips = 训练脚本存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,预训练模型存放在运行参数 <strong style="color:#010101">%s</strong> 中,训练输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。 new_train_gpu_tooltips = 训练脚本存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,预训练模型存放在运行参数 <strong style="color:#010101">%s</strong> 中,训练输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。
new_debug_gpu_tooltips = 项目代码存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,选择的模型存储在 <strong style="color:#010101">%s</strong> 中,调试输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。 new_debug_gpu_tooltips = 项目代码存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,选择的模型存储在 <strong style="color:#010101">%s</strong> 中,调试输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。


+ 34
- 65
package-lock.json View File

@@ -1982,28 +1982,6 @@
"object.assign": "^4.1.0" "object.assign": "^4.1.0"
} }
}, },
"babel-polyfill": {
"version": "6.26.0",
"resolved": "https://registry.npm.taobao.org/babel-polyfill/download/babel-polyfill-6.26.0.tgz",
"integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=",
"requires": {
"babel-runtime": "^6.26.0",
"core-js": "^2.5.0",
"regenerator-runtime": "^0.10.5"
},
"dependencies": {
"core-js": {
"version": "2.6.12",
"resolved": "https://registry.npm.taobao.org/core-js/download/core-js-2.6.12.tgz?cache=0&sync_timestamp=1611040749668&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcore-js%2Fdownload%2Fcore-js-2.6.12.tgz",
"integrity": "sha1-2TM9+nsGXjR8xWgiGdb2kIWcwuw="
},
"regenerator-runtime": {
"version": "0.10.5",
"resolved": "https://registry.npm.taobao.org/regenerator-runtime/download/regenerator-runtime-0.10.5.tgz",
"integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg="
}
}
},
"babel-runtime": { "babel-runtime": {
"version": "6.26.0", "version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
@@ -2194,9 +2172,9 @@
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
}, },
"blueimp-md5": { "blueimp-md5": {
"version": "2.18.0",
"resolved": "https://registry.npm.taobao.org/blueimp-md5/download/blueimp-md5-2.18.0.tgz",
"integrity": "sha1-EVK+EzXwxrORHtnjbbVPPmrFKTU="
"version": "2.19.0",
"resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz",
"integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w=="
}, },
"bn.js": { "bn.js": {
"version": "5.1.1", "version": "5.1.1",
@@ -3536,6 +3514,11 @@
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz",
"integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI=" "integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI="
}, },
"dayjs": {
"version": "1.10.7",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz",
"integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig=="
},
"de-indent": { "de-indent": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz",
@@ -4152,25 +4135,29 @@
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
}, },
"esdk-obs-browserjs": { "esdk-obs-browserjs": {
"version": "3.20.7",
"resolved": "https://registry.npm.taobao.org/esdk-obs-browserjs/download/esdk-obs-browserjs-3.20.7.tgz",
"integrity": "sha1-vhziRlKEhW3PgZPl0DyX68bJI0s=",
"version": "3.22.3",
"resolved": "https://registry.npmjs.org/esdk-obs-browserjs/-/esdk-obs-browserjs-3.22.3.tgz",
"integrity": "sha512-MATZXp0FwjPtKG9tpdfURa3koUarR/ev+tbO0oUKgj0GRt0798ZxmfCvYvRpgNst4w1ht4E79ikD4H40UYLgPA==",
"requires": { "requires": {
"axios": "^0.19.0",
"babel-polyfill": "^6.26.0",
"blueimp-md5": "^2.10.0",
"js-base64": "^2.3.2",
"jssha": "^2.3.1",
"urijs": "^1.19.1"
"axios": "^0.26.1",
"blueimp-md5": "^2.18.0",
"js-base64": "^3.7.1",
"jssha": "^3.2.0",
"urijs": "^1.19.7"
}, },
"dependencies": { "dependencies": {
"axios": { "axios": {
"version": "0.19.2",
"resolved": "https://registry.npm.taobao.org/axios/download/axios-0.19.2.tgz?cache=0&sync_timestamp=1608609215811&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faxios%2Fdownload%2Faxios-0.19.2.tgz",
"integrity": "sha1-PqNsXYgY0NX4qKl6bTa4bNwAyyc=",
"version": "0.26.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
"integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
"requires": { "requires": {
"follow-redirects": "1.5.10"
"follow-redirects": "^1.14.8"
} }
},
"js-base64": {
"version": "3.7.3",
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.3.tgz",
"integrity": "sha512-PAr6Xg2jvd7MCR6Ld9Jg3BmTcjYsHEBx1VlwEwULb/qowPf5VD9kEMagj23Gm7JRnSvE/Da/57nChZjnvL8v6A=="
} }
} }
}, },
@@ -5382,27 +5369,9 @@
} }
}, },
"follow-redirects": { "follow-redirects": {
"version": "1.5.10",
"resolved": "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.5.10.tgz?cache=0&sync_timestamp=1611606737937&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.5.10.tgz",
"integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=",
"requires": {
"debug": "=3.1.0"
},
"dependencies": {
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz?cache=0&sync_timestamp=1607566533140&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-3.1.0.tgz",
"integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
}, },
"fomantic-ui": { "fomantic-ui": {
"version": "2.8.4", "version": "2.8.4",
@@ -7884,9 +7853,9 @@
} }
}, },
"jssha": { "jssha": {
"version": "2.4.2",
"resolved": "https://registry.npm.taobao.org/jssha/download/jssha-2.4.2.tgz",
"integrity": "sha1-2VCwlWNJKL1rK9odQtqaOnYtZek="
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.0.tgz",
"integrity": "sha512-w9OtT4ALL+fbbwG3gw7erAO0jvS5nfvrukGPMWIAoea359B26ALXGpzy4YJSp9yGnpUvuvOw1nSjSoHDfWSr1w=="
}, },
"just-debounce": { "just-debounce": {
"version": "1.0.0", "version": "1.0.0",
@@ -14400,9 +14369,9 @@
} }
}, },
"urijs": { "urijs": {
"version": "1.19.6",
"resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.6.tgz",
"integrity": "sha512-eSXsXZ2jLvGWeLYlQA3Gh36BcjF+0amo92+wHPyN1mdR8Nxf75fuEuYTd9c0a+m/vhCjRK0ESlE9YNLW+E1VEw=="
"version": "1.19.11",
"resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz",
"integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ=="
}, },
"urix": { "urix": {
"version": "0.1.0", "version": "0.1.0",


+ 1
- 1
routers/admin/cloudbrains.go View File

@@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 5
- 1
routers/api/v1/api.go View File

@@ -549,6 +549,8 @@ func RegisterRoutes(m *macaron.Macaron) {


}, reqToken()) }, reqToken())


m.Get("/compute-nodes", reqToken(), user.GetComputeNodes)

// Notifications // Notifications
m.Group("/notifications", func() { m.Group("/notifications", func() {
m.Combo(""). m.Combo("").
@@ -745,7 +747,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/file_notebook", func() { m.Group("/file_notebook", func() {
m.Get("", repo.GetFileNoteBookInfo) m.Get("", repo.GetFileNoteBookInfo)
m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook)
m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus)
}) })


m.Group("/repos", func() { m.Group("/repos", func() {
@@ -1024,6 +1026,8 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/query_model_byName", repo.QueryModelByName) m.Get("/query_model_byName", repo.QueryModelByName)
m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_model_for_predict", repo.QueryModelListForPredict)
m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict)
m.Get("/query_train_job", repo.QueryTrainJobList)
m.Get("/query_train_job_version", repo.QueryTrainJobVersionList)
m.Get("/query_train_model", repo.QueryTrainModelList) m.Get("/query_train_model", repo.QueryTrainModelList)
m.Post("/create_model_convert", repo.CreateModelConvert) m.Post("/create_model_convert", repo.CreateModelConvert)
m.Post("/convert_stop", repo.StopModelConvert) m.Post("/convert_stop", repo.StopModelConvert)


+ 3
- 0
routers/api/v1/repo/cloudbrain.go View File

@@ -110,6 +110,9 @@ func GeneralCloudBrainJobStop(ctx *context.APIContext) {
func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) {
cloudbrainTask.FileNotebookCreate(ctx.Context, option) cloudbrainTask.FileNotebookCreate(ctx.Context, option)
} }
func FileNoteBookStatus(ctx *context.APIContext, option api.CreateFileNotebookJobOption) {
cloudbrainTask.FileNotebookStatus(ctx.Context, option)
}


func GetFileNoteBookInfo(ctx *context.APIContext) { func GetFileNoteBookInfo(ctx *context.APIContext) {
//image description spec description waiting count //image description spec description waiting count


+ 2
- 2
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -623,7 +623,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
} }


jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark),
string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)}
string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)}
for _, v := range jobTypeList { for _, v := range jobTypeList {
if _, ok := cloudOneJobTypeRes[v]; !ok { if _, ok := cloudOneJobTypeRes[v]; !ok {
cloudOneJobTypeRes[v] = 0 cloudOneJobTypeRes[v] = 0
@@ -730,7 +730,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 32
- 0
routers/api/v1/repo/modelmanage.go View File

@@ -4,8 +4,10 @@ import (
"net/http" "net/http"


"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
routerRepo "code.gitea.io/gitea/routers/repo" routerRepo "code.gitea.io/gitea/routers/repo"
) )


@@ -54,6 +56,21 @@ func QueryModelListForPredict(ctx *context.APIContext) {
routerRepo.QueryModelListForPredict(ctx.Context) routerRepo.QueryModelListForPredict(ctx.Context)
} }


func QueryTrainJobList(ctx *context.APIContext) {
result, err := routerRepo.QueryTrainJobListApi(ctx.Context)
if err != nil {
log.Info("query error." + err.Error())
ctx.JSON(http.StatusOK, nil)
} else {
re := make([]*api.Cloudbrain, 0)
for _, task := range result {
conRe := convert.ToCloudBrain(task)
re = append(re, conRe)
}
ctx.JSON(http.StatusOK, re)
}
}

func QueryTrainModelList(ctx *context.APIContext) { func QueryTrainModelList(ctx *context.APIContext) {
result, err := routerRepo.QueryTrainModelFileById(ctx.Context) result, err := routerRepo.QueryTrainModelFileById(ctx.Context)
if err != nil { if err != nil {
@@ -63,6 +80,21 @@ func QueryTrainModelList(ctx *context.APIContext) {
ctx.JSON(http.StatusOK, re) ctx.JSON(http.StatusOK, re)
} }


func QueryTrainJobVersionList(ctx *context.APIContext) {
result, err := routerRepo.QueryTrainJobVersionListApi(ctx.Context)
if err != nil {
log.Info("query error." + err.Error())
ctx.JSON(http.StatusOK, nil)
} else {
re := make([]*api.Cloudbrain, 0)
for _, task := range result {
conRe := convert.ToCloudBrain(task)
re = append(re, conRe)
}
ctx.JSON(http.StatusOK, re)
}
}

func convertFileFormat(result []storage.FileInfo) []FileInfo { func convertFileFormat(result []storage.FileInfo) []FileInfo {
re := make([]FileInfo, 0) re := make([]FileInfo, 0)
if result != nil { if result != nil {


+ 20
- 0
routers/api/v1/user/repo.go View File

@@ -5,6 +5,7 @@
package user package user


import ( import (
"code.gitea.io/gitea/modules/modelarts"
"net/http" "net/http"


"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
@@ -146,3 +147,22 @@ func ListOrgRepos(ctx *context.APIContext) {


listUserRepos(ctx, ctx.Org.Organization, ctx.IsSigned) listUserRepos(ctx, ctx.Org.Organization, ctx.IsSigned)
} }

func GetComputeNodes(ctx *context.APIContext) {
taskeType := ctx.QueryInt("type")
if taskeType == 2 {
ctx.JSON(http.StatusOK, []int{1})
} else {
modelarts.InitMultiNode()
if modelarts.MultiNodeConfig != nil {
for _, info := range modelarts.MultiNodeConfig.Info {
if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, ctx.User.ID); isInOrg {
ctx.JSON(http.StatusOK, info.Node)
return
}
}
}
ctx.JSON(http.StatusOK, []int{1})
}

}

+ 20
- 13
routers/repo/ai_model_manage.go View File

@@ -2,7 +2,6 @@ package repo


import ( import (
"archive/zip" "archive/zip"
"code.gitea.io/gitea/services/repository"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -12,6 +11,8 @@ import (
"regexp" "regexp"
"strings" "strings"


"code.gitea.io/gitea/services/repository"

"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@@ -710,36 +711,42 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con
} }


func QueryTrainJobVersionList(ctx *context.Context) { func QueryTrainJobVersionList(ctx *context.Context) {

VersionListTasks, err := QueryTrainJobVersionListApi(ctx)
if err != nil {
ctx.JSON(200, nil)
} else {
ctx.JSON(200, VersionListTasks)
}
}

func QueryTrainJobVersionListApi(ctx *context.Context) ([]*models.Cloudbrain, error) {
log.Info("query train job version list. start.") log.Info("query train job version list. start.")
JobID := ctx.Query("jobId") JobID := ctx.Query("jobId")
if JobID == "" { if JobID == "" {
JobID = ctx.Query("JobId") JobID = ctx.Query("JobId")
} }

VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID)

log.Info("query return count=" + fmt.Sprint(count)) log.Info("query return count=" + fmt.Sprint(count))


return VersionListTasks, err
}

func QueryTrainJobList(ctx *context.Context) {
VersionListTasks, err := QueryTrainJobListApi(ctx)
if err != nil { if err != nil {
ctx.ServerError("QueryTrainJobList:", err)
ctx.JSON(200, nil)
} else { } else {
ctx.JSON(200, VersionListTasks) ctx.JSON(200, VersionListTasks)
} }
} }


func QueryTrainJobList(ctx *context.Context) {
log.Info("query train job list. start.")
func QueryTrainJobListApi(ctx *context.Context) ([]*models.Cloudbrain, error) {
repoId := ctx.QueryInt64("repoId") repoId := ctx.QueryInt64("repoId")

VersionListTasks, count, err := models.QueryModelTrainJobList(repoId) VersionListTasks, count, err := models.QueryModelTrainJobList(repoId)
log.Info("query return count=" + fmt.Sprint(count)) log.Info("query return count=" + fmt.Sprint(count))


if err != nil {
ctx.ServerError("QueryTrainJobList:", err)
} else {
ctx.JSON(200, VersionListTasks)
}

return VersionListTasks, err
} }


func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) { func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) {


+ 105
- 33
routers/repo/cloudbrain.go View File

@@ -285,7 +285,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
} }
var datasetInfos map[string]models.DatasetInfo var datasetInfos map[string]models.DatasetInfo
var datasetNames string var datasetNames string
//var
var attachSize int
if uuids != "" { if uuids != "" {
datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids)
if err != nil { if err != nil {
@@ -294,6 +294,18 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return return
} }

if jobType == string(models.JobTypeDebug) {
for _, infos := range datasetInfos {
attachSize += infos.Size
}
if attachSize > int(setting.DebugAttachSize*1000*1000*1000) {
log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize)) // GB
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", int(setting.DebugAttachSize*1000*1000*1000)), tpl, &form)
return
}
}
} }


command := cloudbrain.GetCloudbrainDebugCommand() command := cloudbrain.GetCloudbrainDebugCommand()
@@ -387,7 +399,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
req.ModelVersion = form.ModelVersion req.ModelVersion = form.ModelVersion
req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
req.PreTrainModelUrl = form.PreTrainModelUrl req.PreTrainModelUrl = form.PreTrainModelUrl

} }


_, err = cloudbrain.GenerateTask(req) _, err = cloudbrain.GenerateTask(req)
@@ -735,6 +746,7 @@ func CloudBrainRestart(ctx *context.Context) {
}) })
} }



func hasDatasetDeleted(task *models.Cloudbrain) bool { func hasDatasetDeleted(task *models.Cloudbrain) bool {
if task.Uuid == "" { if task.Uuid == "" {
return false return false
@@ -843,7 +855,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo


if task.JobType == string(models.JobTypeBenchmark) { if task.JobType == string(models.JobTypeBenchmark) {
task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
} else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
} else if models.IsModelBenchMarkJobType(task.JobType) {
task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
task.BenchmarkTypeName = task.JobType task.BenchmarkTypeName = task.JobType
ctx.Data["BenchmarkTypeName"] = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType
@@ -911,10 +923,13 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo
func CloudBrainDebug(ctx *context.Context) { func CloudBrainDebug(ctx *context.Context) {
task := ctx.Cloudbrain task := ctx.Cloudbrain
debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
if task.BootFile != "" {
ctx.Redirect(getFileUrl(debugUrl, task.BootFile))


if ctx.QueryTrim("file") != "" {
ctx.Redirect(getFileUrl(debugUrl, ctx.QueryTrim("file")))
} else { } else {
if task.BootFile != "" {
go cloudbrainTask.UploadNotebookFiles(task)
}
ctx.Redirect(debugUrl) ctx.Redirect(debugUrl)
} }


@@ -1638,6 +1653,21 @@ func CloudBrainDownloadModel(ctx *context.Context) {
ctx.Resp.Header().Set("Cache-Control", "max-age=0") ctx.Resp.Header().Set("Cache-Control", "max-age=0")
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
} }

func CloudBrainDownloadMultiModel(ctx *context.Context) {
parentDir := ctx.Query("parentDir")
jobName := ctx.Query("jobName")
filePath := "jobs/" + jobName + "/model/" + parentDir
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
if err == nil {
returnFileName := jobName + ".zip"
MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
}

func CloudBrainDownloadInferenceResult(ctx *context.Context) { func CloudBrainDownloadInferenceResult(ctx *context.Context) {
parentDir := ctx.Query("parentDir") parentDir := ctx.Query("parentDir")
fileName := ctx.Query("fileName") fileName := ctx.Query("fileName")
@@ -1674,6 +1704,8 @@ func GetRate(ctx *context.Context) {
ctx.Redirect(setting.Snn4imagenetServerHost) ctx.Redirect(setting.Snn4imagenetServerHost)
} else if job.JobType == string(models.JobTypeBrainScore) { } else if job.JobType == string(models.JobTypeBrainScore) {
ctx.Redirect(setting.BrainScoreServerHost) ctx.Redirect(setting.BrainScoreServerHost)
} else if job.JobType == string(models.JobTypeSnn4Ecoset) {
ctx.Redirect(setting.Snn4EcosetServerHost)
} else { } else {
log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
} }
@@ -2146,7 +2178,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) {
} }


var jobTypes []string var jobTypes []string
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset), string(models.JobTypeModelSafety))
ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{ ListOptions: models.ListOptions{
Page: page, Page: page,
@@ -2179,14 +2211,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) {
ciTasks[i].BenchmarkTypeName = "" ciTasks[i].BenchmarkTypeName = ""
if ciTasks[i].JobType == string(models.JobTypeBenchmark) { if ciTasks[i].JobType == string(models.JobTypeBenchmark) {
ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
} else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) {
} else if models.IsModelBenchMarkJobType(ciTasks[i].JobType) {
ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType


if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) {
ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost
} else {
} else if ciTasks[i].JobType == string(models.JobTypeBrainScore) {
ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost
} else {
ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost
} }


} }
@@ -2536,7 +2570,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
displayJobName := form.DisplayJobName displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName) jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
image := form.Image image := form.Image
uuid := form.Attachment
jobType := form.JobType jobType := form.JobType
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
branchName := cloudbrain.DefaultBranchName branchName := cloudbrain.DefaultBranchName
@@ -2578,7 +2611,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
return return
} }


if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
if !models.IsModelBenchMarkJobType(jobType) {
log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx, jobType) cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr("jobtype error", tpl, &form) ctx.RenderWithErr("jobtype error", tpl, &form)
@@ -2607,29 +2640,41 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
mkModelPath(modelPath) mkModelPath(modelPath)
uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")


snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name)
uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/")
command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description))
downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, benchmarkPath, "", "", ctx.User.Name)
uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description))


} }
benchmarkChildTypeID := 0 benchmarkChildTypeID := 0
brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name)
uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/")
downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, benchmarkPath, "", "", ctx.User.Name)
uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
benchmarkChildTypeID = form.BenchmarkChildTypeID benchmarkChildTypeID = form.BenchmarkChildTypeID
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description))
} }
var uuid string
var datasetInfos map[string]models.DatasetInfo
var datasetNames string
if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) {
downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, benchmarkPath, "", "", ctx.User.Name)
uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
command = fmt.Sprintf(cloudbrain.Snn4EcosetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description))

attachment, err := getEcosetAttachment()
if err != nil {
log.Error("load benchmark code failed", err)
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
return
}
uuid = attachment.UUID
datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid)


datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return
} }

spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
JobType: models.JobTypeBenchmark, JobType: models.JobTypeBenchmark,
ComputeResource: models.GPU, ComputeResource: models.GPU,
@@ -2661,8 +2706,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
JobType: jobType, JobType: jobType,
Description: form.Description, Description: form.Description,
BranchName: branchName, BranchName: branchName,
@@ -2674,6 +2717,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
Spec: spec, Spec: spec,
} }
if form.ModelName != "" {
req.ModelName = form.ModelName
req.LabelName = form.LabelName
req.CkptName = form.CkptName
req.ModelVersion = form.ModelVersion
req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
req.PreTrainModelUrl = form.PreTrainModelUrl
}


_, err = cloudbrain.GenerateTask(req) _, err = cloudbrain.GenerateTask(req)
if err != nil { if err != nil {
@@ -2685,6 +2736,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
} }


func getEcosetAttachment() (*models.Attachment, error) {
ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName)
if err != nil {
return nil, err
}

datasetInfo, err := models.GetDatasetByRepo(ecosetRepo)
if err != nil {
return nil, err
}

return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID)

}

func getBrainRegion(benchmarkChildTypeID int) string { func getBrainRegion(benchmarkChildTypeID int) string {
values := []string{"V1", "V2", "V4", "IT"} values := []string{"V1", "V2", "V4", "IT"}
return values[benchmarkChildTypeID] return values[benchmarkChildTypeID]
@@ -2745,18 +2811,24 @@ func InferenceCloudBrainJobShow(ctx *context.Context) {
cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference) cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference)
} }


func DownloadInferenceResultFile(ctx *context.Context) {
func DownloadGPUInferenceResultFile(ctx *context.Context) {
var jobID = ctx.Params(":jobid") var jobID = ctx.Params(":jobid")
var versionName = ctx.Query("version_name")
task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil { if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
return return
} }

allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl)
returnFileName := task.DisplayJobName + ".zip"
MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile)
parentDir := ctx.Query("parentDir")
filePath := "jobs/" + task.JobName + "/result/" + parentDir
log.Info("prefix=" + filePath)
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
if err == nil {
returnFileName := task.DisplayJobName + ".zip"
MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
} }


func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) { func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) {


+ 1
- 0
routers/repo/dataset.go View File

@@ -373,6 +373,7 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
} }


data, err := json.Marshal(datasets) data, err := json.Marshal(datasets)
log.Info("datakey", string(data))
if err != nil { if err != nil {
log.Error("json.Marshal failed:", err.Error()) log.Error("json.Marshal failed:", err.Error())
ctx.JSON(200, map[string]string{ ctx.JSON(200, map[string]string{


+ 6
- 1
routers/repo/grampus.go View File

@@ -1239,7 +1239,7 @@ func GrampusTrainJobShow(ctx *context.Context) {
return return
} }
task.ContainerIp = "" task.ContainerIp = ""
task.User, _ = models.GetUserByID(task.UserID)
if task.DeletedAt.IsZero() { //normal record if task.DeletedAt.IsZero() { //normal record
result, err := grampus.GetJob(task.JobID) result, err := grampus.GetJob(task.JobID)
if err != nil { if err != nil {
@@ -1308,6 +1308,7 @@ func GrampusTrainJobShow(ctx *context.Context) {
taskList := make([]*models.Cloudbrain, 0) taskList := make([]*models.Cloudbrain, 0)
taskList = append(taskList, task) taskList = append(taskList, task)
prepareSpec4Show(ctx, task) prepareSpec4Show(ctx, task)
ctx.Data["version_list_task"] = taskList ctx.Data["version_list_task"] = taskList
ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false) ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)
ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
@@ -1682,6 +1683,10 @@ func GrampusNotebookRestart(ctx *context.Context) {
if res.GrampusResult.ErrorCode != 0 || res.NewId == "" { if res.GrampusResult.ErrorCode != 0 || res.NewId == "" {
log.Error("ManageNotebook2 failed:" + res.GrampusResult.ErrorMsg) log.Error("ManageNotebook2 failed:" + res.GrampusResult.ErrorMsg)
errorMsg = ctx.Tr("repo.debug_again_fail") errorMsg = ctx.Tr("repo.debug_again_fail")
if res.GrampusResult.ErrorCode == 5005 {
errorMsg = ctx.Tr("repo.debug_again_fail_forever")
}

break break
} }




+ 66
- 5
routers/repo/modelarts.go View File

@@ -218,6 +218,22 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm
return return
} }
} }

var datasetInfos map[string]models.DatasetInfo
var attachSize int
if uuid != "" {
datasetInfos, _, err = models.GetDatasetInfo(uuid)
for _, infos := range datasetInfos {
attachSize += infos.Size
}
if attachSize > int(setting.DebugAttachSize*1000*1000*1000) {
log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize)) //GB
notebookNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", int(setting.DebugAttachSize*1000*1000*1000)), tplModelArtsNotebookNew, &form)
return
}
}

var aiCenterCode = models.AICenterOfCloudBrainTwo var aiCenterCode = models.AICenterOfCloudBrainTwo
if setting.ModelartsCD.Enabled { if setting.ModelartsCD.Enabled {
aiCenterCode = models.AICenterOfChengdu aiCenterCode = models.AICenterOfChengdu
@@ -439,9 +455,13 @@ func NotebookDebug2(ctx *context.Context) {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return return
} }
if task.BootFile != "" {
ctx.Redirect(getFileUrl(result.Url, task.BootFile) + "?token=" + result.Token)

if ctx.QueryTrim("file") != "" {
ctx.Redirect(getFileUrl(result.Url, ctx.QueryTrim("file")) + "?token=" + result.Token)
} else { } else {
if task.BootFile != "" {
go cloudbrainTask.UploadNotebookFiles(task)
}
ctx.Redirect(result.Url + "?token=" + result.Token) ctx.Redirect(result.Url + "?token=" + result.Token)
} }


@@ -463,7 +483,7 @@ func getFileUrl(url string, filename string) string {
} }
} }


return url + middle + path.Base(filename)
return url + middle + filename
} }


func NotebookRestart(ctx *context.Context) { func NotebookRestart(ctx *context.Context) {
@@ -630,7 +650,7 @@ func NotebookStop(ctx *context.Context) {
if task.Status != string(models.ModelArtsRunning) { if task.Status != string(models.ModelArtsRunning) {
log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"]) log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"])
resultCode = "-1" resultCode = "-1"
errorMsg = "the job is not running"
errorMsg = ctx.Tr("cloudbrain.Already_stopped")
break break
} }


@@ -2589,7 +2609,8 @@ func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModel
ctx.Data["datasetType"] = models.TypeCloudBrainTwo ctx.Data["datasetType"] = models.TypeCloudBrainTwo
waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "")
ctx.Data["WaitCount"] = waitCount ctx.Data["WaitCount"] = waitCount

NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference))
ctx.Data["NotStopTaskCount"] = NotStopTaskCount
return nil return nil
} }
func InferenceJobShow(ctx *context.Context) { func InferenceJobShow(ctx *context.Context) {
@@ -2653,6 +2674,46 @@ func InferenceJobShow(ctx *context.Context) {
ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow) ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow)
} }


func MultiModelDownload(ctx *context.Context) {
var (
err error
)
jobID := ctx.Params(":jobid")
versionName := ctx.Query("version_name")
parentDir := ctx.Query("parent_dir")

task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
if err != nil {
log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error())
return
}

if task.ComputeResource == models.NPUResource {
path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir), "/")
path = strings.TrimSuffix(path, "/")
path += "/"
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
if err == nil {
returnFileName := task.DisplayJobName + ".zip"
ObsDownloadManyFile(path, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
} else if task.ComputeResource == models.GPUResource {
filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
if err == nil {
returnFileName := task.DisplayJobName + ".zip"
MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
}

}

func ModelDownload(ctx *context.Context) { func ModelDownload(ctx *context.Context) {
var ( var (
err error err error


+ 7
- 3
routers/routes/routes.go View File

@@ -1186,6 +1186,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate)
m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel) m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
}) })
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
@@ -1209,6 +1210,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel)
//m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) //m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel) m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel)
m.Get("/download_multi_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadMultiModel)
//m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir) //m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir)
//m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) //m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew)
@@ -1221,8 +1223,8 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/:jobid", func() { m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow) m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow)
m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult)
m.Get("/downloadall", repo.DownloadInferenceResultFile)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadMultiModel)
m.Get("/downloadall", cloudbrain.AdminOrJobCreaterRightForTrain, repo.DownloadGPUInferenceResultFile)
}) })
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainInferencForm{}), repo.CloudBrainInferenceJobCreate) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainInferencForm{}), repo.CloudBrainInferenceJobCreate)
@@ -1248,6 +1250,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusStopJob) m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusStopJob)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusTrainJobDel) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusTrainJobDel)
m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload)
m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.GrampusTrainJobVersionNew) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.GrampusTrainJobVersionNew)
m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobVersionCreate) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobVersionCreate)
}) })
@@ -1333,6 +1336,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobStop) m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobStop)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobDel) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobDel)
m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload)
m.Get("/download_log_file", cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobDownloadLogFile) m.Get("/download_log_file", cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobDownloadLogFile)
m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, context.PointAccount(), repo.TrainJobNewVersion) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, context.PointAccount(), repo.TrainJobNewVersion)
m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
@@ -1348,7 +1352,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/:jobid", func() { m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.InferenceJobShow) m.Get("", reqRepoCloudBrainReader, repo.InferenceJobShow)
m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ResultDownload) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ResultDownload)
m.Get("/downloadall", repo.DownloadMultiResultFile)
m.Get("/downloadall", cloudbrain.AdminOrJobCreaterRightForTrain, repo.DownloadMultiResultFile)
}) })
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceJobNew) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceJobNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsInferenceJobForm{}), repo.InferenceJobCreate) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsInferenceJobForm{}), repo.InferenceJobCreate)


+ 1
- 1
routers/user/home.go View File

@@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 14
- 14
services/cloudbrain/clear.go View File

@@ -14,21 +14,21 @@ import (


func ClearCloudbrainResultSpace() { func ClearCloudbrainResultSpace() {
log.Info("clear cloudbrain one result space begin.") log.Info("clear cloudbrain one result space begin.")
if !setting.ClearStrategy.Enabled{
if !setting.ClearStrategy.Enabled {
return return
} }


tasks, err := models.GetCloudBrainOneStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize)
tasks, err := models.GetGPUStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize)
if err != nil { if err != nil {
log.Warn("Failed to get cloudbrain, clear result failed.", err) log.Warn("Failed to get cloudbrain, clear result failed.", err)
return return
} }
debugTasks, err := models.GetCloudBrainOneStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize)
debugTasks, err := models.GetGPUStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize)
if err != nil { if err != nil {
log.Warn("Failed to get debug cloudbrain.", err) log.Warn("Failed to get debug cloudbrain.", err)


} }
tasks=append(tasks,debugTasks...)
tasks = append(tasks, debugTasks...)


if err != nil { if err != nil {
log.Warn("Failed to get cloudbrain, clear result failed.", err) log.Warn("Failed to get cloudbrain, clear result failed.", err)
@@ -38,7 +38,7 @@ func ClearCloudbrainResultSpace() {
for _, task := range tasks { for _, task := range tasks {
err := DeleteCloudbrainOneJobStorage(task.JobName) err := DeleteCloudbrainOneJobStorage(task.JobName)
if err == nil { if err == nil {
log.Info("clear job in cloudbrain table:"+task.JobName)
log.Info("clear job in cloudbrain table:" + task.JobName)
ids = append(ids, task.ID) ids = append(ids, task.ID)
} }
} }
@@ -69,10 +69,10 @@ func clearMinioHistoryTrashFile() {
SortModTimeAscend(miniofiles) SortModTimeAscend(miniofiles)
for _, file := range miniofiles { for _, file := range miniofiles {


if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {


has,err:=models.IsCloudbrainExistByJobName(file.Name())
if err==nil && !has {
has, err := models.IsCloudbrainExistByJobName(file.Name())
if err == nil && !has {
dirPath := setting.CBCodePathPrefix + file.Name() + "/" dirPath := setting.CBCodePathPrefix + file.Name() + "/"
log.Info("clear job in minio trash:" + file.Name()) log.Info("clear job in minio trash:" + file.Name())
storage.Attachments.DeleteDir(dirPath) storage.Attachments.DeleteDir(dirPath)
@@ -90,7 +90,7 @@ func clearMinioHistoryTrashFile() {
} }
} }


func clearLocalHistoryTrashFile() {
func clearLocalHistoryTrashFile() {
files, err := ioutil.ReadDir(setting.JobPath) files, err := ioutil.ReadDir(setting.JobPath)
processCount := 0 processCount := 0
if err != nil { if err != nil {
@@ -99,11 +99,11 @@ func clearLocalHistoryTrashFile() {
SortModTimeAscend(files) SortModTimeAscend(files)
for _, file := range files { for _, file := range files {
//清理n天前的历史垃圾数据,清理job目录 //清理n天前的历史垃圾数据,清理job目录
if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
has,err:=models.IsCloudbrainExistByJobName(file.Name())
if err==nil && !has{
if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
has, err := models.IsCloudbrainExistByJobName(file.Name())
if err == nil && !has {
os.RemoveAll(setting.JobPath + file.Name()) os.RemoveAll(setting.JobPath + file.Name())
log.Info("clear job in local trash:"+file.Name())
log.Info("clear job in local trash:" + file.Name())
processCount++ processCount++
} }
if processCount == setting.ClearStrategy.BatchSize { if processCount == setting.ClearStrategy.BatchSize {
@@ -127,7 +127,7 @@ func SortModTimeAscend(files []os.FileInfo) {


func DeleteCloudbrainOneJobStorage(jobName string) error { func DeleteCloudbrainOneJobStorage(jobName string) error {


if jobName==""{
if jobName == "" {
return nil return nil
} }
//delete local //delete local


+ 2
- 2
services/cloudbrain/cloudbrainTask/count.go View File

@@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s
ComputeResource: models.GPUResource, ComputeResource: models.GPUResource,
}, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): {
CloudBrainTypes: []int{models.TypeCloudBrainOne}, CloudBrainTypes: []int{models.TypeCloudBrainOne},
JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet},
JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset},
NotFinalStatuses: CloudbrainOneNotFinalStatuses, NotFinalStatuses: CloudbrainOneNotFinalStatuses,
ComputeResource: models.GPUResource, ComputeResource: models.GPUResource,
}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): {
@@ -76,7 +76,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s


func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) {
jobNewType := jobType jobNewType := jobType
if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) {
if models.IsModelBenchMarkJobType(jobType) {
jobNewType = string(models.JobTypeBenchmark) jobNewType = string(models.JobTypeBenchmark)
} }




+ 240
- 28
services/cloudbrain/cloudbrainTask/notebook.go View File

@@ -4,6 +4,9 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"path" "path"
"strings"

"code.gitea.io/gitea/modules/notebook"


"code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/modelarts_cd" "code.gitea.io/gitea/modules/modelarts_cd"
@@ -29,6 +32,9 @@ import (
) )


const NoteBookExtension = ".ipynb" const NoteBookExtension = ".ipynb"
const CPUType = 0
const GPUType = 1
const NPUType = 2


func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) {


@@ -66,7 +72,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
} }


//create repo if not exist //create repo if not exist
repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName)
repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName)
if repo == nil { if repo == nil {
repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{
Name: setting.FileNoteBook.ProjectName, Name: setting.FileNoteBook.ProjectName,
@@ -80,19 +86,222 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
AutoInit: true, AutoInit: true,
DefaultBranch: "master", DefaultBranch: "master",
}) })
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName)))
return
}
} else {

noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type))
if noteBook != nil {

if isRepoConfilcts(option, noteBook) {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict")))
return
}

if isNotebookSpecMath(option, noteBook) {
if !isRepoMatch(option, noteBook) {
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName)
if err != nil {
log.Error("download code failed", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return
}
}
if !isRepoFileMatch(option, noteBook) {
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName)
noteBook.BranchName += ";" + option.BranchName
noteBook.Description += ";" + getDescription(option)

err := models.UpdateJob(noteBook)
if err != nil {
log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"])
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error()))
return
}
}

ctx.JSON(http.StatusOK, models.BaseMessageApi{
Code: 0,
Message: noteBook.JobID,
})
return
}

}
}

if option.Type <= GPUType {
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo)
} else {
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo)
}

}
func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) {
if ctx.Written() {
return
} }

if path.Ext(option.File) != NoteBookExtension {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong")))
return
}

isNotebookFileExist, _ := isNoteBookFileExist(ctx, option)
if !isNotebookFileExist {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist")))
return
}

task, err := models.GetCloudbrainByJobID(option.JobId)
if err != nil { if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName)))
log.Error("job not found:"+option.JobId, err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job."))
return return
} }
if option.Type <= 1 {
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo)
if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) {
log.Warn("Boot file is empty or status is running. ")
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running."))
return
}
if !isRepoFileMatch(option, task) {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file."))
return
}
debugBaseUrl, token, err := getBaseUrlAndToken(task)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error()))
return
}

if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) {
ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
} else { } else {
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed."))

}

}

func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) {
var debugBaseUrl string
var token string
if task.Type == models.TypeCloudBrainOne {
debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab"

} else {
var result *models.GetNotebook2Result
var err error
if task.Type == models.TypeCloudBrainTwo {
result, err = modelarts.GetNotebook2(task.JobID)
} else if task.Type == models.TypeCDCenter {
result, err = modelarts_cd.GetNotebook(task.JobID)
}
if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" {
log.Error("notebook job not found:"+task.JobID, err)
return "", "", fmt.Errorf("can not get job or job is invalid.")
}

debugBaseUrl = result.Url
token = result.Token

} }
return debugBaseUrl, token, nil
}

func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool {
c := &notebook.NotebookContent{
Url: debugBaseUrl,
Path: bootFile,
PathType: "file",
Token: token,
}
if c.IsNotebookFileCanBrowser() {
return true
} else {
c.SetCookiesAndCsrf()
c.UploadNoteBookFile(task)
return c.IsNotebookFileCanBrowser()
}

}

func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
if option.Type == NPUType || option.Type == CPUType {
return true
}
spec, err := models.GetCloudbrainSpecByID(book.ID)
if err != nil {
log.Warn("can not get spec ", err)
return false
}
return spec.AccCardsNum > 0
}

func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
branches := strings.Split(book.BranchName, ";")

for i, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName {
return true
}
}
}

return false


} }


func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")

for _, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName {
return true
}
}
}
return false

}

func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
branches := strings.Split(book.BranchName, ";")

for i, bootFile := range bootFiles {
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile {
return true
}
}

return false

}
func UploadNotebookFiles(task *models.Cloudbrain) {
if task.Status == string(models.JobRunning) && task.BootFile != "" {

debugBaseUrl, token, err := getBaseUrlAndToken(task)
if err != nil {
log.Error("can not get base url:", err)
return
}
bootFiles := strings.Split(task.BootFile, ";")

for _, bootFile := range bootFiles {
uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token)
}

}
}

func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) {


displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name)
@@ -131,17 +340,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
} else { } else {
if count >= 1 { if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
ctx.JSON(http.StatusOK,models.BaseMessageApi{
Code: 2,
ctx.JSON(http.StatusOK, models.BaseMessageApi{
Code: 2,
Message: ctx.Tr("repo.cloudbrain.morethanonejob"), Message: ctx.Tr("repo.cloudbrain.morethanonejob"),
}) })
return return
} }
} }


errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName)
if errStr != "" {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist")))
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
if err != nil {
log.Error("download code failed", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return return
} }
command := cloudbrain.GetCloudbrainDebugCommand() command := cloudbrain.GetCloudbrainDebugCommand()
@@ -185,7 +395,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
JobType: jobType, JobType: jobType,
Description: getDescription(option), Description: getDescription(option),
BranchName: option.BranchName, BranchName: option.BranchName,
BootFile: option.File,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
Params: "{\"parameter\":[]}", Params: "{\"parameter\":[]}",
CommitID: "", CommitID: "",
BenchmarkTypeID: 0, BenchmarkTypeID: 0,
@@ -206,8 +416,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot


} }


func getCodePath(jobName string) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath
func getCloudbrainType(optionType int) int {
if optionType < 1 {
return models.TypeCloudBrainOne
}
if setting.ModelartsCD.Enabled {
return models.TypeCDCenter
}
return models.TypeCloudBrainTwo
}

func getCodePath(jobName string, repo *models.Repository) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name
} }


func getDescription(option api.CreateFileNotebookJobOption) string { func getDescription(option api.CreateFileNotebookJobOption) string {
@@ -237,8 +457,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
} else { } else {
if count >= 1 { if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
ctx.JSON(http.StatusOK,models.BaseMessageApi{
Code: 2,
ctx.JSON(http.StatusOK, models.BaseMessageApi{
Code: 2,
Message: ctx.Tr("repo.cloudbrain.morethanonejob"), Message: ctx.Tr("repo.cloudbrain.morethanonejob"),
}) })
return return
@@ -260,7 +480,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
} }
} }


err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName)
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
if err != nil { if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return return
@@ -297,8 +517,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
Description: getDescription(option), Description: getDescription(option),
ImageId: setting.FileNoteBook.ImageIdNPU, ImageId: setting.FileNoteBook.ImageIdNPU,
Spec: spec, Spec: spec,
BootFile: "",
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, AutoStopDurationMs: modelarts.AutoStopDurationMs / 4,
BranchName: option.BranchName,
} }


if setting.ModelartsCD.Enabled { if setting.ModelartsCD.Enabled {
@@ -347,17 +568,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO
return true, nil return true, nil
} }


func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string {
err := downloadCode(repo, codePath, branchName)
if err != nil {
return "cloudbrain.load_code_failed"
}

err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/")
if err != nil {
return "cloudbrain.load_code_failed"
}
return ""
func getBootFile(filePath string, ownerName string, projectName string) string {
return ownerName + "/" + projectName + "/" + filePath
} }


func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) {


+ 2
- 2
templates/custom/task_wait_count.tmpl View File

@@ -1,7 +1,7 @@
<div style="display:inline-block;"> <div style="display:inline-block;">
<div style="display:flex;align-items:center;color:#f2711c;"> <div style="display:flex;align-items:center;color:#f2711c;">
<i class="ri-error-warning-line" style="margin-right: 0.5rem; font-size: 14px"></i> <i class="ri-error-warning-line" style="margin-right: 0.5rem; font-size: 14px"></i>
<span style="font-size: 12px">{{.i18n.Tr "repo.wait_count_start"}} <span class="__task_wait_count__">{{.WaitCount}}</span> {{.i18n.Tr "repo.wait_count_end"}}</span>
<span style="font-size: 12px">{{.i18n.Tr "repo.wait_count_start"}} <span class="__task_wait_count__">{{if not .WaitCount}}1{{else}}{{addOne .WaitCount}}{{end}}</span> {{.i18n.Tr "repo.wait_count_end"}}</span>
</div> </div>
</div> </div>
<script> <script>
@@ -12,7 +12,7 @@
var specsSelEl = $('select#__specs__'); var specsSelEl = $('select#__specs__');
var seldOption = specsSelEl.find('option:selected'); var seldOption = specsSelEl.find('option:selected');
var queueCode = seldOption.attr('queueCode'); var queueCode = seldOption.attr('queueCode');
$('span.__task_wait_count__').text(queuesDetail[queueCode] || 0);
$('span.__task_wait_count__').text((queuesDetail[queueCode] || 0) +1);
}; };
$('body').on('change', 'select#__specs__', function(e) { $('body').on('change', 'select#__specs__', function(e) {
changeSpecs(); changeSpecs();


+ 123
- 257
templates/repo/cloudbrain/benchmark/new.tmpl View File

@@ -1,37 +1,9 @@
{{template "base/head" .}} {{template "base/head" .}}
<style>
.unite {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}

.title {
font-size: 16px !important;
padding-left: 3rem !important;
}

.min_title{
font-size: 14px !important;
margin-bottom: 2rem !important;
}

.width81 {
margin-left: 1.5rem;
width: 81% !important;
}

.width48 {
width: 48.5% !important;
}

.nowrapx {
white-space: nowrap !important;
}
</style>
{{template "custom/global_mask" .}} {{template "custom/global_mask" .}}
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-repo-link="{{.RepoLink}}" data-flag-model="true"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -78,21 +50,12 @@
onkeydown="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea> onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div> </div>

<!--<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<select id="cloudbrain_gpu_type" class="ui search dropdown width48" placeholder="选择GPU类型"
name="gpu_type">
{{range .benchmark_gpu_types}}
<option value="{{.Queue}}">{{.Value}}</option>
{{end}}
</select>
</div>-->
<div class="required unite min_title two inline fields" style="margin-left: 80px;">
<div class="required min_title two inline fields" style="margin-left: 80px;">
<div class="required ten wide field" style="width: 26.5% !important;"> <div class="required ten wide field" style="width: 26.5% !important;">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_type"}}</label>&nbsp; <label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_type"}}</label>&nbsp;
<select id="cloudbrain_job_type" class="ui search dropdown job_type"
<select id="cloudbrain_job_type" class="ui search dropdown job_type"
placeholder="select {{.i18n.Tr "cloudbrain.task_type"}}" name="job_type"> placeholder="select {{.i18n.Tr "cloudbrain.task_type"}}" name="job_type">
<option value="SNN4ECOSET">SNN4ECOSET</option>
<option value="SNN4IMAGENET">SNN4IMAGENET</option> <option value="SNN4IMAGENET">SNN4IMAGENET</option>
<option value="BRAINSCORE">BRAINSCORE</option> <option value="BRAINSCORE">BRAINSCORE</option>
</select> </select>
@@ -116,22 +79,10 @@
<a id="benchmark_model_example" href="https://openi.pcl.ac.cn/BDIP/snn4imagenet" <a id="benchmark_model_example" href="https://openi.pcl.ac.cn/BDIP/snn4imagenet"
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a> target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div> </div>
{{template "custom/select_model" .}}
<div id="images-new-cb"> <div id="images-new-cb">
</div> </div>
{{template "custom/select_dataset_train" .}}
<!--<div class="required min_title inline field" style="margin-top:2rem;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px'
name="resource_spec_id">
{{range .benchmark_resource_specs}}
<option name="resource_spec_id" value="{{.Id}}">
{{$.i18n.Tr "cloudbrain.gpu_num"}}:{{.GpuNum}},{{$.i18n.Tr "cloudbrain.cpu_num"}}:{{.CpuNum}},{{$.i18n.Tr "cloudbrain.memory"}}(MB):{{.MemMiB}},{{$.i18n.Tr "cloudbrain.shared_memory"}}(MB):{{.ShareMemMiB}}
</option>
{{end}}
</select>
</div>-->
<div class="required min_title inline field" style="margin-top:2rem;"> <div class="required min_title inline field" style="margin-top:2rem;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48" <select id="__specs__" class="ui search dropdown width48"
@@ -156,7 +107,7 @@
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}"> <button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}} {{.i18n.Tr "repo.cloudbrain.new"}}
</button> </button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
<a class="ui button" href="{{.RepoLink}}/cloudbrain/benchmark">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div> </div>
</div> </div>
</form> </form>
@@ -199,20 +150,11 @@
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea> onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div> </div>


<!--<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<select id="cloudbrain_gpu_type" class="ui search dropdown" placeholder="选择GPU类型"
style='width:385px' name="gpu_type">
{{range .benchmark_gpu_types}}
<option value="{{.Queue}}">{{.Value}}</option>
{{end}}
</select>
</div>-->
<div class="required unite inline min_title fields" style="width: 90%;margin-left: 5.7rem;">&nbsp;
<div class="required inline min_title fields" style="width: 90%;margin-left: 5.7rem;">&nbsp;
<div class="required eight wide field"> <div class="required eight wide field">
<label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}}</label> <label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}}</label>


<select class="ui fluid selection search dropdown" id="benchmark_types_id"
<select class="ui fluid selection search dropdown benchmark_types_id"
name="benchmark_types_id"> name="benchmark_types_id">
{{range .benchmark_types}} {{range .benchmark_types}}
{{if eq .Id $.benchmarkTypeID}} {{if eq .Id $.benchmarkTypeID}}
@@ -226,26 +168,13 @@
<div class="eight wide field" id="engine_name"> <div class="eight wide field" id="engine_name">
<input type="hidden" id="benchmark_child_types_id_hidden" name="benchmark_child_types_id_hidden" value="{{.benchmark_child_types_id_hidden}}"> <input type="hidden" id="benchmark_child_types_id_hidden" name="benchmark_child_types_id_hidden" value="{{.benchmark_child_types_id_hidden}}">
<label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_child_type"}}</label> <label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_child_type"}}</label>
<select class="ui fluid selection dropdown nowrapx" id="benchmark_child_types_id" style='width: 100%;' name="benchmark_child_types_id">
<select class="ui fluid selection dropdown " id="benchmark_child_types_id" style='width: 100%;' name="benchmark_child_types_id">
</select> </select>
</div> </div>
</div> </div>
<div id="images-new-cb"> <div id="images-new-cb">
</div> </div>


<!--<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px'
name="resource_spec_id">
{{range .benchmark_resource_specs}}
<option name="resource_spec_id" value="{{.Id}}">
{{$.i18n.Tr "cloudbrain.gpu_num"}}:{{.GpuNum}},{{$.i18n.Tr "cloudbrain.cpu_num"}}:{{.CpuNum}},{{$.i18n.Tr "cloudbrain.memory"}}(MB):{{.MemMiB}},{{$.i18n.Tr "cloudbrain.shared_memory"}}(MB):{{.ShareMemMiB}}
</option>
{{end}}
</select>
</div>-->

<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48" <select id="__specs__" class="ui search dropdown width48"
@@ -282,197 +211,134 @@
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a> target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div> </div>



<div class="inline unite min_title field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</div>
</form>
{{else if eq .benchmarkMode "aisafety"}}
<form id="form_id" class="ui form alogrithm_form" action="{{.Link}}?benchmarkMode=alogrithm" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<input type="hidden" name="job_type" value="BENCHMARK">
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_scenes"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="item alogrithm_benchmark"
href="{{.Link}}?benchmarkMode=alogrithm">{{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}}</a>
<a class="item model_benchmark"
href="{{.Link}}?benchmarkMode=model">{{.i18n.Tr "repo.cloudbrain.benchmark.model"}}</a>
<a class="active item aisafety_benchmark"
href="{{.RepoLink}}/modelsafety/create_gpu">模型安全评测</a>
</div>
</div>

<div>
<div class="min_title inline field" style="margin-top:-10px;">
<label class="label-fix-width" style="font-weight: normal;"></label>
{{template "custom/task_wait_count" .}}
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 80%;" name="display_job_name" id="trainjob_job_name"
placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}"
tabindex="3" autofocus required maxlength="36">
<span class="tooltips" style="display: block;margin-left: 11.5rem;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
</div>
<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;"
for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="254"
placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}
onchange="this.value=this.value.substring(0, 255)"
onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div>


<div id="images-new-cb">
</div>

<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' ovalue="{{.spec_id}}"
name="spec_id">
</select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
</div>

<div class="inline min_title field required">
<label class="label-fix-width" style="font-weight: normal;">推理程序</label>
<input disabled="disabled" style="width: 33.5%;" name="test_file" id="test_file" value="test.py"
tabindex="3" autofocus required maxlength="254">
<a id="test_href_id" href="https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark"
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div>


<div class="inline unite min_title field">
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;"></label> <label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}"> <button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}} {{.i18n.Tr "repo.cloudbrain.new"}}
</button> </button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
<a class="ui button" href="{{.RepoLink}}/cloudbrain/benchmark">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div> </div>
</div> </div>
</form> </form>

{{end}} {{end}}
</div> </div>
</div> </div>
</div> </div>
{{template "base/footer" .}} {{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script> <script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script> <script>
let form = document.getElementById('form_id');
let createFlag = false
form.onsubmit = function (e) {
if(createFlag) return false
createFlag = true
}
let repolink = {{.RepoLink }}
let url_href = window.location.pathname.split('create')[0]
$(".ui.button").attr('href', url_href)
$('.menu .item')
.tab();
$('#benchmark_types_id').change(function () {
setChildType();
})
$(document).ready(() => {
$('.ui.search.dropdown.job_type').dropdown({
onChange: function (value, text, $selectedItem) {
if (value === "BRAINSCORE") {
$('#brainscore_child_type').css('display', 'block')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann')
} else {
$('#brainscore_child_type').css('display', 'none')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet')
}
}
})
})
function setChildType() {
let type_id = $('#benchmark_types_id').val();
if (type_id == 3) {
$('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
$('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
} else {
$('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
$('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
}
let child_selected_id = $('#benchmark_child_types_id_hidden').val();
$.get(`${repolink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => {
const n_length = data['child_types'].length
let html = ''
for (let i = 0; i < n_length; i++) {
if (child_selected_id == data['child_types'][i].id) {
html += `<option value="${data['child_types'][i].id}" selected="true">${data['child_types'][i].value}</option>`;
} else {
html += `<option value="${data['child_types'][i].id}">${data['child_types'][i].value}</option>`;
}
}
document.getElementById("benchmark_child_types_id").innerHTML = html;
})
}
document.onreadystatechange = function () {
if (document.readyState === "complete") {
if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') {
setChildType();
}
}
}
var isValidate = false;
function validate() {
$('.ui.form')
.form({
on: 'blur',
fields: {
image: {
identifier: 'image',
rules: [
{
type: 'empty',
promt: ''
}
]
},
display_job_name: {
identifier: 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
promt: ''
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function () {
// $('.ui.page.dimmer').dimmer('show')
document.getElementById("mask").style.display = "block"
isValidate = true;
},
onFailure: function (e) {
isValidate = false;
return false;
}
})
}
// let repolink = {{.RepoLink }}
// $('#benchmark_types_id').change(function () {
// console.log("----")
// // setChildType();
// })
// $("#benchmark_types_id").dropdown({
// onChange:function (value, text, $selectedItem){
// console.log(value)
// }
// })
// $(document).ready(() => {
// $('.ui.search.dropdown.job_type').dropdown({
// onChange: function (value, text, $selectedItem) {
// console.log(value)
// if (value === "BRAINSCORE") {
// $('#brainscore_child_type').css('display', 'block')
// $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann')
// } else {
// $('#brainscore_child_type').css('display', 'none')
// $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet')
// }
// }
// })
// })
// function setChildType(type_id=1) {
// if (type_id == 3) {
// $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
// $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
// } else {
// $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
// $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
// }
// let child_selected_id = $('#benchmark_child_types_id_hidden').val();
// $.get(`${repolink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => {
// const n_length = data['child_types'].length
// let html = ''
// for (let i = 0; i < n_length; i++) {
// if (child_selected_id == data['child_types'][i].id) {
// html += `<option value="${data['child_types'][i].id}" selected="true">${data['child_types'][i].value}</option>`;
// } else {
// html += `<option value="${data['child_types'][i].id}">${data['child_types'][i].value}</option>`;
// }
// }
// document.getElementById("benchmark_child_types_id").innerHTML = html;
// })
// }
// $(document).ready(function (){
// console.log("00")
// $(".ui.selection.dropdown.benchmark_types_id").dropdown({
// onChange:function (value, text, $selectedItem){
// console.log(value)
// setChildType(value)
// }
// })
// })
// document.onreadystatechange = function () {
// if (document.readyState === "complete") {
// if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') {
// setChildType();
// }
// }
// }
// var isValidate = false;
// function validate() {
// $('.ui.form')
// .form({
// on: 'blur',
// fields: {
// image: {
// identifier: 'image',
// rules: [
// {
// type: 'empty',
// promt: ''
// }
// ]
// },
// display_job_name: {
// identifier: 'display_job_name',
// rules: [
// {
// type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
// promt: ''
// }
// ]
// },
// spec_id: {
// identifier: 'spec_id',
// rules: [{ type: 'empty' }]
// }
// },
// onSuccess: function () {
// // $('.ui.page.dimmer').dimmer('show')
// document.getElementById("mask").style.display = "block"
// isValidate = true;
// },
// onFailure: function (e) {
// isValidate = false;
// return false;
// }
// })
// }


validate()
$('.ui.create_train_job.green.button').click(function (e) {
validate()
})
// validate()
// $('.ui.create_train_job.green.button').click(function (e) {
// validate()
// })


;(function() { ;(function() {
var SPECS = {{ .benchmark_specs }}; var SPECS = {{ .benchmark_specs }};


+ 30
- 21
templates/repo/cloudbrain/benchmark/show.tmpl View File

@@ -206,7 +206,7 @@
<table class="ti-form"> <table class="ti-form">
<tbody class="ti-text-form"> <tbody class="ti-text-form">


{{if eq .JobType "BENCHMARK"}}
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_train"}} {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_train"}}
@@ -214,11 +214,7 @@


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w"> <div class="text-span text-span-w">
{{if eq .JobType "BENCHMARK"}}
train.py train.py
{{else}}
--
{{end}}
</div> </div>
</td> </td>
</tr> </tr>
@@ -228,16 +224,39 @@
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w">
{{if eq .JobType "BENCHMARK"}}
test.py test.py
{{else}}
--
{{end}}
</div> </div>
</td> </td>
</tr> </tr>
{{else}}
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.model_name"}}
</td>


<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .ModelName}}{{.ModelName}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelconvert.modelversion"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .ModelVersion}}{{.ModelVersion}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.infer_job_model_file"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .CkptName}}{{.CkptName}}{{else}}--{{end}}</div>
</td>
</tr>
{{end}}
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.description"}} {{$.i18n.Tr "repo.modelarts.train_job.description"}}
@@ -271,17 +290,7 @@
<div class="text-span text-span-w"></div> <div class="text-span text-span-w"></div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.model_manager"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.DatasetName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}} {{$.i18n.Tr "repo.cloudbrain_creator"}}


+ 1
- 1
templates/repo/cloudbrain/inference/new.tmpl View File

@@ -40,7 +40,7 @@
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
{{template "custom/alert_cb" .}} {{template "custom/alert_cb" .}}
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
<h4 class="ui top attached header"> <h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new_infer"}} {{.i18n.Tr "repo.modelarts.train_job.new_infer"}}
</h4> </h4>


+ 12
- 4
templates/repo/cloudbrain/inference/show.tmpl View File

@@ -371,10 +371,18 @@
<div class="ui tab" data-tab="four"> <div class="ui tab" data-tab="four">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/cloudbrain/inference-job/{{.JobID}}/downloadall?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">




+ 2
- 1
templates/repo/cloudbrain/new.tmpl View File

@@ -23,7 +23,7 @@
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}" data-flag-model="true"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -38,6 +38,7 @@
<form id="form_id" class="ui form" action="{{.Link}}" method="post"> <form id="form_id" class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}} {{.CsrfTokenHtml}}
<input type="hidden" name='isBranches' value="{{.Branches}}"> <input type="hidden" name='isBranches' value="{{.Branches}}">
<input type="hidden" id="ai_image_name" value="{{.image}}">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4> <h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>


+ 1
- 1
templates/repo/cloudbrain/trainjob/new.tmpl View File

@@ -56,7 +56,7 @@
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}


+ 22
- 3
templates/repo/cloudbrain/trainjob/show.tmpl View File

@@ -123,7 +123,17 @@
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}}
</td>


<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.User.Name}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.start_time"}} {{$.i18n.Tr "repo.modelarts.train_job.start_time"}}
@@ -351,9 +361,18 @@
<div class="ui tab" data-tab="four{{$k}}"> <div class="ui tab" data-tab="four{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>
<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}/download_multi_model?version_name={{.VersionName}}&jobName={{.JobName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">
</div> </div>


+ 2
- 3
templates/repo/grampus/notebook/gpu/new.tmpl View File

@@ -4,8 +4,7 @@
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}" data-flag-model="true"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -20,8 +19,8 @@
<form id="form_id" class="ui form" action="{{.Link}}" method="post"> <form id="form_id" class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}} {{.CsrfTokenHtml}}
<input type="hidden" name="type" value="0"> <input type="hidden" name="type" value="0">
<input type="hidden" id="ai_image_name" value="{{.image}}">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4> <h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain"> <div class="ui blue mini menu compact selectcloudbrain">


+ 16
- 3
templates/repo/grampus/notebook/npu/new.tmpl View File

@@ -3,7 +3,7 @@
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -93,8 +93,21 @@
<div class="inline min_title required field"> <div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label>
<select class="ui search dropdown cloudbrain_image width48" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" style='width:385px' name="image_id"> <select class="ui search dropdown cloudbrain_image width48" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" style='width:385px' name="image_id">
{{range .images}}
<option name="image_id" value="{{.ID}}">{{.Name}}</option>
{{if .image_id}}
{{range .images}}
{{if eq $.image_id .ID}}
<option value="{{.ID}}">{{.Name}}</option>
{{end}}
{{end}}
{{range .images}}
{{if ne $.image_id .ID}}
<option value="{{.ID}}">{{.Name}}</option>
{{end}}
{{end}}
{{else}}
{{range .images}}
<option name="image_id" value="{{.ID}}">{{.Name}}</option>
{{end}}
{{end}} {{end}}
</select> </select>
</div> </div>


+ 23
- 4
templates/repo/grampus/trainjob/show.tmpl View File

@@ -123,6 +123,17 @@
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.User.Name}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.run_version"}} {{$.i18n.Tr "repo.modelarts.run_version"}}
</td> </td>


@@ -386,10 +397,18 @@
<div class="ui tab" data-tab="third{{$k}}"> <div class="ui tab" data-tab="third{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/grampus/train-job/{{.JobID}}/download_multi_model?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">
</div> </div>


+ 12
- 4
templates/repo/modelarts/inferencejob/show.tmpl View File

@@ -353,10 +353,18 @@
<div class="ui tab" data-tab="third"> <div class="ui tab" data-tab="third">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/modelarts/inference-job/{{.JobID}}/downloadall?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">




+ 16
- 3
templates/repo/modelarts/notebook/new.tmpl View File

@@ -4,7 +4,7 @@
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -77,8 +77,21 @@
<div class="inline min_title required field"> <div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label>
<select id="cloudbrain_image" class="ui search dropdown width48" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" name="image_id"> <select id="cloudbrain_image" class="ui search dropdown width48" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" name="image_id">
{{range .images}}
<option name="image_id" value="{{.Id}}">{{.Value}}</option>
{{if .image_id}}
{{range .images}}
{{if eq $.image_id .Id}}
<option value="{{.Id}}">{{.Value}}</option>
{{end}}
{{end}}
{{range .images}}
{{if ne $.image_id .Id}}
<option value="{{.Id}}">{{.Value}}</option>
{{end}}
{{end}}
{{else}}
{{range .images}}
<option name="image_id" value="{{.Id}}">{{.Value}}</option>
{{end}}
{{end}} {{end}}
</select> </select>
</div> </div>


+ 24
- 4
templates/repo/modelarts/trainjob/show.tmpl View File

@@ -159,6 +159,17 @@
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.User.Name}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.run_version"}} {{$.i18n.Tr "repo.modelarts.run_version"}}
</td> </td>


@@ -411,10 +422,19 @@
<div class="ui tab" data-tab="third{{$k}}"> <div class="ui tab" data-tab="third{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/modelarts/train-job/{{.JobID}}/download_multi_model?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}" style="max-height: 500px;overflow:auto;"> <div id="dir_list{{.VersionName}}" style="max-height: 500px;overflow:auto;">




+ 2
- 35
templates/repo/modelsafety/new.tmpl View File

@@ -49,7 +49,7 @@
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
{{$Grampus := (or (eq (index (SubJumpablePath .Link) 1) "create_grampus_gpu") (eq (index (SubJumpablePath .Link) 1) "create_grampus_npu"))}} {{$Grampus := (or (eq (index (SubJumpablePath .Link) 1) "create_grampus_gpu") (eq (index (SubJumpablePath .Link) 1) "create_grampus_npu"))}}
<div class="cloudbrain-type" style="display: none;" data-grampus="{{$Grampus}}" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}"></div>
<div class="cloudbrain-type" style="display: none;" data-grampus="{{$Grampus}}" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -141,40 +141,7 @@
onkeydown="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea> onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div> </div>
<input type="hidden" id="ai_model_version" name="model_version" value="{{$.model_version}}">
<div class="required inline min_title fields" style="width: 93.5%;">
<label class="label-fix-width label-required" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>
<div class="six wide field">
<div class="ui fluid selection dropdown" id="select_model">
<input type="hidden" name="model_name" required value="{{.model_name}}">
<div class="text"></div>
<i class="dropdown icon"></i>
<div class="menu" id="model_name">
</div>
</div>
</div>
<input type="hidden" name="pre_train_model_url" value="{{.train_url}}">
<div class="three wide field">
<div class="ui fluid selection dropdown" id="select_model_version">
<input type="hidden" name="train_url" required value="{{.train_url}}">
<div class="text"></div>
<i class="dropdown icon"></i>
<div class="menu" id="model_name_version"></div>
</div>
</div>
<div class="five wide field">
<div class="ui fluid selection dropdown" id="select_model_checkpoint">
<input type="hidden" name="ckpt_name" required value="{{.ckpt_name}}">
<div class="text"></div>
<i class="dropdown icon"></i>
<div class="menu" id="model_checkpoint">
</div>
</div>
</div>
<span >
<i class="question circle icon" data-content="{{.i18n.Tr "cloudbrain.model_file_postfix_rule"}}" data-position="top center" data-variation="inverted mini"></i>
</span>
</div>
{{template "custom/select_model" .}}
<div class="required inline min_title field " style="display: none;"> <div class="required inline min_title field " style="display: none;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
<select class="ui dropdown width48" id="trainjob_resource_pool" name="pool_id"> <select class="ui dropdown width48" id="trainjob_resource_pool" name="pool_id">


+ 29
- 0
web_src/js/components/dataset/selectDataset.vue View File

@@ -176,6 +176,12 @@
> >
{{ i18n.dataset_unzip_failed }} {{ i18n.dataset_unzip_failed }}
</span> </span>
<span
class="unzip-failed"
v-if="data.Size > exceedSize"
>
{{ i18n.dataset_exceeds_failed }}{{exceedSize/(1000*1000*1000)}}G
</span>
</span> </span>
</span> </span>
</el-tree> </el-tree>
@@ -282,6 +288,12 @@
> >
{{ i18n.dataset_unzip_failed }} {{ i18n.dataset_unzip_failed }}
</span> </span>
<span
class="unzip-failed"
v-if="data.Size > exceedSize"
>
{{ i18n.dataset_exceeds_failed }}{{exceedSize/(1000*1000*1000)}}G
</span>
</span> </span>
</span> </span>
</el-tree> </el-tree>
@@ -388,6 +400,12 @@
> >
{{ i18n.dataset_unzip_failed }} {{ i18n.dataset_unzip_failed }}
</span> </span>
<span
class="unzip-failed"
v-if="data.Size > exceedSize"
>
{{ i18n.dataset_exceeds_failed }}{{exceedSize/(1000*1000*1000)}}G
</span>
</span> </span>
</span> </span>
</el-tree> </el-tree>
@@ -494,6 +512,12 @@
> >
{{ i18n.dataset_unzip_failed }} {{ i18n.dataset_unzip_failed }}
</span> </span>
<span
class="unzip-failed"
v-if="data.Size > exceedSize"
>
{{ i18n.dataset_exceeds_failed }}{{exceedSize/(1000*1000*1000)}}G
</span>
</span> </span>
</span> </span>
</el-tree> </el-tree>
@@ -625,6 +649,7 @@ export default {
paramsFavorite: { page: 1, q: "" }, paramsFavorite: { page: 1, q: "" },
loadingFavorite: false, loadingFavorite: false,
initFavoriteTreeNode: [], initFavoriteTreeNode: [],
exceedSize:0,
}; };
}, },
methods: { methods: {
@@ -883,6 +908,9 @@ export default {
if (curchild.DecompressState !== 1) { if (curchild.DecompressState !== 1) {
curchild.disabled = true; curchild.disabled = true;
} }
if(curchild.Size>this.exceedSize && this.exceedSize){
curchild.disabled = true;
}
curchild.ref = ref; curchild.ref = ref;
curchild.label = curchild.Name; curchild.label = curchild.Name;
preChild.push(curchild); preChild.push(curchild);
@@ -971,6 +999,7 @@ export default {
mounted() { mounted() {
this.type = $(".cloudbrain-type").data("cloudbrain-type"); this.type = $(".cloudbrain-type").data("cloudbrain-type");
this.repoLink = $(".cloudbrain-type").data("repo-link"); this.repoLink = $(".cloudbrain-type").data("repo-link");
this.exceedSize = $(".cloudbrain-type").data("exceed-size");
if ($(".cloudbrain-type").data("dataset-uuid")) { if ($(".cloudbrain-type").data("dataset-uuid")) {
this.hasSelectDatasetList = $(".cloudbrain-type") this.hasSelectDatasetList = $(".cloudbrain-type")
.data("dataset-uuid") .data("dataset-uuid")


+ 1
- 196
web_src/js/features/cloudbrainShow.js View File

@@ -544,6 +544,7 @@ export default async function initCloudrainSow() {
$.get(url, (data) => { $.get(url, (data) => {
if (data.StatusOK == 0) { // 成功 0 if (data.StatusOK == 0) { // 成功 0
if (data.Dirs) { if (data.Dirs) {
data.Dirs.length !==0 && $(`#${version_name}-result-down`).show()
renderDir(path, data, version_name, downloadFlag, gpuFlag); renderDir(path, data, version_name, downloadFlag, gpuFlag);
} }
if (init === "init") { if (init === "init") {
@@ -752,200 +753,4 @@ export default async function initCloudrainSow() {
html += "</div>"; html += "</div>";
$(`#dir_list${version_name}`).append(html); $(`#dir_list${version_name}`).append(html);
} }

let nameMap, nameList;
let RepoLink = $(".cloudbrain-type").data("repo-link");
let type = $(".cloudbrain-type").data("cloudbrain-type");
let flagModel = $(".cloudbrain-type").data("flag-model");
// 获取模型列表和模型名称对应的模型版本
$(document).ready(function () {
if (!flagModel) return;
else {
$.get(
`${RepoLink}/modelmanage/query_model_for_predict?type=${type}`,
(data) => {
nameMap = data.nameMap;
nameList = data.nameList;
let html = `<div class="item"></div>`;
nameList.forEach((element) => {
html += `<div class="item" data-value=${element}>${element}</div>`;
});
if (nameList.length !== 0) {
$("#model_name").append(html);
}
let faildModelName = $('input[name="model_name"]').val();
let faildModelVersion = $('input[name="model_version"]').val();
let dataID;

// 新建错误的表单返回初始化
if (faildModelName && nameList.includes(faildModelName)) {
$("#select_model").dropdown("set text", faildModelName);
$("#select_model").dropdown("set value", faildModelName);
nameMap[faildModelName].forEach((element) => {
if (element.Version === faildModelVersion) {
dataID = element.ID;
}
});
initModelVerison(faildModelName, nameMap, faildModelVersion);
initModelckpt(dataID);
}
}
);
}

$("#select_model").dropdown({
onChange: function (value, text, $selectedItem) {
$("#model_name_version").empty();
if (value) {
let html = "";
nameMap[value].forEach((element) => {
//let { trainTaskInfo } = element;
//trainTaskInfo = JSON.parse(trainTaskInfo);
html += `<div class="item" data-label="${element.label}" data-id="${element.id}" data-value="${element.path}">${element.version}</div>`;
});
$("#model_name_version").append(html);
const initVersionText = $(
"#model_name_version div.item:first-child"
).text();
const initVersionValue = $(
"#model_name_version div.item:first-child"
).data("value");

$("#select_model_version").dropdown("set text", initVersionText);
$("#select_model_version").dropdown(
"set value",
initVersionValue,
initVersionText,
$("#model_name_version div.item:first-child")
);
} else {
$("#select_model_version").dropdown("set text", "");
$("#select_model_version").dropdown("set value", "");
$("#select_model_checkpoint").dropdown("set text", "");
$("#select_model_checkpoint").dropdown("set value", "");
$("#model_checkpoint").empty();
}
},
});

$("#select_model_version").dropdown({
onChange: function (value, text, $selectedItem) {
if (!value) return;
const dataID =
$selectedItem && $selectedItem[0].getAttribute("data-id");
$("input#ai_model_version").val(text);
$("#select_model_checkpoint").dropdown("set text", "");
$("#select_model_checkpoint").addClass("loading");

$("#model_checkpoint").empty();
let html = "";
loadCheckpointList(dataID).then((res) => {
res.forEach((element) => {
const ckptSuffix = element.FileName.split(".");
const loadCheckpointFile = [
"ckpt",
"pb",
"h5",
"json",
"pkl",
"pth",
"t7",
"pdparams",
"onnx",
"pbtxt",
"keras",
"mlmodel",
"cfg",
"pt",
];
if (
!element.IsDir &&
loadCheckpointFile.includes(ckptSuffix[ckptSuffix.length - 1])
) {
html += `<div class="item" data-value="${element.FileName}">${element.FileName}</div>`;
}
});
$("#model_checkpoint").append(html);
$("#select_model_checkpoint").removeClass("loading");
if (html) {
$("#select_model_checkpoint").removeClass("error");
}
const initVersionText = $(
"#model_checkpoint div.item:first-child"
).text();
const initVersionValue = $(
"#model_checkpoint div.item:first-child"
).data("value");

$("#select_model_checkpoint").dropdown("set text", initVersionText);
$("#select_model_checkpoint").dropdown(
"set value",
initVersionValue,
initVersionText,
$("#model_name_version div.item:first-child")
);
});
},
});
});
function initModelVerison(value, nameMap, faildModelVersion) {
let faildTrainUrl = $('input[name="pre_train_model_url"]').val();
let html = "";
nameMap[value].forEach((element) => {
let { TrainTaskInfo } = element;
TrainTaskInfo = JSON.parse(TrainTaskInfo);
html += `<div class="item" data-label="${element.Label}" data-id="${element.ID}" data-value="${element.Path}">${element.Version}</div>`;
});
$("#model_name_version").append(html);
$("#select_model_version").dropdown("set text", faildModelVersion);
$("#select_model_version").dropdown("set value", faildTrainUrl);
}
function initModelckpt(dataID) {
let faildCkptName = $('input[name="ckpt_name"]').val();
$("#select_model_checkpoint").addClass("loading");
$("#model_checkpoint").empty();
let html = "";
loadCheckpointList(dataID).then((res) => {
res.forEach((element) => {
const ckptSuffix = element.FileName.split(".");
const loadCheckpointFile = [
"ckpt",
"pb",
"h5",
"json",
"pkl",
"pth",
"t7",
"pdparams",
"onnx",
"pbtxt",
"keras",
"mlmodel",
"cfg",
"pt",
];
if (
!element.IsDir &&
loadCheckpointFile.includes(ckptSuffix[ckptSuffix.length - 1])
) {
html += `<div class="item" data-value=${element.FileName}>${element.FileName}</div>`;
}
});
$("#model_checkpoint").append(html);
$("#select_model_checkpoint").removeClass("loading");
$("#select_model_checkpoint").dropdown("set text", faildCkptName);
$("#select_model_checkpoint").dropdown("set value", faildCkptName);
});
}
function loadCheckpointList(value) {
return new Promise((resolve, reject) => {
$.get(
`${RepoLink}/modelmanage/query_modelfile_for_predict`,
{ id: value },
(data) => {
resolve(data);
}
);
});
}
} }

+ 3
- 40
web_src/js/features/cloudrbanin.js View File

@@ -58,8 +58,8 @@ export default async function initCloudrain() {
const duration = data.JobDuration; const duration = data.JobDuration;
const aiCenter = data.AiCenter || '--' const aiCenter = data.AiCenter || '--'
$("#duration-" + ID).text(duration); $("#duration-" + ID).text(duration);
$("#cluster-" + ID).text(aiCenter);
$("#" + versionname + "-ai_center").text(data.AiCenter);
data.AiCenter != undefined && $("#cluster-" + ID).text(aiCenter);
data.AiCenter != undefined && $("#" + versionname + "-ai_center").text(data.AiCenter);
if (status != status_text) { if (status != status_text) {
$("#" + ID + "-icon") $("#" + ID + "-icon")
.removeClass() .removeClass()
@@ -224,7 +224,7 @@ export default async function initCloudrain() {
data.StartTime !== undefined && data.StartTime > 0 && $("#" + versionname + "-startTime").text(timeFormat(new Date(data.StartTime * 1000))); data.StartTime !== undefined && data.StartTime > 0 && $("#" + versionname + "-startTime").text(timeFormat(new Date(data.StartTime * 1000)));
$("#" + versionname + "-duration").text(data.JobDuration); $("#" + versionname + "-duration").text(data.JobDuration);
$("#" + versionname + "-status").text(data.JobStatus); $("#" + versionname + "-status").text(data.JobStatus);
$("#" + versionname + "-ai_center").text(data.AiCenter);
data.AiCenter != undefined && $("#" + versionname + "-ai_center").text(data.AiCenter);


if (stopArray.includes(data.JobStatus)) { if (stopArray.includes(data.JobStatus)) {
$("#" + versionname + "-stop").addClass("disabled"); $("#" + versionname + "-stop").addClass("disabled");
@@ -483,43 +483,6 @@ export default async function initCloudrain() {
const redirect_to = this.dataset.linkpath; const redirect_to = this.dataset.linkpath;
debugAgain(ID, repoPath, redirect_to); debugAgain(ID, repoPath, redirect_to);
}); });

function setWaitNums() {
if ($(".cloudbrain-type").length === 0 && $(".gpu-type").length === 0) {
return;
}
if (
$(".cloudbrain-type").length !== 0 &&
!$(".cloudbrain-type").data("queue")
) {
return;
}
let waitNums = $(".cloudbrain-type").data("queue").split("map")[1];
let test = new Map();
let waitNumsArray = waitNums.split(" ");
waitNumsArray.forEach((element, index) => {
if (index === 0) {
test.set(element.slice(1, -2), parseInt(element.slice(-1)));
} else if (index === waitNumsArray.length - 1) {
test.set(element.slice(0, -3), parseInt(element.slice(-2, -1)));
} else {
test.set(element.slice(0, -2), parseInt(element.slice(-1)));
}
});
$(".ui.search.dropdown.gpu-type").dropdown({
onChange: function (value, text, $selectedItem) {
let gpuTypeNums = test.get(value);
let gpuTypeNumString =
$(".cloudbrain-type").data("queue-start") +
" " +
gpuTypeNums +
" " +
$(".cloudbrain-type").data("queue-end");
$("#gpu-nums").text(gpuTypeNumString);
},
});
}
setWaitNums();
} }


function userSearchControll() { function userSearchControll() {


+ 2
- 0
web_src/js/features/i18nVue.js View File

@@ -74,6 +74,7 @@ export const i18nVue = {
dataset_search_placeholder: "搜数据集名称/描述...", dataset_search_placeholder: "搜数据集名称/描述...",
dataset_unziping: "正在解压缩", dataset_unziping: "正在解压缩",
dataset_unzip_failed: "解压失败", dataset_unzip_failed: "解压失败",
dataset_exceeds_failed: "数据集大小超过",
dataset_my_upload: "我上传的", dataset_my_upload: "我上传的",
dataset_current_repo: "本项目", dataset_current_repo: "本项目",
dataset_public: "公开数据集", dataset_public: "公开数据集",
@@ -195,6 +196,7 @@ export const i18nVue = {
dataset_search_placeholder: "Search dataset name/description ...", dataset_search_placeholder: "Search dataset name/description ...",
dataset_unziping: "Decompressing", dataset_unziping: "Decompressing",
dataset_unzip_failed: "Decompression failed", dataset_unzip_failed: "Decompression failed",
dataset_exceeds_failed: "Dataset size exceeds ",
dataset_my_upload: "Upload by me", dataset_my_upload: "Upload by me",
dataset_current_repo: "Current Repository", dataset_current_repo: "Current Repository",
dataset_public: "Public dataset", dataset_public: "Public dataset",


+ 244
- 1
web_src/js/standalone/cloudbrainNew.js View File

@@ -6,7 +6,7 @@
if (createFlag) return false; if (createFlag) return false;
createFlag = true; createFlag = true;
}; };
$("select.dropdown").dropdown();
// $("select.dropdown").dropdown();
$(document).keydown(function (event) { $(document).keydown(function (event) {
switch (event.keyCode) { switch (event.keyCode) {
case 13: case 13:
@@ -182,4 +182,247 @@
} }
validate(); validate();
}); });


//管理镜像相关的东西
let nameMap, nameList;
let RepoLink = $(".cloudbrain-type").data("repo-link");
let type = $(".cloudbrain-type").data("cloudbrain-type");
let flagModel = $(".cloudbrain-type").data("flag-model");
// 获取模型列表和模型名称对应的模型版本
$(document).ready(function () {
if (!flagModel) return;
else {
$.get(
`${RepoLink}/modelmanage/query_model_for_predict?type=${type}`,
(data) => {
nameMap = data.nameMap;
nameList = data.nameList;
let html = `<div class="item"></div>`;
nameList.forEach((element) => {
html += `<div class="item" data-value=${element}>${element}</div>`;
});
if (nameList.length !== 0) {
$("#model_name").append(html);
}
let faildModelName = $('input[name="model_name"]').val();
let faildModelVersion = $('input[name="model_version"]').val();
let dataID;
// 新建错误的表单返回初始化
if (faildModelName && nameList.includes(faildModelName)) {
$("#select_model").dropdown("set text", faildModelName);
$("#select_model").dropdown("set value", faildModelName);
nameMap[faildModelName].forEach((element) => {
if (element.version === faildModelVersion) {
dataID = element.id;
}
});
initModelVerison(faildModelName, nameMap, faildModelVersion);
initModelckpt(dataID);
}
}
);
}

$("#select_model").dropdown({
onChange: function (value, text, $selectedItem) {
$("#model_name_version").empty();
if (value) {
let html = "";
nameMap[value].forEach((element) => {
//let { trainTaskInfo } = element;
//trainTaskInfo = JSON.parse(trainTaskInfo);
html += `<div class="item" data-label="${element.label}" data-id="${element.id}" data-value="${element.path}">${element.version}</div>`;
});
$("#model_name_version").append(html);
const initVersionText = $(
"#model_name_version div.item:first-child"
).text();
const initVersionValue = $(
"#model_name_version div.item:first-child"
).data("value");

$("#select_model_version").dropdown("set text", initVersionText);
$("#select_model_version").dropdown(
"set value",
initVersionValue,
initVersionText,
$("#model_name_version div.item:first-child")
);
} else {
$("#select_model_version").dropdown("set text", "");
$("#select_model_version").dropdown("set value", "");
$("#select_model_checkpoint").dropdown("set text", "");
$("#select_model_checkpoint").dropdown("set value", "");
$("#model_checkpoint").empty();
}
},
});

$("#select_model_version").dropdown({
onChange: function (value, text, $selectedItem) {
if (!value) return;
const dataID =
$selectedItem && $selectedItem[0].getAttribute("data-id");
$("input#ai_model_version").val(text);
$("#select_model_checkpoint").dropdown("set text", "");
$("#select_model_checkpoint").addClass("loading");

$("#model_checkpoint").empty();
let html = "";
loadCheckpointList(dataID).then((res) => {
res.forEach((element) => {
const ckptSuffix = element.FileName.split(".");
const loadCheckpointFile = [
"ckpt",
"pb",
"h5",
"json",
"pkl",
"pth",
"t7",
"pdparams",
"onnx",
"pbtxt",
"keras",
"mlmodel",
"cfg",
"pt",
];
if (
!element.IsDir &&
loadCheckpointFile.includes(ckptSuffix[ckptSuffix.length - 1])
) {
html += `<div class="item" data-value="${element.FileName}">${element.FileName}</div>`;
}
});
$("#model_checkpoint").append(html);
$("#select_model_checkpoint").removeClass("loading");
if (html) {
$("#select_model_checkpoint").removeClass("error");
}
const initVersionText = $(
"#model_checkpoint div.item:first-child"
).text();
const initVersionValue = $(
"#model_checkpoint div.item:first-child"
).data("value");

$("#select_model_checkpoint").dropdown("set text", initVersionText);
$("#select_model_checkpoint").dropdown(
"set value",
initVersionValue,
initVersionText,
$("#model_name_version div.item:first-child")
);
});
},
});
});
function initModelVerison(value, nameMap, faildModelVersion) {
let faildTrainUrl = $('input[name="pre_train_model_url"]').val();
let html = "";
nameMap[value].forEach((element) => {
html += `<div class="item" data-label="${element.label}" data-id="${element.id}" data-value="${element.path}">${element.version}</div>`;
});
$("#model_name_version").append(html);
$("#select_model_version").dropdown("set text", faildModelVersion);
$("#select_model_version").dropdown("set value", faildTrainUrl);
}
function initModelckpt(dataID) {
let faildCkptName = $('input[name="ckpt_name"]').val();
$("#select_model_checkpoint").addClass("loading");
$("#model_checkpoint").empty();
let html = "";
loadCheckpointList(dataID).then((res) => {
res.forEach((element) => {
const ckptSuffix = element.FileName.split(".");
const loadCheckpointFile = [
"ckpt",
"pb",
"h5",
"json",
"pkl",
"pth",
"t7",
"pdparams",
"onnx",
"pbtxt",
"keras",
"mlmodel",
"cfg",
"pt",
];
if (
!element.IsDir &&
loadCheckpointFile.includes(ckptSuffix[ckptSuffix.length - 1])
) {
html += `<div class="item" data-value=${element.FileName}>${element.FileName}</div>`;
}
});
$("#model_checkpoint").append(html);
$("#select_model_checkpoint").removeClass("loading");
$("#select_model_checkpoint").dropdown("set text", faildCkptName);
$("#select_model_checkpoint").dropdown("set value", faildCkptName);
});
}
function loadCheckpointList(value) {
return new Promise((resolve, reject) => {
$.get(
`${RepoLink}/modelmanage/query_modelfile_for_predict`,
{ id: value },
(data) => {
resolve(data);
}
);
});
}
// 评测任务相关创建func
let repoLink = $(".cloudbrain-type").data("repo-link");
function setChildType(type_id=1) {
if (type_id == 3) {
$('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
$('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
} else {
$('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
$('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
}
let child_selected_id = $('#benchmark_child_types_id_hidden').val();
$.get(`${repoLink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => {
const n_length = data['child_types'].length
let html = ''
for (let i = 0; i < n_length; i++) {
if (child_selected_id == data['child_types'][i].id) {
html += `<option value="${data['child_types'][i].id}" selected="true">${data['child_types'][i].value}</option>`;
} else {
html += `<option value="${data['child_types'][i].id}">${data['child_types'][i].value}</option>`;
}
}
document.getElementById("benchmark_child_types_id").innerHTML = html;
})
}
$(document).ready(function () {
if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') {
setChildType();
}
$(".ui.selection.dropdown.benchmark_types_id").dropdown({
onChange:function (value, text, $selectedItem){
setChildType(value)
}
})
$('.ui.search.dropdown.job_type').dropdown({
onChange: function (value, text, $selectedItem) {
if (value === "BRAINSCORE") {
$('#brainscore_child_type').css('display', 'block')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann')
}else if(value === "SNN4ECOSET"){
$('#brainscore_child_type').css('display', 'none')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4ecoset')
}else {
$('#brainscore_child_type').css('display', 'none')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet')
}
}
})
})
})(); })();

+ 2
- 0
web_src/less/openi.less View File

@@ -752,6 +752,8 @@ i.SUCCEEDED {
.disabled { .disabled {
cursor: pointer; cursor: pointer;
pointer-events: none; pointer-events: none;
opacity: .45!important;
color: rgba(0,0,0,.6);
} }
.left2 { .left2 {
margin-left: -2px !important; margin-left: -2px !important;


+ 11
- 2
web_src/vuepages/apis/modules/notobook.js View File

@@ -29,7 +29,7 @@ export const getCb1Notebook = (path,jobid) => {
}); });
}; };


// Notebook获取云脑I调试任务状态
// Notebook获取云脑II调试任务状态
export const getCb2Notebook = (path,jobid) => { export const getCb2Notebook = (path,jobid) => {
return service({ return service({
url: `/api/v1/${path}/modelarts/notebook/${jobid}`, url: `/api/v1/${path}/modelarts/notebook/${jobid}`,
@@ -37,7 +37,16 @@ export const getCb2Notebook = (path,jobid) => {
params: {}, params: {},
}); });
}; };

// Notebook查询文件在环境中是否已准备好
// type, file, branch_name, owner_name, project_name,job_id
export const getFileInfoNotebook = (data) => {
return service({
url: "/api/v1/file_notebook/status",
method: "post",
data,
params: {},
});
};
export const stopNotebook = (url) => { export const stopNotebook = (url) => {
return service({ return service({
url: url, url: url,


+ 60
- 29
web_src/vuepages/pages/notebook/debug/index.vue View File

@@ -178,9 +178,9 @@
</div> </div>
</template> </template>
<script> <script>
import { getFileNotebook,createNotebook,getCb1Notebook,getCb2Notebook,stopNotebook } from "~/apis/modules/notobook";
import { getFileNotebook,createNotebook,getCb1Notebook,getCb2Notebook,getFileInfoNotebook,stopNotebook } from "~/apis/modules/notobook";
import { Message } from "element-ui"; import { Message } from "element-ui";
let timerCb1,timerCb2
let timerCb1,timerCb2,timerCb3
let {AppSubUrl} = window.config let {AppSubUrl} = window.config
const finalState = [ const finalState = [
"STOPPED", "STOPPED",
@@ -253,27 +253,28 @@ export default {
if(this.activeLoadFirst){ if(this.activeLoadFirst){
this.loading = true this.loading = true
} }
getFileNotebook().then((res)=>{
if(res.data.code==0){
this.notebookInfo = res.data
}else{
Message.error(res.data.message)
}
this.loading = false
this.activeLoadFirst = false
}).catch((err)=>{
Message.error(err)
this.loading = false
this.activeLoadFirst = false
})
getFileNotebook().then((res)=>{
if(res.data.code==0){
this.notebookInfo = res.data
}else{
Message.error(res.data.message)
}
this.loading = false
this.activeLoadFirst = false
}).catch((err)=>{
Message.error(err)
this.loading = false
this.activeLoadFirst = false
})
}, },
getCb1NotebookInfo(path,id,index){
getCb1NotebookInfo(path,id,index,data){
getCb1Notebook(path,id).then((res)=>{ getCb1Notebook(path,id).then((res)=>{
if(res.status===200){ if(res.status===200){
if(res.data.JobStatus==="RUNNING"){ if(res.data.JobStatus==="RUNNING"){
this.btnStatus[index]=2
this.deubgUrlGpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${id}/debug`
this.deubgUrlGpuStop = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${id}/stop`
let fileData = {job_id:id,...data}
timerCb3 = setInterval(() => {
setTimeout(this.getFileInfoReadyNotebook(fileData,index), 0)
}, 5000)
clearInterval(timerCb1) clearInterval(timerCb1)
} }
if(finalState.includes(res.data.JobStatus)){ if(finalState.includes(res.data.JobStatus)){
@@ -281,23 +282,53 @@ export default {
clearInterval(timerCb1) clearInterval(timerCb1)
} }
} }
})
}).catch((err)=>{
this.btnStatus[index]=0
clearInterval(timerCb1)
Message.error(err)
})
}, },
getCb2NotebookInfo(path,id){
getCb2NotebookInfo(path,id,data){
getCb2Notebook(path,id).then((res)=>{ getCb2Notebook(path,id).then((res)=>{
if(res.status===200){ if(res.status===200){
if(res.data.JobStatus==="RUNNING"){
this.btnStatus[2]=2
this.deubgUrlNpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${id}/debug`
this.deubgUrlNpuStop = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${id}/stop`
clearInterval(timerCb2)
if(res.data.JobStatus==="RUNNING"){
let fileData = {job_id:id,...data}
timerCb3 = setInterval(() => {
setTimeout(this.getFileInfoReadyNotebook(fileData,2), 0)
}, 5000)
clearInterval(timerCb2)
} }
if(finalState.includes(res.data.JobStatus)){ if(finalState.includes(res.data.JobStatus)){
this.btnStatus[2] = 0 this.btnStatus[2] = 0
clearInterval(timerCb2) clearInterval(timerCb2)
} }
} }
})
}).catch((err)=>{
this.btnStatus[index]=0
clearInterval(timerCb2)
Message.error(err)
})
},
getFileInfoReadyNotebook(data,index){
getFileInfoNotebook(data).then((res)=>{
console.log(res)
if(res.data.code===0){
if(index===2){
this.btnStatus[2]=2
this.deubgUrlNpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${data.job_id}/debug?file=${this.fileInfo.owner_name}/${this.fileInfo.project_name}/${this.fileInfo.file}`
this.deubgUrlNpuStop = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${data.job_id}/stop`
}else{
this.btnStatus[index]=2
this.deubgUrlGpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${data.job_id}/debug?file=${this.fileInfo.owner_name}/${this.fileInfo.project_name}/${this.fileInfo.file}`
this.deubgUrlGpuStop = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${data.job_id}/stop`
}
clearInterval(timerCb3)
}
}).catch((err)=>{
this.btnStatus[index]=0
clearInterval(timerCb3)
Message.error(err)
})
}, },
stopDebug(index){ stopDebug(index){
this.btnStatus[index]=3 this.btnStatus[index]=3
@@ -323,11 +354,11 @@ export default {
if(res.data.code===0 && res.status===200){ if(res.data.code===0 && res.status===200){
if(index===2){ if(index===2){
timerCb2 = setInterval(() => { timerCb2 = setInterval(() => {
setTimeout(this.getCb2NotebookInfo(repoPath,res.data.message), 0)
setTimeout(this.getCb2NotebookInfo(repoPath,res.data.message,data), 0)
}, 10000) }, 10000)
}else{ }else{
timerCb1 = setInterval(() => { timerCb1 = setInterval(() => {
setTimeout(this.getCb1NotebookInfo(repoPath,res.data.message,index), 0)
setTimeout(this.getCb1NotebookInfo(repoPath,res.data.message,index,data), 0)
}, 10000) }, 10000)
} }
this.alertCb = false this.alertCb = false


Loading…
Cancel
Save