From c35c8f2eb4cdbac91f23b75fe40bc0e4a0dd9407 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 14 Dec 2022 09:51:35 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E8=AF=84=E6=B5=8B?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 11 +++++ models/cloudbrain.go | 16 +++--- modules/cloudbrain/cloudbrain.go | 23 ++++++--- modules/setting/setting.go | 45 ++++++++++------- routers/admin/cloudbrains.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 4 +- routers/repo/cloudbrain.go | 76 +++++++++++++++++++++++------ routers/user/home.go | 2 +- services/cloudbrain/cloudbrainTask/count.go | 4 +- 9 files changed, 132 insertions(+), 51 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 2b747db21..2788ef559 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) { return getAttachmentsByCommentID(x, commentID) } +func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) { + attach := &Attachment{DatasetID: datasetId, Name: fileName} + has, err := x.Get(attach) + if err != nil { + return nil, err + } else if !has { + return nil, err + } + return attach, nil +} + func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) return attachments, e.Where("comment_id=?", commentID).Find(&attachments) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index aeed8629c..1a150181e 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -60,6 +60,7 @@ const ( JobTypeModelSafety JobType = "MODELSAFETY" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" + JobTypeSnn4Ecoset JobType = "SNN4ECOSET" JobTypeTrain JobType = "TRAIN" JobTypeInference JobType = "INFERENCE" @@ -204,7 +205,7 @@ type Cloudbrain struct { BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp - Cleared bool `xorm:"DEFAULT false"` + Cleared bool `xorm:"DEFAULT false"` Spec *Specification `xorm:"-"` } @@ -328,6 +329,9 @@ func IsModelArtsDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool { return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) } +func IsBenchMarkJobType(jobType string) bool { + return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset) +} func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { isActivated := result.JobStatus.CreatedTime > 0 @@ -1906,7 +1910,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } -func IsCloudbrainExistByJobName(jobName string)(bool,error){ +func IsCloudbrainExistByJobName(jobName string) (bool, error) { return x.Unscoped().Exist(&Cloudbrain{ JobName: jobName, }) @@ -2070,25 +2074,25 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra Limit(limit). Find(&cloudbrains) } + /** 本方法考虑了再次调试的情况，多次调试取最后一次的任务的结束时间 - */ +*/ func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 - sql:=`SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) + sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) id, job_name, job_id,status,end_time,updated_unix,cleared FROM cloudbrain where type=0 and job_type='DEBUG' ORDER BY job_name, updated_unix DESC) a where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix/model/benchmark-log.txt` - BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` + Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel' -d '%s' >/model/benchmark-log.txt` SubTaskName = "task1" @@ -60,6 +61,7 @@ type GenerateCloudBrainTaskReq struct { ModelPath string BenchmarkPath string Snn4ImageNetPath string + Snn4EcosetPath string BrainScorePath string JobType string Description string @@ -288,6 +290,15 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { }, }) } + if req.Snn4EcosetPath != "" { //ecoset benchmark + volumes = append(volumes, models.Volume{ + HostPath: models.StHostPath{ + Path: req.Snn4EcosetPath, + MountPath: Snn4EcosetMountPath, + ReadOnly: true, + }, + }) + } if len(req.DatasetInfos) == 1 { volumes = append(volumes, models.Volume{ @@ -405,7 +416,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { } func IsBenchmarkJob(jobType string) bool { - return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType + return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType } func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { @@ -653,7 +664,7 @@ func IsElementExist(s []string, str string) bool { return false } -func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) { +func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain, error) { _, err := strconv.ParseInt(id, 10, 64) var job *models.Cloudbrain if err != nil { @@ -661,10 +672,10 @@ func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) { job, err = models.GetCloudbrainByJobID(id) } else { job, err = models.GetCloudbrainByID(id) - if err!=nil{ + if err != nil { job, err = models.GetCloudbrainByJobID(id) } } - return job,err + return job, err } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index bf7eb2352..15bd21841 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -519,7 +519,6 @@ var ( CullIdleTimeout string CullInterval string - //benchmark config IsBenchmarkEnabled bool BenchmarkOwner string @@ -544,6 +543,12 @@ var ( BrainScoreName string BrainScoreServerHost string + IsSnn4EcosetEnabled bool + Snn4EcosetOwner string + Snn4EcosetName string + Snn4EcosetServerHost string + Snn4AttachmentName string + //blockchain config BlockChainHost string CommitValidDate string @@ -614,14 +619,14 @@ var ( UsageRateBeginTime string }{} - ClearStrategy= struct { - Enabled bool - ResultSaveDays int - BatchSize int - DebugJobSize int - TrashSaveDays int - Cron string - RunAtStart bool + ClearStrategy = struct { + Enabled bool + ResultSaveDays int + BatchSize int + DebugJobSize int + TrashSaveDays int + Cron string + RunAtStart bool }{} C2NetInfos *C2NetSqInfos @@ -1515,6 +1520,12 @@ func NewContext() { BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") + sec = Cfg.Section("snn4ecoset") + IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false) + Snn4EcosetOwner = sec.Key("OWNER").MustString("") + Snn4EcosetName = sec.Key("NAME").MustString("") + Snn4imagenetServerHost = sec.Key("HOST").MustString("") + sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") @@ -1691,16 +1702,16 @@ func getModelartsCDConfig() { getNotebookFlavorInfos() } -func getClearStrategy(){ +func getClearStrategy() { sec := Cfg.Section("clear_strategy") - ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false) - ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) - ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) - ClearStrategy.DebugJobSize=sec.Key("DEBUG_BATCH_SIZE").MustInt(100) - ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) - ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?") - ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) + ClearStrategy.Enabled = sec.Key("ENABLED").MustBool(false) + ClearStrategy.ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt(30) + ClearStrategy.BatchSize = sec.Key("BATCH_SIZE").MustInt(500) + ClearStrategy.DebugJobSize = sec.Key("DEBUG_BATCH_SIZE").MustInt(100) + ClearStrategy.TrashSaveDays = sec.Key("TRASH_SAVE_DAYS").MustInt(90) + ClearStrategy.Cron = sec.Key("CRON").MustString("* 0,30 2-8 * * ?") + ClearStrategy.RunAtStart = sec.Key("RUN_AT_START").MustBool(false) } func getGrampusConfig() { diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 91685251b..2002e69f4 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0d68fff30..d22af04e2 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -647,7 +647,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), - string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} + string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -754,7 +754,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a23cd5462..42d433005 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -379,7 +379,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { req.ModelVersion = form.ModelVersion req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl req.PreTrainModelUrl = form.PreTrainModelUrl - } _, err = cloudbrain.GenerateTask(req) @@ -711,7 +710,6 @@ func CloudBrainRestart(ctx *context.Context) { }) } - func getOldJobPath(task *models.Cloudbrain) string { return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName } @@ -786,7 +784,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo if task.JobType == string(models.JobTypeBenchmark) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { + } else if models.IsBenchMarkJobType(task.JobType) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkTypeName = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType @@ -854,10 +852,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile!=""{ - ctx.Redirect(getFileUrl(debugUrl,task.BootFile)) + if task.BootFile != "" { + ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) - }else{ + } else { ctx.Redirect(debugUrl) } @@ -1617,6 +1615,8 @@ func GetRate(ctx *context.Context) { ctx.Redirect(setting.Snn4imagenetServerHost) } else if job.JobType == string(models.JobTypeBrainScore) { ctx.Redirect(setting.BrainScoreServerHost) + } else if job.JobType == string(models.JobTypeSnn4Ecoset) { + ctx.Redirect(setting.Snn4EcosetServerHost) } else { log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) } @@ -2117,14 +2117,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { ciTasks[i].BenchmarkTypeName = "" if ciTasks[i].JobType == string(models.JobTypeBenchmark) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) { + } else if models.IsBenchMarkJobType(ciTasks[i].JobType) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost - } else { + } else if ciTasks[i].JobType == string(models.JobTypeBrainScore) { ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost + } else { + ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost } } @@ -2474,7 +2476,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := form.Image - uuid := form.Attachment jobType := form.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath branchName := cloudbrain.DefaultBranchName @@ -2516,7 +2517,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) return } - if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { + if !models.IsBenchMarkJobType(jobType) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("jobtype error", tpl, &form) @@ -2560,14 +2561,33 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) benchmarkChildTypeID = form.BenchmarkChildTypeID command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) } + snn4EcosetPath := setting.JobPath + jobName + cloudbrain.Snn4EcosetMountPath + var uuid string + var datasetInfos map[string]models.DatasetInfo + var datasetNames string + if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) { + err = downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, snn4EcosetPath, "", "", ctx.User.Name) + if err != nil { + log.Error("load benchmark code failed", err) + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) + return + } + err = uploadCodeToMinio(snn4EcosetPath+"/", jobName, cloudbrain.Snn4EcosetMountPath+"/") + command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) + + attachment, err := getEcosetAttachment() + if err != nil { + log.Error("load benchmark code failed", err) + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) + return + } + uuid = attachment.UUID + datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid) - datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) - if err != nil { - log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx, jobType) - ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) - return } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeBenchmark, ComputeResource: models.GPU, @@ -2601,6 +2621,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + Snn4EcosetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4EcosetMountPath+"/"), JobType: jobType, Description: form.Description, BranchName: branchName, @@ -2612,6 +2633,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), Spec: spec, } + if form.ModelName != "" { + req.ModelName = form.ModelName + req.LabelName = form.LabelName + req.CkptName = form.CkptName + req.ModelVersion = form.ModelVersion + req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl + } _, err = cloudbrain.GenerateTask(req) if err != nil { @@ -2623,6 +2652,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } +func getEcosetAttachment() (*models.Attachment, error) { + ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName) + if err != nil { + return nil, err + } + + datasetInfo, err := models.GetDatasetByRepo(ecosetRepo) + if err != nil { + return nil, err + } + + return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID) + +} + func getBrainRegion(benchmarkChildTypeID int) string { values := []string{"V1", "V2", "V4", "IT"} return values[benchmarkChildTypeID] diff --git a/routers/user/home.go b/routers/user/home.go index 62b0357ad..e4ff50a45 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 985706911..97a9f6f28 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s ComputeResource: models.GPUResource, }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, - JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, + JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset}, NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { @@ -66,7 +66,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { jobNewType := jobType - if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) { + if models.IsBenchMarkJobType(jobType) { jobNewType = string(models.JobTypeBenchmark) } From 4ddd3abab34a30bf28bc0fd838e3f157e650652a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 14 Dec 2022 15:56:50 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 33 ++++----------------------------- modules/setting/setting.go | 3 ++- routers/repo/cloudbrain.go | 39 ++++++++++++--------------------------- 3 files changed, 18 insertions(+), 57 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 005a06ecd..eb0fd5cfa 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -30,12 +30,11 @@ const ( BenchMarkResourceID = 1 Snn4imagenetMountPath = "/snn4imagenet" BrainScoreMountPath = "/brainscore" - Snn4EcosetMountPath = "/snn4ecoset" TaskInfoName = "/taskInfo" - Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel' --modeldescription '%s' >/model/benchmark-log.txt` - BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel' -d '%s' >/model/benchmark-log.txt` - - SubTaskName = "task1" + Snn4imagenetCommand = `/opt/conda/bin/python /code/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /code/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt` + Snn4EcosetCommand = `/opt/conda/bin/python /code/testSNN_script_user.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + SubTaskName = "task1" Success = "S000" @@ -61,7 +60,6 @@ type GenerateCloudBrainTaskReq struct { ModelPath string BenchmarkPath string Snn4ImageNetPath string - Snn4EcosetPath string BrainScorePath string JobType string Description string @@ -260,20 +258,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { }, { HostPath: models.StHostPath{ - Path: req.Snn4ImageNetPath, - MountPath: Snn4imagenetMountPath, - ReadOnly: true, - }, - }, - { - HostPath: models.StHostPath{ - Path: req.BrainScorePath, - MountPath: BrainScoreMountPath, - ReadOnly: true, - }, - }, - { - HostPath: models.StHostPath{ Path: req.ResultPath, MountPath: ResultPath, ReadOnly: false, @@ -290,15 +274,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { }, }) } - if req.Snn4EcosetPath != "" { //ecoset benchmark - volumes = append(volumes, models.Volume{ - HostPath: models.StHostPath{ - Path: req.Snn4EcosetPath, - MountPath: Snn4EcosetMountPath, - ReadOnly: true, - }, - }) - } if len(req.DatasetInfos) == 1 { volumes = append(volumes, models.Volume{ diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 15bd21841..8fd2df842 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1524,7 +1524,8 @@ func NewContext() { IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false) Snn4EcosetOwner = sec.Key("OWNER").MustString("") Snn4EcosetName = sec.Key("NAME").MustString("") - Snn4imagenetServerHost = sec.Key("HOST").MustString("") + Snn4EcosetServerHost = sec.Key("HOST").MustString("") + Snn4AttachmentName = sec.Key("DATASET").MustString("") sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 42d433005..483fb74ca 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2477,8 +2477,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := form.Image jobType := form.JobType - codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath - branchName := cloudbrain.DefaultBranchName + repo := ctx.Repo.Repository tpl := tplCloudBrainBenchmarkNew @@ -2539,42 +2538,31 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) } } - downloadCode(repo, codePath, branchName) - uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") - modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" mkModelPath(modelPath) uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") - snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name) - uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/") - command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description)) + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, codePath, "", "", ctx.User.Name) + uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } benchmarkChildTypeID := 0 - brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name) - uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, codePath, "", "", ctx.User.Name) + uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") benchmarkChildTypeID = form.BenchmarkChildTypeID - command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) + command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } - snn4EcosetPath := setting.JobPath + jobName + cloudbrain.Snn4EcosetMountPath var uuid string var datasetInfos map[string]models.DatasetInfo var datasetNames string if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) { - err = downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, snn4EcosetPath, "", "", ctx.User.Name) - if err != nil { - log.Error("load benchmark code failed", err) - cloudBrainNewDataPrepare(ctx, jobType) - ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) - return - } - err = uploadCodeToMinio(snn4EcosetPath+"/", jobName, cloudbrain.Snn4EcosetMountPath+"/") - command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) + downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, codePath, "", "", ctx.User.Name) + uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + command = fmt.Sprintf(cloudbrain.Snn4EcosetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) attachment, err := getEcosetAttachment() if err != nil { @@ -2619,12 +2607,9 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), - Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), - Snn4EcosetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4EcosetMountPath+"/"), JobType: jobType, Description: form.Description, - BranchName: branchName, + BranchName: cloudbrain.DefaultBranchName, BootFile: form.BootFile, Params: form.Params, CommitID: "", From afd9b31a1c5946729d1e922f5e32f9b3e8c4356f Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 11:19:23 +0800 Subject: [PATCH 03/10] fix issue --- package-lock.json | 99 +++---- templates/repo/cloudbrain/benchmark/new.tmpl | 402 +++++++++------------------ web_src/js/standalone/cloudbrainNew.js | 48 +++- 3 files changed, 215 insertions(+), 334 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7dc42d04c..ee922450a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1982,28 +1982,6 @@ "object.assign": "^4.1.0" } }, - "babel-polyfill": { - "version": "6.26.0", - "resolved": "https://registry.npm.taobao.org/babel-polyfill/download/babel-polyfill-6.26.0.tgz", - "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=", - "requires": { - "babel-runtime": "^6.26.0", - "core-js": "^2.5.0", - "regenerator-runtime": "^0.10.5" - }, - "dependencies": { - "core-js": { - "version": "2.6.12", - "resolved": "https://registry.npm.taobao.org/core-js/download/core-js-2.6.12.tgz?cache=0&sync_timestamp=1611040749668&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcore-js%2Fdownload%2Fcore-js-2.6.12.tgz", - "integrity": "sha1-2TM9+nsGXjR8xWgiGdb2kIWcwuw=" - }, - "regenerator-runtime": { - "version": "0.10.5", - "resolved": "https://registry.npm.taobao.org/regenerator-runtime/download/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=" - } - } - }, "babel-runtime": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", @@ -2194,9 +2172,9 @@ "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "blueimp-md5": { - "version": "2.18.0", - "resolved": "https://registry.npm.taobao.org/blueimp-md5/download/blueimp-md5-2.18.0.tgz", - "integrity": "sha1-EVK+EzXwxrORHtnjbbVPPmrFKTU=" + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz", + "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==" }, "bn.js": { "version": "5.1.1", @@ -3536,6 +3514,11 @@ "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz", "integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI=" }, + "dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "de-indent": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", @@ -4152,25 +4135,29 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "esdk-obs-browserjs": { - "version": "3.20.7", - "resolved": "https://registry.npm.taobao.org/esdk-obs-browserjs/download/esdk-obs-browserjs-3.20.7.tgz", - "integrity": "sha1-vhziRlKEhW3PgZPl0DyX68bJI0s=", + "version": "3.22.3", + "resolved": "https://registry.npmjs.org/esdk-obs-browserjs/-/esdk-obs-browserjs-3.22.3.tgz", + "integrity": "sha512-MATZXp0FwjPtKG9tpdfURa3koUarR/ev+tbO0oUKgj0GRt0798ZxmfCvYvRpgNst4w1ht4E79ikD4H40UYLgPA==", "requires": { - "axios": "^0.19.0", - "babel-polyfill": "^6.26.0", - "blueimp-md5": "^2.10.0", - "js-base64": "^2.3.2", - "jssha": "^2.3.1", - "urijs": "^1.19.1" + "axios": "^0.26.1", + "blueimp-md5": "^2.18.0", + "js-base64": "^3.7.1", + "jssha": "^3.2.0", + "urijs": "^1.19.7" }, "dependencies": { "axios": { - "version": "0.19.2", - "resolved": "https://registry.npm.taobao.org/axios/download/axios-0.19.2.tgz?cache=0&sync_timestamp=1608609215811&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faxios%2Fdownload%2Faxios-0.19.2.tgz", - "integrity": "sha1-PqNsXYgY0NX4qKl6bTa4bNwAyyc=", + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", "requires": { - "follow-redirects": "1.5.10" + "follow-redirects": "^1.14.8" } + }, + "js-base64": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.3.tgz", + "integrity": "sha512-PAr6Xg2jvd7MCR6Ld9Jg3BmTcjYsHEBx1VlwEwULb/qowPf5VD9kEMagj23Gm7JRnSvE/Da/57nChZjnvL8v6A==" } } }, @@ -5382,27 +5369,9 @@ } }, "follow-redirects": { - "version": "1.5.10", - "resolved": "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.5.10.tgz?cache=0&sync_timestamp=1611606737937&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.5.10.tgz", - "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", - "requires": { - "debug": "=3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz?cache=0&sync_timestamp=1607566533140&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-3.1.0.tgz", - "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "fomantic-ui": { "version": "2.8.4", @@ -7884,9 +7853,9 @@ } }, "jssha": { - "version": "2.4.2", - "resolved": "https://registry.npm.taobao.org/jssha/download/jssha-2.4.2.tgz", - "integrity": "sha1-2VCwlWNJKL1rK9odQtqaOnYtZek=" + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.0.tgz", + "integrity": "sha512-w9OtT4ALL+fbbwG3gw7erAO0jvS5nfvrukGPMWIAoea359B26ALXGpzy4YJSp9yGnpUvuvOw1nSjSoHDfWSr1w==" }, "just-debounce": { "version": "1.0.0", @@ -14400,9 +14369,9 @@ } }, "urijs": { - "version": "1.19.6", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.6.tgz", - "integrity": "sha512-eSXsXZ2jLvGWeLYlQA3Gh36BcjF+0amo92+wHPyN1mdR8Nxf75fuEuYTd9c0a+m/vhCjRK0ESlE9YNLW+E1VEw==" + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" }, "urix": { "version": "0.1.0", diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index d1e42b54a..669fd42ab 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -1,37 +1,9 @@ {{template "base/head" .}} - {{template "custom/global_mask" .}}
{{template "repo/header" .}}
+ {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -78,21 +50,12 @@ onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)">{{.description}}
- - -
+
  - @@ -116,22 +79,10 @@ {{.i18n.Tr "cloudbrain.view_sample"}}
- + {{template "custom/select_model" .}}
- {{template "custom/select_dataset_train" .}} - +
{{range .benchmark_types}} {{if eq .Id $.benchmarkTypeID}} @@ -226,26 +168,13 @@
-
- -
- -
- -
- {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} - {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} - 模型安全评测 -
-
- -
-
- - {{template "custom/task_wait_count" .}} -
-
- - - {{.i18n.Tr "repo.cloudbrain_jobname_err"}} -
-
- - -
- - -
-
- -
- - - - {{.i18n.Tr "custom.resource_description"}} -
- -
- - - {{.i18n.Tr "cloudbrain.view_sample"}} -
- - -
+
- {{.i18n.Tr "repo.cloudbrain.cancel"}} + {{.i18n.Tr "repo.cloudbrain.cancel"}}
- {{end}}
{{template "base/footer" .}} + diff --git a/web_src/js/standalone/cloudbrainNew.js b/web_src/js/standalone/cloudbrainNew.js index bc54adc65..1823312db 100644 --- a/web_src/js/standalone/cloudbrainNew.js +++ b/web_src/js/standalone/cloudbrainNew.js @@ -6,7 +6,7 @@ if (createFlag) return false; createFlag = true; }; - $("select.dropdown").dropdown(); + // $("select.dropdown").dropdown(); $(document).keydown(function (event) { switch (event.keyCode) { case 13: @@ -182,4 +182,50 @@ } validate(); }); + + // 评测任务相关创建func + let repoLink = $(".cloudbrain-type").data("repo-link"); + function setChildType(type_id=1) { + if (type_id == 3) { + $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark'); + $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark'); + } else { + $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark'); + $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark'); + } + let child_selected_id = $('#benchmark_child_types_id_hidden').val(); + $.get(`${repoLink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => { + const n_length = data['child_types'].length + let html = '' + for (let i = 0; i < n_length; i++) { + if (child_selected_id == data['child_types'][i].id) { + html += ``; + } else { + html += ``; + } + } + document.getElementById("benchmark_child_types_id").innerHTML = html; + }) + } + $(document).ready(function () { + if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') { + setChildType(); + } + $(".ui.selection.dropdown.benchmark_types_id").dropdown({ + onChange:function (value, text, $selectedItem){ + setChildType(value) + } + }) + $('.ui.search.dropdown.job_type').dropdown({ + onChange: function (value, text, $selectedItem) { + if (value === "BRAINSCORE") { + $('#brainscore_child_type').css('display', 'block') + $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann') + } else { + $('#brainscore_child_type').css('display', 'none') + $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet') + } + } + }) + }) })(); From e6b3253add4985da944c968d4b98d2445de942cb Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 21 Dec 2022 11:24:40 +0800 Subject: [PATCH 04/10] fix issue --- templates/repo/cloudbrain/benchmark/new.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index 669fd42ab..3772d99f5 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -3,7 +3,7 @@
{{template "repo/header" .}}
- + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} From b0f18ff074e744fa37c8519a38299bd9ca8f2652 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 21 Dec 2022 17:35:47 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E6=8F=90=E4=BA=A4diam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 7508b70d3..6608736e8 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2148,7 +2148,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset), string(models.JobTypeModelSafety)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, From 733f1d95e973f590092510330ea573ae19725549 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 22 Dec 2022 10:39:05 +0800 Subject: [PATCH 06/10] fix issue --- templates/repo/cloudbrain/benchmark/new.tmpl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index 3772d99f5..db7d52ba5 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -340,15 +340,15 @@ // validate() // }) - // ;(function() { - // var SPECS = {{ .benchmark_specs }}; - // var showPoint = {{ .CloudBrainPaySwitch }}; - // window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { - // gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, - // free: {{$.i18n.Tr "cloudbrain.free"}}, - // point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, - // memory: {{$.i18n.Tr "cloudbrain.memory"}}, - // shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, - // }); - // })(); + ;(function() { + var SPECS = {{ .benchmark_specs }}; + var showPoint = {{ .CloudBrainPaySwitch }}; + window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { + gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, + free: {{$.i18n.Tr "cloudbrain.free"}}, + point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, + memory: {{$.i18n.Tr "cloudbrain.memory"}}, + shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, + }); + })(); From d7efd833c35c6e4b2343117793af6de71d6834f6 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 22 Dec 2022 14:45:18 +0800 Subject: [PATCH 07/10] fix issue --- web_src/js/standalone/cloudbrainNew.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web_src/js/standalone/cloudbrainNew.js b/web_src/js/standalone/cloudbrainNew.js index 1823312db..4033efaa7 100644 --- a/web_src/js/standalone/cloudbrainNew.js +++ b/web_src/js/standalone/cloudbrainNew.js @@ -221,7 +221,10 @@ if (value === "BRAINSCORE") { $('#brainscore_child_type').css('display', 'block') $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann') - } else { + }else if(value === "SNN4ECOSET"){ + $('#brainscore_child_type').css('display', 'none') + $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4ecoset') + }else { $('#brainscore_child_type').css('display', 'none') $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet') } From efca9ac3166ae53be1afada88c304239a9f44029 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 23 Dec 2022 09:32:12 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 2 +- modules/cloudbrain/cloudbrain.go | 6 +++--- routers/repo/cloudbrain.go | 29 +++++++++++++++++------------ services/cloudbrain/cloudbrainTask/count.go | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index c028ab3e7..9b069365d 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -336,7 +336,7 @@ func IsModelArtsDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool { return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) } -func IsBenchMarkJobType(jobType string) bool { +func IsModelBenchMarkJobType(jobType string) bool { return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset) } diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 35e5cf903..adc308b04 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -32,9 +32,9 @@ const ( Snn4imagenetMountPath = "/snn4imagenet" BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" - Snn4imagenetCommand = `/opt/conda/bin/python /code/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` - BrainScoreCommand = `bash /code/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt` - Snn4EcosetCommand = `/opt/conda/bin/python /code/testSNN_script_user.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + Snn4imagenetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /benchmark/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt` + Snn4EcosetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` SubTaskName = "task1" Success = "S000" diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 6608736e8..51a1acba1 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -843,7 +843,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo if task.JobType == string(models.JobTypeBenchmark) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if models.IsBenchMarkJobType(task.JobType) { + } else if models.IsModelBenchMarkJobType(task.JobType) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkTypeName = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType @@ -2181,7 +2181,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { ciTasks[i].BenchmarkTypeName = "" if ciTasks[i].JobType == string(models.JobTypeBenchmark) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if models.IsBenchMarkJobType(ciTasks[i].JobType) { + } else if models.IsModelBenchMarkJobType(ciTasks[i].JobType) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType @@ -2541,7 +2541,8 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := form.Image jobType := form.JobType - + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + branchName := cloudbrain.DefaultBranchName repo := ctx.Repo.Repository tpl := tplCloudBrainBenchmarkNew @@ -2580,7 +2581,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) return } - if !models.IsBenchMarkJobType(jobType) { + if !models.IsModelBenchMarkJobType(jobType) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("jobtype error", tpl, &form) @@ -2602,21 +2603,25 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) } } + downloadCode(repo, codePath, branchName) + uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" mkModelPath(modelPath) uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") - codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, codePath, "", "", ctx.User.Name) - uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } benchmarkChildTypeID := 0 + if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, codePath, "", "", ctx.User.Name) - uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") benchmarkChildTypeID = form.BenchmarkChildTypeID command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } @@ -2624,8 +2629,8 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) var datasetInfos map[string]models.DatasetInfo var datasetNames string if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) { - downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, codePath, "", "", ctx.User.Name) - uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") command = fmt.Sprintf(cloudbrain.Snn4EcosetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) attachment, err := getEcosetAttachment() @@ -2673,7 +2678,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), JobType: jobType, Description: form.Description, - BranchName: cloudbrain.DefaultBranchName, + BranchName: branchName, BootFile: form.BootFile, Params: form.Params, CommitID: "", diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 98ba0afb7..172fa1502 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -76,7 +76,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { jobNewType := jobType - if models.IsBenchMarkJobType(jobType) { + if models.IsModelBenchMarkJobType(jobType) { jobNewType = string(models.JobTypeBenchmark) } From 51c205d1c725a390bf6def72bb8d1d78d788f398 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 4 Jan 2023 09:40:45 +0800 Subject: [PATCH 09/10] fix issue --- templates/repo/cloudbrain/benchmark/show.tmpl | 45 ++++++++++++++------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl index 0c0ef1a52..45e8d0058 100755 --- a/templates/repo/cloudbrain/benchmark/show.tmpl +++ b/templates/repo/cloudbrain/benchmark/show.tmpl @@ -206,7 +206,7 @@ - + {{if eq .JobType "BENCHMARK"}} @@ -228,16 +226,30 @@ + {{else}} + + + + + + + + + + {{end}} - - - - - +
{{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_train"}} @@ -214,11 +214,9 @@
- {{if eq .JobType "BENCHMARK"}} + train.py - {{else}} - -- - {{end}} +
-
- {{if eq .JobType "BENCHMARK"}} test.py - {{else}} - -- - {{end}}
+ {{$.i18n.Tr "repo.modelarts.model_name"}} + +
{{if .ModelName}}{{.ModelName}}{{else}}--{{end}}
+
+ {{$.i18n.Tr "repo.modelconvert.modelversion"}} + +
{{if .ModelVersion}}{{.ModelVersion}}{{else}}--{{end}}
+
{{$.i18n.Tr "repo.modelarts.train_job.description"}} @@ -271,17 +283,7 @@
- {{$.i18n.Tr "repo.model_manager"}} - -
- {{.DatasetName}} -
-
{{$.i18n.Tr "repo.cloudbrain_creator"}} @@ -423,6 +425,7 @@ })();