diff --git a/models/attachment.go b/models/attachment.go index 2b747db21..2788ef559 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) { return getAttachmentsByCommentID(x, commentID) } +func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) { + attach := &Attachment{DatasetID: datasetId, Name: fileName} + has, err := x.Get(attach) + if err != nil { + return nil, err + } else if !has { + return nil, err + } + return attach, nil +} + func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) return attachments, e.Where("comment_id=?", commentID).Find(&attachments) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index db98bb662..7b19c120d 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -60,6 +60,7 @@ const ( JobTypeModelSafety JobType = "MODELSAFETY" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" + JobTypeSnn4Ecoset JobType = "SNN4ECOSET" JobTypeTrain JobType = "TRAIN" JobTypeInference JobType = "INFERENCE" @@ -335,6 +336,9 @@ func IsModelArtsDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool { return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) } +func IsModelBenchMarkJobType(jobType string) bool { + return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset) +} func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { isActivated := result.JobStatus.CreatedTime > 0 diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 827e2b736..19b1ab506 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -32,10 +32,10 @@ const ( Snn4imagenetMountPath = "/snn4imagenet" BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" - Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` - BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` - - SubTaskName = "task1" + Snn4imagenetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /benchmark/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt` + Snn4EcosetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + SubTaskName = "task1" Success = "S000" @@ -259,20 +259,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { }, { HostPath: models.StHostPath{ - Path: req.Snn4ImageNetPath, - MountPath: Snn4imagenetMountPath, - ReadOnly: true, - }, - }, - { - HostPath: models.StHostPath{ - Path: req.BrainScorePath, - MountPath: BrainScoreMountPath, - ReadOnly: true, - }, - }, - { - HostPath: models.StHostPath{ Path: req.ResultPath, MountPath: ResultPath, ReadOnly: false, @@ -406,7 +392,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { } func IsBenchmarkJob(jobType string) bool { - return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType + return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType } func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { @@ -675,6 +661,7 @@ func IsElementExist(s []string, str string) bool { return false } + func GetCloudBrainByIdOrJobId(id string, initialQuery string) (*models.Cloudbrain, error) { _, err := strconv.ParseInt(id, 10, 64) var job *models.Cloudbrain diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 977b5daba..09e7259f2 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -544,6 +544,12 @@ var ( BrainScoreName string BrainScoreServerHost string + IsSnn4EcosetEnabled bool + Snn4EcosetOwner string + Snn4EcosetName string + Snn4EcosetServerHost string + Snn4AttachmentName string + //blockchain config BlockChainHost string CommitValidDate string @@ -1524,6 +1530,13 @@ func NewContext() { BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") + sec = Cfg.Section("snn4ecoset") + IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false) + Snn4EcosetOwner = sec.Key("OWNER").MustString("") + Snn4EcosetName = sec.Key("NAME").MustString("") + Snn4EcosetServerHost = sec.Key("HOST").MustString("") + Snn4AttachmentName = sec.Key("DATASET").MustString("") + sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") diff --git a/package-lock.json b/package-lock.json index 7dc42d04c..ee922450a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1982,28 +1982,6 @@ "object.assign": "^4.1.0" } }, - "babel-polyfill": { - "version": "6.26.0", - "resolved": "https://registry.npm.taobao.org/babel-polyfill/download/babel-polyfill-6.26.0.tgz", - "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=", - "requires": { - "babel-runtime": "^6.26.0", - "core-js": "^2.5.0", - "regenerator-runtime": "^0.10.5" - }, - "dependencies": { - "core-js": { - "version": "2.6.12", - "resolved": "https://registry.npm.taobao.org/core-js/download/core-js-2.6.12.tgz?cache=0&sync_timestamp=1611040749668&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcore-js%2Fdownload%2Fcore-js-2.6.12.tgz", - "integrity": "sha1-2TM9+nsGXjR8xWgiGdb2kIWcwuw=" - }, - "regenerator-runtime": { - "version": "0.10.5", - "resolved": "https://registry.npm.taobao.org/regenerator-runtime/download/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=" - } - } - }, "babel-runtime": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", @@ -2194,9 +2172,9 @@ "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "blueimp-md5": { - "version": "2.18.0", - "resolved": "https://registry.npm.taobao.org/blueimp-md5/download/blueimp-md5-2.18.0.tgz", - "integrity": "sha1-EVK+EzXwxrORHtnjbbVPPmrFKTU=" + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz", + "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==" }, "bn.js": { "version": "5.1.1", @@ -3536,6 +3514,11 @@ "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz", "integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI=" }, + "dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "de-indent": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", @@ -4152,25 +4135,29 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "esdk-obs-browserjs": { - "version": "3.20.7", - "resolved": "https://registry.npm.taobao.org/esdk-obs-browserjs/download/esdk-obs-browserjs-3.20.7.tgz", - "integrity": "sha1-vhziRlKEhW3PgZPl0DyX68bJI0s=", + "version": "3.22.3", + "resolved": "https://registry.npmjs.org/esdk-obs-browserjs/-/esdk-obs-browserjs-3.22.3.tgz", + "integrity": "sha512-MATZXp0FwjPtKG9tpdfURa3koUarR/ev+tbO0oUKgj0GRt0798ZxmfCvYvRpgNst4w1ht4E79ikD4H40UYLgPA==", "requires": { - "axios": "^0.19.0", - "babel-polyfill": "^6.26.0", - "blueimp-md5": "^2.10.0", - "js-base64": "^2.3.2", - "jssha": "^2.3.1", - "urijs": "^1.19.1" + "axios": "^0.26.1", + "blueimp-md5": "^2.18.0", + "js-base64": "^3.7.1", + "jssha": "^3.2.0", + "urijs": "^1.19.7" }, "dependencies": { "axios": { - "version": "0.19.2", - "resolved": "https://registry.npm.taobao.org/axios/download/axios-0.19.2.tgz?cache=0&sync_timestamp=1608609215811&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faxios%2Fdownload%2Faxios-0.19.2.tgz", - "integrity": "sha1-PqNsXYgY0NX4qKl6bTa4bNwAyyc=", + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", "requires": { - "follow-redirects": "1.5.10" + "follow-redirects": "^1.14.8" } + }, + "js-base64": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.3.tgz", + "integrity": "sha512-PAr6Xg2jvd7MCR6Ld9Jg3BmTcjYsHEBx1VlwEwULb/qowPf5VD9kEMagj23Gm7JRnSvE/Da/57nChZjnvL8v6A==" } } }, @@ -5382,27 +5369,9 @@ } }, "follow-redirects": { - "version": "1.5.10", - "resolved": "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.5.10.tgz?cache=0&sync_timestamp=1611606737937&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.5.10.tgz", - "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", - "requires": { - "debug": "=3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz?cache=0&sync_timestamp=1607566533140&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-3.1.0.tgz", - "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "fomantic-ui": { "version": "2.8.4", @@ -7884,9 +7853,9 @@ } }, "jssha": { - "version": "2.4.2", - "resolved": "https://registry.npm.taobao.org/jssha/download/jssha-2.4.2.tgz", - "integrity": "sha1-2VCwlWNJKL1rK9odQtqaOnYtZek=" + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.0.tgz", + "integrity": "sha512-w9OtT4ALL+fbbwG3gw7erAO0jvS5nfvrukGPMWIAoea359B26ALXGpzy4YJSp9yGnpUvuvOw1nSjSoHDfWSr1w==" }, "just-debounce": { "version": "1.0.0", @@ -14400,9 +14369,9 @@ } }, "urijs": { - "version": "1.19.6", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.6.tgz", - "integrity": "sha512-eSXsXZ2jLvGWeLYlQA3Gh36BcjF+0amo92+wHPyN1mdR8Nxf75fuEuYTd9c0a+m/vhCjRK0ESlE9YNLW+E1VEw==" + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" }, "urix": { "version": "0.1.0", diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 91685251b..2002e69f4 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index bb04038b9..4710bf974 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -623,7 +623,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), - string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} + string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -730,7 +730,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b01c0ade8..264410ac2 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -399,7 +399,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { req.ModelVersion = form.ModelVersion req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl req.PreTrainModelUrl = form.PreTrainModelUrl - } _, err = cloudbrain.GenerateTask(req) @@ -747,6 +746,7 @@ func CloudBrainRestart(ctx *context.Context) { }) } + func hasDatasetDeleted(task *models.Cloudbrain) bool { if task.Uuid == "" { return false @@ -855,7 +855,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo if task.JobType == string(models.JobTypeBenchmark) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { + } else if models.IsModelBenchMarkJobType(task.JobType) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkTypeName = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType @@ -1704,6 +1704,8 @@ func GetRate(ctx *context.Context) { ctx.Redirect(setting.Snn4imagenetServerHost) } else if job.JobType == string(models.JobTypeBrainScore) { ctx.Redirect(setting.BrainScoreServerHost) + } else if job.JobType == string(models.JobTypeSnn4Ecoset) { + ctx.Redirect(setting.Snn4EcosetServerHost) } else { log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) } @@ -2176,7 +2178,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset), string(models.JobTypeModelSafety)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, @@ -2209,14 +2211,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { ciTasks[i].BenchmarkTypeName = "" if ciTasks[i].JobType == string(models.JobTypeBenchmark) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) { + } else if models.IsModelBenchMarkJobType(ciTasks[i].JobType) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost - } else { + } else if ciTasks[i].JobType == string(models.JobTypeBrainScore) { ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost + } else { + ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost } } @@ -2566,7 +2570,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := form.Image - uuid := form.Attachment jobType := form.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath branchName := cloudbrain.DefaultBranchName @@ -2608,7 +2611,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) return } - if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { + if !models.IsModelBenchMarkJobType(jobType) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("jobtype error", tpl, &form) @@ -2637,29 +2640,41 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) mkModelPath(modelPath) uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") - snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath + benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name) - uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/") - command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description)) + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") + command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } benchmarkChildTypeID := 0 - brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath + if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name) - uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") benchmarkChildTypeID = form.BenchmarkChildTypeID - command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) + command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } + var uuid string + var datasetInfos map[string]models.DatasetInfo + var datasetNames string + if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) { + downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") + command = fmt.Sprintf(cloudbrain.Snn4EcosetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) + + attachment, err := getEcosetAttachment() + if err != nil { + log.Error("load benchmark code failed", err) + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) + return + } + uuid = attachment.UUID + datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid) - datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) - if err != nil { - log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx, jobType) - ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) - return } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeBenchmark, ComputeResource: models.GPU, @@ -2691,8 +2706,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), - Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), JobType: jobType, Description: form.Description, BranchName: branchName, @@ -2704,6 +2717,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), Spec: spec, } + if form.ModelName != "" { + req.ModelName = form.ModelName + req.LabelName = form.LabelName + req.CkptName = form.CkptName + req.ModelVersion = form.ModelVersion + req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl + } _, err = cloudbrain.GenerateTask(req) if err != nil { @@ -2715,6 +2736,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } +func getEcosetAttachment() (*models.Attachment, error) { + ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName) + if err != nil { + return nil, err + } + + datasetInfo, err := models.GetDatasetByRepo(ecosetRepo) + if err != nil { + return nil, err + } + + return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID) + +} + func getBrainRegion(benchmarkChildTypeID int) string { values := []string{"V1", "V2", "V4", "IT"} return values[benchmarkChildTypeID] diff --git a/routers/user/home.go b/routers/user/home.go index 62b0357ad..e4ff50a45 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 4ae742c3a..172fa1502 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s ComputeResource: models.GPUResource, }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, - JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, + JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset}, NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { @@ -76,7 +76,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { jobNewType := jobType - if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) { + if models.IsModelBenchMarkJobType(jobType) { jobNewType = string(models.JobTypeBenchmark) } diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index d1e42b54a..db7d52ba5 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -1,37 +1,9 @@ {{template "base/head" .}} - {{template "custom/global_mask" .}}
{{template "repo/header" .}}
+ {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -78,21 +50,12 @@ onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)">{{.description}}
- - -
+
  - @@ -116,22 +79,10 @@ {{.i18n.Tr "cloudbrain.view_sample"}}
- + {{template "custom/select_model" .}}
- {{template "custom/select_dataset_train" .}} - +
{{range .benchmark_types}} {{if eq .Id $.benchmarkTypeID}} @@ -226,26 +168,13 @@
-
- -
- -
- -
- {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} - {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} - 模型安全评测 -
-
- -
-
- - {{template "custom/task_wait_count" .}} -
-
- - - {{.i18n.Tr "repo.cloudbrain_jobname_err"}} -
-
- - -
- - -
-
- -
- - - - {{.i18n.Tr "custom.resource_description"}} -
- -
- - - {{.i18n.Tr "cloudbrain.view_sample"}} -
- - -
+
- {{.i18n.Tr "repo.cloudbrain.cancel"}} + {{.i18n.Tr "repo.cloudbrain.cancel"}}
- {{end}}
{{template "base/footer" .}} +