diff --git a/Makefile b/Makefile index f35d5fa2a..8e7f46fea 100755 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/))) -WEBPACK_SOURCES := $(shell find web_src/js web_src/less -type f) +WEBPACK_SOURCES := $(shell find web_src/js web_src/less web_src/vuepages -type f) WEBPACK_CONFIGS := webpack.config.js WEBPACK_CONFIGS_TMP := webpack_temp.config.js WEBPACK_CONFIGS_PRO := webpack_pro.config.js diff --git a/models/action.go b/models/action.go index d0a763e08..4b6f1dbad 100755 --- a/models/action.go +++ b/models/action.go @@ -65,7 +65,6 @@ const ( ActionCreateImage //36 ActionImageRecommend //37 ActionChangeUserAvatar //38 - ) // Action represents user operation type and other information to diff --git a/models/cloudbrain.go b/models/cloudbrain.go index bcaea544a..6135dac40 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -53,8 +53,11 @@ const ( JobFailed CloudbrainStatus = "FAILED" JobRunning CloudbrainStatus = "RUNNING" + ModelSafetyTesting CloudbrainStatus = "TESTING" + JobTypeDebug JobType = "DEBUG" JobTypeBenchmark JobType = "BENCHMARK" + JobTypeModelSafety JobType = "MODELSAFETY" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" @@ -172,26 +175,26 @@ type Cloudbrain struct { ImageID string //grampus image_id AiCenter string //grampus ai center: center_id+center_name - TrainUrl string //输出模型的obs路径 - BranchName string //分支名称 - Parameters string //传给modelarts的param参数 - BootFile string //启动文件 - DataUrl string //数据集的obs路径 - LogUrl string //日志输出的obs路径 - PreVersionId int64 //父版本的版本id - FlavorCode string //modelarts上的规格id - Description string `xorm:"varchar(256)"` //描述 - WorkServerNumber int //节点数 - FlavorName string //规格名称 - EngineName string //引擎名称 - TotalVersionCount int //任务的所有版本数量，包括删除的 - LabelName string //标签名称 - ModelName string //模型名称 - ModelVersion string //模型版本 - CkptName string //权重文件名称 - PreTrainModelUrl string //预训练模型地址 - ResultUrl string //推理结果的obs路径 - + TrainUrl string //输出模型的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id + Description string `xorm:"varchar(256)"` //描述 + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 + LabelName string //标签名称 + ModelName string //模型名称 + ModelVersion string //模型版本 + CkptName string //权重文件名称 + PreTrainModelUrl string //预训练模型地址 + ResultUrl string //推理结果的obs路径 + ResultJson string `xorm:"varchar(4000)"` User *User `xorm:"-"` Repo *Repository `xorm:"-"` BenchmarkType string `xorm:"-"` //算法评测，模型评测 @@ -1490,6 +1493,8 @@ type GrampusTasks struct { ReplicaNum int `json:"replicaNum"` Datasets []GrampusDataset `json:"datasets"` Models []GrampusDataset `json:"models"` + Code GrampusDataset `json:"code"` + BootFile string `json:"bootFile"` } type GrampusDataset struct { @@ -2003,6 +2008,13 @@ func GetStoppedJobWithNoStartTimeEndTime() ([]*Cloudbrain, error) { return cloudbrains, x.SQL("select * from cloudbrain where status in (?,?,?,?,?,?,?) and (start_time is null or end_time is null) limit 100", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded).Find(&cloudbrains) } +func GetModelSafetyTestTask() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0) + sess := x.Where("job_type = ?", string(JobTypeModelSafety)) + err := sess.Find(&cloudbrains) + return cloudbrains, err +} + func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) return int(count), err @@ -2017,7 +2029,12 @@ func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) { } func GetBenchmarkCountByUserID(userID int64) (int, error) { - count, err := x.In("status", JobWaiting, JobRunning).And("(job_type = ? or job_type = ? or job_type = ?) and user_id = ? and type = ?", string(JobTypeBenchmark), string(JobTypeBrainScore), string(JobTypeSnn4imagenet), userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + count, err := x.In("status", JobWaiting, JobRunning).And("(job_type = ? or job_type = ? or job_type = ?) and user_id = ? and type = ?", string(JobTypeBenchmark), string(JobTypeModelSafety), string(JobTypeBrainScore), string(JobTypeSnn4imagenet), userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + return int(count), err +} + +func GetModelSafetyCountByUserID(userID int64) (int, error) { + count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain)) return int(count), err } @@ -2281,8 +2298,7 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", - "created_unix", "start_time", "end_time").Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 58a93d476..48df111a0 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -34,6 +34,7 @@ type TaskDetail struct { CardDuration string `json:"CardDuration"` AiCenter string `json:"AiCenter"` FlavorName string `json:"FlavorName"` + WorkServerNum int64 `json:"WorkServerNum"` Spec *Specification `json:"Spec"` } @@ -44,16 +45,45 @@ func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) return x.SQL(countSql).Count() } func GetTodayCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count FROM " + + countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) return x.SQL(countSql).Count() } +func GetTodayRunningCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + " and (status='" + string(JobRunning) + "'" + + " or status='" + string(ModelArtsTrainJobInit) + "')" + return x.SQL(countSql).Count() +} +func GetTodayWaitingCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + " and status='" + string(JobWaiting) + "'" + return x.SQL(countSql).Count() +} + func GetCreatorCount() (int64, error) { countSql := "SELECT count(distinct user_id) FROM public.cloudbrain" return x.SQL(countSql).Count() } +func GetCloudbrainTypeCount() ([]map[string]string, error) { + countSql := "SELECT type,count(*) num FROM public.cloudbrain group by type order by num desc" + return x.QueryString(countSql) +} + +func GetCloudbrainStatusCount() ([]map[string]string, error) { + countSql := "SELECT status,count(*) num FROM public.cloudbrain group by status order by num desc" + return x.QueryString(countSql) +} + +func GetCloudbrainTpyeDurationSum() ([]map[string]string, error) { + countSql := "SELECT type,sum(duration) FROM public.cloudbrain group by type order by sum(duration) desc" + return x.QueryString(countSql) +} + func GetRecordBeginTime() ([]*CloudbrainInfo, error) { sess := x.NewSession() defer sess.Close() diff --git a/models/repo.go b/models/repo.go index 6e5ee3ff6..2c4fda39b 100755 --- a/models/repo.go +++ b/models/repo.go @@ -679,7 +679,7 @@ func (repo *Repository) getAssignees(e Engine) (_ []*User, err error) { userIDs[i] = accesses[i].UserID } - if err = e.In("id", userIDs).Find(&users); err != nil { + if err = e.In("id", userIDs).OrderBy("name asc").Find(&users); err != nil { return nil, err } } diff --git a/modules/aisafety/resty.go b/modules/aisafety/resty.go new file mode 100644 index 000000000..be6468529 --- /dev/null +++ b/modules/aisafety/resty.go @@ -0,0 +1,249 @@ +package aisafety + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "sort" + "strings" + + "code.gitea.io/gitea/modules/log" + + "github.com/go-resty/resty/v2" +) + +var ( + restyClient *resty.Client + HOST string + KEY string +) + +type TaskReq struct { + UnionId string //评测任务ID，唯一标识，由后端生成UUID + EvalName string //评测任务名称 + EvalContent string //评测任务描述 + TLPath string // + Indicators []string //评测指标，由GetAlgorithmList接口返回的指标列表中的title属性值 + CDPath string + CDName string //对抗数据集名称 + BDPath string + BDName string //原数据集名称 +} + +type ReturnMsg struct { + Code string `json:"code"` + Msg string `json:"msg"` + Data ReturnData `json:"data"` + Times int64 `json:"times"` +} + +type ReturnData struct { + ID int `json:"id"` + No string `json:"no"` + StandardJson string `json:"standardJson"` + Code int `json:"code"` + Msg string `json:"msg"` + Status int `json:"status"` +} + +const ( + APPID = "1" + LogPageSize = 500 + LogPageTokenExpired = "5m" + pageSize = 15 + Success = "S000" +) + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + } + return restyClient +} + +func checkSetting() { + if len(HOST) != 0 && len(KEY) != 0 { + return + } + _ = loginCloudbrain() +} + +func loginCloudbrain() error { + HOST = "http://221.122.70.196:8081/atp-api" + KEY = "1" + return nil +} + +func createSign(params map[string]interface{}, signKey string) string { + var sceneList []string + for k := range params { + sceneList = append(sceneList, k) + } + sort.Strings(sceneList) + re := "" + for _, key := range sceneList { + if params[key] != nil { + re += key + "=" + fmt.Sprint(params[key]) + "&" + } + } + re += "key=" + signKey + log.Info("sign key:" + re) + h := md5.New() + h.Write([]byte(re)) + return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) +} + +func getParams(req TaskReq) (map[string]interface{}, string) { + params := make(map[string]interface{}) + reStr := "" + + params["unionId"] = req.UnionId + reStr += "unionId=" + req.UnionId + params["evalName"] = req.EvalName + reStr += "&evalName=" + req.EvalName + params["evalContent"] = req.EvalContent + reStr += "&evalContent=" + url.QueryEscape(req.EvalContent) + params["TLPath"] = req.TLPath + reStr += "&TLPath=" + url.QueryEscape(req.TLPath) + + params["CDName"] = req.CDName + reStr += "&CDName=" + url.QueryEscape(req.CDName) + + params["BDName"] = req.BDName + reStr += "&BDName=" + url.QueryEscape(req.BDName) + + if req.CDPath != "" { + params["CDPath"] = req.CDPath + reStr += "&CDPath=" + url.QueryEscape(req.CDPath) + } + if req.BDPath != "" { + params["BDPath"] = req.BDPath + reStr += "&BDPath=" + url.QueryEscape(req.BDPath) + } + indicators := "" + if len(req.Indicators) > 0 { + for _, tmp := range req.Indicators { + indicators += tmp + "|" + } + } + if len(indicators) > 0 { + indicators = indicators[0 : len(indicators)-1] + } + + params["Indicators"] = indicators + log.Info("indicators=" + indicators) + reStr += "&Indicators=" + url.QueryEscape(indicators) + + return params, reStr +} + +func CreateSafetyTask(req TaskReq, jsonstr string) (string, error) { + checkSetting() + client := getRestyClient() + + //reqPara, _ := json.Marshal(body) + //log.Warn("job req:", jsonstr) + + params, urlQuerys := getParams(req) + + bodyMap := make(map[string]interface{}) + //reJsonMap := make(map[string]interface{}) + bodyMap["resJson"] = jsonstr + //bodyMap["externalEvalParam"] = reJsonMap + + //reqPara, _ := json.Marshal(bodyMap) + //log.Warn("job req json:", string(reqPara)) + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetHeader("appId", APPID). + SetHeader("sign", createSign(params, KEY)). + //SetAuthToken(TOKEN). + SetBody(bodyMap). + Post(HOST + "/v1/external/eval-standard/create?" + urlQuerys) + log.Info("url=" + HOST + "/v1/external/eval-standard/create?" + urlQuerys) + + responseStr := string(res.Body()) + log.Info("CreateSafetyTask responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) + + if err != nil { + return "", fmt.Errorf("resty create job: %s", err) + } else { + log.Info("result string=" + " res code=" + fmt.Sprint(res.StatusCode())) + } + + reMap := make(map[string]interface{}) + + err = json.Unmarshal(res.Body(), &reMap) + if err == nil && reMap["code"] == "0" { + dataMap := (reMap["data"]).(map[string]interface{}) + return fmt.Sprint(dataMap["serialNo"]), nil + } + return "", nil +} + +func GetAlgorithmList() (map[string]interface{}, error) { + checkSetting() + client := getRestyClient() + params := make(map[string]interface{}) + + jsonResult := make(map[string]interface{}) + sign := createSign(params, KEY) + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetHeader("appId", APPID). + SetHeader("sign", sign). + Get(HOST + "/v1/external/eval-standard/algorithmList") + + log.Info("url=" + HOST + "/v1/external/eval-standard/algorithmList" + " sign=" + sign + " appId=" + APPID) + + jsonerr := json.Unmarshal(res.Body(), &jsonResult) + if jsonerr == nil { + log.Info("jsonResult code=" + fmt.Sprint(jsonResult["msg"])) + } + responseStr := string(res.Body()) + log.Info("GetAlgorithmList responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) + + if err != nil { + log.Info("error =" + err.Error()) + return nil, fmt.Errorf("resty GetJob: %v", err) + } else { + reMap := make(map[string]interface{}) + err = json.Unmarshal(res.Body(), &reMap) + if err == nil && reMap["code"] == "0" { + return reMap, nil + } else { + return nil, fmt.Errorf("get error,code not 0") + } + } + +} + +func GetTaskStatus(jobID string) (*ReturnMsg, error) { + checkSetting() + client := getRestyClient() + var reMap ReturnMsg + params := make(map[string]interface{}) + params["serialNo"] = jobID + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetHeader("appId", APPID). + SetHeader("sign", createSign(params, KEY)). + SetResult(&reMap). + Get(HOST + "/v1/external/eval-standard/query?serialNo=" + jobID) + + log.Info("url=" + HOST + "/v1/external/eval-standard/query?serialNo=" + jobID) + responseStr := string(res.Body()) + log.Info("GetTaskStatus responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) + + if err != nil { + log.Info("error =" + err.Error()) + return nil, fmt.Errorf("Get task status error: %v", err) + } else { + return &reMap, nil + } +} diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 4e527b6bf..e57bd8d7e 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -20,19 +20,19 @@ import ( const ( //Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` - CommandBenchmark = `cd /benchmark && bash run_bk.sh >/model/benchmark-log.txt` - CodeMountPath = "/code" - DataSetMountPath = "/dataset" - ModelMountPath = "/model" + CommandBenchmark = `cd /benchmark && bash run_bk.sh >/model/benchmark-log.txt` + CodeMountPath = "/code" + DataSetMountPath = "/dataset" + ModelMountPath = "/model" PretrainModelMountPath = "/pretrainmodel" - LogFile = "log.txt" - BenchMarkMountPath = "/benchmark" - BenchMarkResourceID = 1 - Snn4imagenetMountPath = "/snn4imagenet" - BrainScoreMountPath = "/brainscore" - TaskInfoName = "/taskInfo" - Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` - BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` + LogFile = "log.txt" + BenchMarkMountPath = "/benchmark" + BenchMarkResourceID = 1 + Snn4imagenetMountPath = "/snn4imagenet" + BrainScoreMountPath = "/brainscore" + TaskInfoName = "/taskInfo" + Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` SubTaskName = "task1" @@ -405,7 +405,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { } func IsBenchmarkJob(jobType string) bool { - return string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType + return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType } func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 04cd7fe41..8dbc8d1ed 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,12 +5,13 @@ package cron import ( - "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/services/cloudbrain/resource" - "code.gitea.io/gitea/modules/modelarts" "context" "time" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/migrations" repository_service "code.gitea.io/gitea/modules/repository" @@ -122,6 +123,17 @@ func registerHandleUnDecompressAttachment() { }) } +func registerHandleModelSafetyTask() { + RegisterTaskFatal("handle_modelsafety_task", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 5m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.TimerHandleModelSafetyTestTask() + return nil + }) +} + func registerHandleBlockChainUnSuccessUsers() { RegisterTaskFatal("handle_blockchain_unsuccess_users", &BaseConfig{ Enabled: true, @@ -279,4 +291,6 @@ func initBasicTasks() { //registerRewardPeriodTask() registerCloudbrainPointDeductTask() + + registerHandleModelSafetyTask() } diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index d72a7b10e..7bacb46d3 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -1,6 +1,7 @@ package grampus import ( + "code.gitea.io/gitea/modules/cloudbrain" "encoding/json" "strings" @@ -73,6 +74,7 @@ type GenerateTrainJobReq struct { PreTrainModelPath string PreTrainModelUrl string Spec *models.Specification + CodeName string } func getEndPoint() string { @@ -102,6 +104,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error centerID, centerName := getCentersParamter(ctx, req) var datasetGrampus, modelGrampus []models.GrampusDataset + var codeGrampus models.GrampusDataset if ProcessorTypeNPU == req.ProcessType { datasetGrampus = getDatasetGrampus(req.DatasetInfos) if len(req.ModelName) != 0 { @@ -114,6 +117,12 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error }, } } + codeGrampus = models.GrampusDataset{ + Name: req.CodeName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ObjectKey: req.CodeObsPath + cloudbrain.DefaultBranchName + ".zip", + } } jobResult, err := createJob(models.CreateGrampusJobRequest{ @@ -130,6 +139,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error ReplicaNum: 1, Datasets: datasetGrampus, Models: modelGrampus, + Code: codeGrampus, + BootFile: req.BootFile, }, }, }) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 6b3d1f128..06521993e 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,7 +1,6 @@ package modelarts import ( - "code.gitea.io/gitea/modules/modelarts_cd" "encoding/json" "errors" "fmt" @@ -9,6 +8,8 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/modelarts_cd" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -70,8 +71,8 @@ const ( var ( poolInfos *models.PoolInfos TrainFlavorInfos *Flavor - SpecialPools *models.SpecialPools - MultiNodeConfig *MultiNodes + SpecialPools *models.SpecialPools + MultiNodeConfig *MultiNodes ) type GenerateTrainJobReq struct { @@ -141,6 +142,7 @@ type GenerateInferenceJobReq struct { ResultUrl string Spec *models.Specification DatasetName string + JobType string } type VersionInfo struct { @@ -173,12 +175,12 @@ type ResourcePool struct { } `json:"resource_pool"` } -type MultiNodes struct{ +type MultiNodes struct { Info []OrgMultiNode `json:"multinode"` } -type OrgMultiNode struct{ +type OrgMultiNode struct { Org string `json:"org"` - Node []int `json:"node"` + Node []int `json:"node"` } // type Parameter struct { @@ -709,7 +711,7 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e Status: models.TempJobStatus, Type: models.TypeCloudBrainTwo, JobName: req.JobName, - JobType: string(models.JobTypeInference), + JobType: req.JobType, }) if err != nil { log.Error("InsertCloudbrainTemp failed: %v", err.Error()) @@ -732,7 +734,7 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e JobID: jobID, JobName: req.JobName, DisplayJobName: req.DisplayJobName, - JobType: string(models.JobTypeInference), + JobType: req.JobType, Type: models.TypeCloudBrainTwo, VersionID: jobResult.VersionID, VersionName: jobResult.VersionName, @@ -769,7 +771,15 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) return err } - notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateInferenceTask) + if req.JobType == string(models.JobTypeModelSafety) { + task, err := models.GetCloudbrainByJobID(jobID) + if err == nil { + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, fmt.Sprint(task.ID), req.DisplayJobName, models.ActionCreateBenchMarkTask) + } + } else { + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateInferenceTask) + } + return nil } @@ -798,8 +808,8 @@ func InitSpecialPool() { } } -func InitMultiNode(){ - if MultiNodeConfig ==nil && setting.ModelArtsMultiNode!=""{ +func InitMultiNode() { + if MultiNodeConfig == nil && setting.ModelArtsMultiNode != "" { json.Unmarshal([]byte(setting.ModelArtsMultiNode), &MultiNodeConfig) } diff --git a/modules/repository/create.go b/modules/repository/create.go index 0844c43c3..049fe3abf 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "strings" + "text/template" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" @@ -34,7 +35,7 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *m Name: opts.Name, Alias: opts.Alias, LowerName: strings.ToLower(opts.Name), - Description: opts.Description, + Description: template.HTMLEscapeString(opts.Description), OriginalURL: opts.OriginalURL, OriginalServiceType: opts.GitServiceType, IsPrivate: opts.IsPrivate, diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d6e4824ef..c6afae05a 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -394,7 +394,7 @@ var ( DefaultGitTreesPerPage int DefaultMaxBlobSize int64 }{ - EnableSwagger: true, + EnableSwagger: false, SwaggerURL: "", MaxResponseItems: 50, DefaultPagingNum: 30, @@ -707,6 +707,22 @@ var ( NPU_MINDSPORE_IMAGE_ID int NPU_TENSORFLOW_IMAGE_ID int }{} + + ModelSafetyTest = struct { + NPUBaseDataSetName string + NPUBaseDataSetUUID string + NPUCombatDataSetName string + NPUCombatDataSetUUID string + + GPUBaseDataSetName string + GPUBaseDataSetUUID string + GPUCombatDataSetName string + GPUCombatDataSetUUID string + }{} + + ModelApp = struct { + DesensitizationUrl string + }{} ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1527,6 +1543,20 @@ func NewContext() { getGrampusConfig() getModelartsCDConfig() getModelConvertConfig() + getModelSafetyConfig() + getModelAppConfig() +} + +func getModelSafetyConfig() { + sec := Cfg.Section("model_safety_test") + ModelSafetyTest.GPUBaseDataSetName = sec.Key("GPUBaseDataSetName").MustString("") + ModelSafetyTest.GPUBaseDataSetUUID = sec.Key("GPUBaseDataSetUUID").MustString("") + ModelSafetyTest.GPUCombatDataSetName = sec.Key("GPUCombatDataSetName").MustString("") + ModelSafetyTest.GPUCombatDataSetUUID = sec.Key("GPUCombatDataSetUUID").MustString("") + ModelSafetyTest.NPUBaseDataSetName = sec.Key("NPUBaseDataSetName").MustString("") + ModelSafetyTest.NPUBaseDataSetUUID = sec.Key("NPUBaseDataSetUUID").MustString("") + ModelSafetyTest.NPUCombatDataSetName = sec.Key("NPUCombatDataSetName").MustString("") + ModelSafetyTest.NPUCombatDataSetUUID = sec.Key("NPUCombatDataSetUUID").MustString("") } func getModelConvertConfig() { @@ -1548,6 +1578,12 @@ func getModelConvertConfig() { ModelConvert.NPU_TENSORFLOW_IMAGE_ID = sec.Key("NPU_TENSORFLOW_IMAGE_ID").MustInt(35) } +func getModelAppConfig() { + sec := Cfg.Section("model_app") + ModelApp.DesensitizationUrl = sec.Key("desensitization_url").MustString("") + +} + func getModelartsCDConfig() { sec := Cfg.Section("modelarts-cd") diff --git a/modules/timer/timer.go b/modules/timer/timer.go deleted file mode 100755 index a0220380c..000000000 --- a/modules/timer/timer.go +++ /dev/null @@ -1,30 +0,0 @@ -package timer - -import ( - "github.com/robfig/cron/v3" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/routers/repo" -) - -func LaunchCronJob() { - log.Trace("Run cron job") - - c := cron.New() - - spec := "*/10 * * * *" - c.AddFunc(spec, repo.HandleUnDecompressAttachment) - - specCheckBlockChainUserSuccess := "*/10 * * * *" - c.AddFunc(specCheckBlockChainUserSuccess, repo.HandleBlockChainUnSuccessUsers) - - specCheckRepoBlockChainSuccess := "*/1 * * * *" - c.AddFunc(specCheckRepoBlockChainSuccess, repo.HandleBlockChainUnSuccessRepos) - - specCheckUnTransformedPRs := "*/1 * * * *" - c.AddFunc(specCheckUnTransformedPRs, repo.HandleBlockChainMergedPulls) - - specCheckBlockChainCommitSuccess := "*/3 * * * *" - c.AddFunc(specCheckBlockChainCommitSuccess, repo.HandleBlockChainUnSuccessCommits) - c.Start() -} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b83fb6369..773a338c1 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1058,6 +1058,7 @@ modify_image=Modify Image image_exist=Image name has been used, please use a new one. image_committing=Image is submitting, please try again later. image_commit_fail=Failed to submit image, please try again later. +image_over_20g=Failed to submit image, the size of image can not be over 20GB. image_not_exist=Image does not exits. image_edit_fail=Failed to edit image, please try again later. image_delete_fail=Failed to delete image, please try again later. @@ -1373,7 +1374,9 @@ fork_guest_user = Sign in to fork this repository. copy_link = Copy copy_link_success = Link has been copied copy_link_error = Use ⌘C or Ctrl-C to copy +copy = Copy copied = Copied OK +copied_error = Copied error unwatch = Unwatch watch = Watch unstar = Unstar @@ -3207,11 +3210,11 @@ wrong_specification=You cannot use this specification, please choose another ite resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). -train_dataset_path_rule = The dataset location is stored in the environment variable data_url, the pre-trained model is storaged in the environment ckpt_url, and the output path is stored in the environment variable train_url. -infer_dataset_path_rule = The dataset location is stored in the environment variable data_url, and the output path is stored in the environment variable result_url. +train_dataset_path_rule = The dataset location is stored in the run parameter data_url, the pre-trained model is storaged in the run parameter ckpt_url, and the output path is stored in the run parameter train_url. +infer_dataset_path_rule = The dataset location is stored in the run parameter data_url, and the output path is stored in the run parameter result_url. view_sample = View sample -inference_output_path_rule = The inference output path is stored in the environment variable result_url. -model_file_path_rule=The model file location is stored in the environment variable ckpt_url +inference_output_path_rule = The inference output path is stored in the run parameter result_url. +model_file_path_rule=The model file location is stored in the run parameter ckpt_url model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] delete_task = Delete task @@ -3231,6 +3234,7 @@ point_hr = Point/hr DEBUG = DEBUG SNN4IMAGENET = BENCHMARK BRAINSCORE = BENCHMARK +MODELSAFETY = BENCHMARK TRAIN = TRAIN INFERENCE = INFERENCE BENCHMARK = BENCHMARK @@ -3244,8 +3248,8 @@ Stopped_success_update_status_fail=Succeed in stopping th job, but failed to upd load_code_failed=Fail to load code, please check if the right branch is selected. error.dataset_select = dataset select error：the count exceed the limit or has same name -new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online -new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online +new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online +new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online new_infer_gpu_tooltips = The dataset is stored in %s, the model file is stored in %s, please store the inference output in %s for subsequent downloads. [points] @@ -3257,3 +3261,20 @@ hours = Hours expected_time = , expected to be available for points_acquisition_instructions = Points Acquisition Instructions insufficient_points_balance = Insufficient points balance + +[modelsafety] +model_security_evaluation = Model Security Evaluation +base_data_set = Base Data Set +combat_data_set = Combat Data Set +evaluation_indicators = Evaluation Indicators +evaluation_result = Evaluation Result +no_data = No Data +untargetted = Untargetted +targetted = targetted +new_model_security_evaluation_tips = Model Security Evaluation just used for image classification + +[model_app] +get_file_fail= Can not get the image content, please try again later. +content_type_unsupported=Allowed image type is jpg, jpeg or png. +process_image_fail=Fail to process image, please try again later. + diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index c6f7e51fd..8ba4d252d 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1059,6 +1059,7 @@ modify_image=修改镜像 image_exist=镜像Tag已被使用，请修改镜像Tag。 image_committing=镜像正在提交中，请稍后再试。 image_commit_fail=提交镜像失败，请稍后再试。 +image_over_20g=提交镜像失败，镜像大小不能超过20GB。 image_not_exist=镜像不存在。 image_edit_fail=编辑镜像失败，请稍后再试。 image_delete_fail=删除镜像失败，请稍后再试。 @@ -1389,7 +1390,9 @@ fork_guest_user=登录并 派生 这个项目。 copy_link=复制链接 copy_link_success=已复制链接 copy_link_error=请按下 ⌘-C 或 Ctrl-C 复制 +copy=复制 copied=复制成功 +copied_error=复制失败 unwatch=取消关注 watch=关注 unstar=取消点赞 @@ -2022,7 +2025,7 @@ settings.githooks=管理 Git 钩子 settings.basic_settings=基本设置 settings.mirror_settings=镜像设置 settings.sync_mirror=同步 -settings.mirror_sync_in_progress=镜像同步正在进行中，请稍后后再试。 +settings.mirror_sync_in_progress=镜像同步正在进行中，请稍后再试。 settings.email_notifications.enable=启用邮件通知 settings.email_notifications.onmention=只在被提到时邮件通知 settings.email_notifications.disable=停用邮件通知 @@ -3225,11 +3228,11 @@ card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 -train_dataset_path_rule = 数据集位置存储在环境变量data_url中，预训练模型存放在环境变量ckpt_url中，训练输出路径存储在环境变量train_url中。 -infer_dataset_path_rule = 数据集位置存储在环境变量data_url中，推理输出路径存储在环境变量result_url中。 +train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。 +infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。 view_sample = 查看样例 -inference_output_path_rule = 推理输出路径存储在环境变量result_url中。 -model_file_path_rule = 模型文件位置存储在环境变量ckpt_url中。 +inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。 +model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。 model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] delete_task = 删除任务 @@ -3249,6 +3252,7 @@ point_hr = 积分/时 DEBUG = 调试任务 SNN4IMAGENET = 评测任务 BRAINSCORE = 评测任务 +MODELSAFETY = 评测任务 TRAIN = 训练任务 INFERENCE = 推理任务 BENCHMARK = 评测任务 @@ -3263,9 +3267,9 @@ load_code_failed=代码加载失败，请确认选择了正确的分支。 error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 -new_train_gpu_tooltips =训练脚本存储在%s中，数据集存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 -new_train_npu_tooltips =训练脚本存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 -new_infer_gpu_tooltips = 数据集存储在%s中，模型文件存储在%s中，推理输出请存储在%s中以供后续下载。 +new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 +new_train_npu_tooltips = 训练脚本存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 +new_infer_gpu_tooltips = 数据集存储在 %s 中，模型文件存储在 %s 中，推理输出请存储在 %s 中以供后续下载。 [points] points = 积分 @@ -3277,4 +3281,19 @@ expected_time = ，预计可用 points_acquisition_instructions = 积分获取说明 insufficient_points_balance = 积分余额不足 +[modelsafety] +model_security_evaluation = 模型安全评测 +base_data_set = 原始数据集 +combat_data_set = 对抗数据集 +evaluation_indicators = 评测指标 +evaluation_result = 评测结果 +no_data = 无数据 +untargetted = 非定向 +targetted = 定向 +new_model_security_evaluation_tips = 模型安全评测只适用于图像分类 + +[model_app] +get_file_fail= 获取上传文件失败，请稍后再试。 +content_type_unsupported=请上传jpg、jpeg或png图片。 +process_image_fail=图片处理失败，请稍后再试。 diff --git a/public/home/home.js b/public/home/home.js index c1849b3e3..853c3ef23 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -81,8 +81,11 @@ var isZh = true; if(lang != null && lang.nodeValue =="en-US" ){ isZh=false; } - document.onreadystatechange = function () { + if(document.readyState != "complete"){ + return; + } + console.log("Start to open WebSocket." + document.readyState); queryRecommendData(); var output = document.getElementById("newmessage"); @@ -102,10 +105,9 @@ document.onreadystatechange = function () { var html = ""; if (data != null){ if(messageQueue.length > maxSize){ - delete messageQueue[0]; - }else{ - messageQueue.push(data); + messageQueue.splice(0,1); } + messageQueue.push(data); var currentTime = new Date().getTime(); for(var i = 0; i < messageQueue.length; i++){ var record = messageQueue[i]; diff --git a/public/img/origin.png b/public/img/origin.png new file mode 100644 index 000000000..26eccf7ce Binary files /dev/null and b/public/img/origin.png differ diff --git a/public/img/tuomin.png b/public/img/tuomin.png new file mode 100644 index 000000000..116662b2f Binary files /dev/null and b/public/img/tuomin.png differ diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index fcb878627..d03c00ae6 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -52,7 +52,7 @@ func CloudBrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } @@ -98,8 +98,6 @@ func CloudBrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) @@ -179,6 +177,7 @@ func DownloadCloudBrains(ctx *context.Context) { log.Warn("Can not get cloud brain info", err) continue } + models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { for k, v := range allValues(row, record, ctx) { @@ -208,8 +207,11 @@ func allValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[str } func getCloudbrainCardType(rs *models.CloudbrainInfo) string { - _, cardType, _ := repo.GetCloudbrainCardNumAndType(rs.Cloudbrain) - return cardType + if rs.Cloudbrain.Spec != nil { + return rs.Cloudbrain.Spec.AccCardType + } else { + return "" + } } func getRepoPathName(rs *models.CloudbrainInfo) string { diff --git a/routers/admin/resources.go b/routers/admin/resources.go index 8a8c55f86..1d3870a14 100644 --- a/routers/admin/resources.go +++ b/routers/admin/resources.go @@ -221,6 +221,7 @@ func GetResourceSceneList(ctx *context.Context) { func AddResourceScene(ctx *context.Context, req models.ResourceSceneReq) { req.CreatorId = ctx.User.ID + req.ExclusiveOrg = strings.ReplaceAll(req.ExclusiveOrg, " ", "") err := resource.AddResourceScene(req) if err != nil { log.Error("AddResourceScene error. %v", err) @@ -238,6 +239,7 @@ func UpdateResourceScene(ctx *context.Context, req models.ResourceSceneReq) { var err error switch action { case "edit": + req.ExclusiveOrg = strings.ReplaceAll(req.ExclusiveOrg, " ", "") err = resource.UpdateResourceScene(req) case "delete": err = resource.DeleteResourceScene(id) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6dd1cb32f..8e1d725ed 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -590,6 +590,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/downloadAll", repo.DownloadCloudBrainBoard) m.Group("/cloudbrain", func() { m.Get("/overview", repo.GetAllCloudbrainsOverview) + m.Get("/overview_duration", repo.GetOverviewDuration) m.Get("/distribution", repo.GetAllCloudbrainsPeriodDistribution) m.Get("/trend", repo.GetAllCloudbrainsTrend) m.Get("/trend_detail_data", repo.GetAllCloudbrainsTrendDetail) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 439190121..ba46ab58c 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "sort" + "strconv" "strings" "time" @@ -67,49 +68,62 @@ func GetCloudbrainTask(ctx *context.APIContext) { log.Error("GetCloudbrainByID failed:", err) return } - jobResult, err := cloudbrain.GetJob(job.JobID) - if err != nil { - ctx.NotFound(err) - log.Error("GetJob failed:", err) - return - } - result, _ := models.ConvertToJobResultPayload(jobResult.Payload) - if err != nil { - ctx.NotFound(err) - log.Error("ConvertToJobResultPayload failed:", err) - return - } - oldStatus := job.Status - job.Status = result.JobStatus.State - taskRoles := result.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { - job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - job.ContainerID = taskRes.TaskStatuses[0].ContainerID - job.Status = taskRes.TaskStatuses[0].State - } - - if result.JobStatus.State != string(models.JobWaiting) { - models.ParseAndSetDurationFromCloudBrainOne(result, job) - if oldStatus != job.Status { - notification.NotifyChangeCloudbrainStatus(job, oldStatus) + if job.JobType == string(models.JobTypeModelSafety) { + routerRepo.GetAiSafetyTaskByJob(job) + job, err = models.GetCloudbrainByID(ID) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "ID": ID, + "JobName": job.JobName, + "JobStatus": job.Status, + "SubState": "", + "CreatedTime": job.CreatedUnix.Format("2006-01-02 15:04:05"), + "CompletedTime": job.UpdatedUnix.Format("2006-01-02 15:04:05"), + "JobDuration": job.TrainJobDuration, + }) + } else { + jobResult, err := cloudbrain.GetJob(job.JobID) + if err != nil { + ctx.NotFound(err) + log.Error("GetJob failed:", err) + return } - err = models.UpdateJob(job) + result, _ := models.ConvertToJobResultPayload(jobResult.Payload) if err != nil { - log.Error("UpdateJob failed:", err) + ctx.NotFound(err) + log.Error("ConvertToJobResultPayload failed:", err) + return + } + oldStatus := job.Status + job.Status = result.JobStatus.State + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + job.ContainerID = taskRes.TaskStatuses[0].ContainerID + job.Status = taskRes.TaskStatuses[0].State } - } - ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": result.Config.JobName, - "JobStatus": result.JobStatus.State, - "SubState": result.JobStatus.SubState, - "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"), - "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"), - "JobDuration": job.TrainJobDuration, - }) + if result.JobStatus.State != string(models.JobWaiting) { + models.ParseAndSetDurationFromCloudBrainOne(result, job) + if oldStatus != job.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "ID": ID, + "JobName": result.Config.JobName, + "JobStatus": result.JobStatus.State, + "SubState": result.JobStatus.SubState, + "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"), + "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"), + "JobDuration": job.TrainJobDuration, + }) + } } func GetCloudBrainInferenceJob(ctx *context.APIContext) { @@ -370,6 +384,96 @@ func CloudbrainForModelConvertGetLog(ctx *context.Context) { ctx.JSON(http.StatusOK, result) } +func ModelSafetyGetLog(ctx *context.APIContext) { + ID := ctx.Params(":id") + job, err := models.GetCloudbrainByID(ID) + if err != nil { + log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"]) + ctx.ServerError(err.Error(), err) + return + } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainTwo { + //TrainJobForModelConvertGetLog(ctx) + var baseLine = ctx.Query("base_line") + var order = ctx.Query("order") + var lines = ctx.Query("lines") + lines_int, err := strconv.Atoi(lines) + if err != nil { + log.Error("change lines(%d) string to int failed", lines_int) + } + + if order != modelarts.OrderDesc && order != modelarts.OrderAsc { + log.Error("order(%s) check failed", order) + ctx.JSON(http.StatusBadRequest, map[string]interface{}{ + "err_msg": "order check failed", + }) + return + } + resultLogFile, err := modelarts.GetTrainJobLogFileNames(job.JobID, strconv.FormatInt(job.VersionID, 10)) + if err != nil { + log.Error("GetTrainJobLogFileNames(%s) failed:%v", job.JobID, err.Error()) + } + result, err := modelarts.GetTrainJobLog(job.JobID, strconv.FormatInt(job.VersionID, 10), baseLine, resultLogFile.LogFileList[0], order, lines_int) + if err != nil { + log.Error("GetTrainJobLog(%s) failed:%v", job.JobID, err.Error()) + } + if err != nil { + log.Error("trainJobGetLog(%s) failed:%v", job.JobID, err.Error()) + // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": job.JobID, + "LogFileName": "", + "StartLine": "0", + "EndLine": "0", + "Content": "", + "Lines": 0, + "CanLogDownload": false, + }) + return + } + ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": job.JobID, + "LogFileName": resultLogFile.LogFileList[0], + "StartLine": result.StartLine, + "EndLine": result.EndLine, + "Content": result.Content, + "Lines": result.Lines, + "CanLogDownload": isCanDownloadLog(ctx, job), + }) + } + } + //result := "" + //ctx.JSON(http.StatusOK, result) +} + +func isCanDownloadLog(ctx *context.APIContext, job *models.Cloudbrain) bool { + if !ctx.IsSigned { + return false + } + return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID +} + +func ModelSafetyDownloadLogFile(ctx *context.Context) { + ID := ctx.Params(":id") + job, err := models.GetCloudbrainByID(ID) + if err != nil { + log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"]) + ctx.ServerError(err.Error(), err) + return + } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainOne { + CloudbrainDownloadLogFile(ctx) + } else if job.Type == models.TypeCloudBrainTwo { + ctx.SetParams("jobid", job.JobID) + ctx.Req.Form.Set("version_name", job.VersionName) + routerRepo.TrainJobDownloadLogFile(ctx) + } + } +} + func CloudbrainDownloadLogFile(ctx *context.Context) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) @@ -378,13 +482,18 @@ func CloudbrainDownloadLogFile(ctx *context.Context) { ctx.ServerError(err.Error(), err) return } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainTwo { + ModelSafetyDownloadLogFile(ctx) + return + } + } logDir := "/model" - if job.JobType == string(models.JobTypeInference) { + if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { logDir = cloudbrain.ResultPath } - prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir - files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") + files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix+job.JobName+logDir, "") if err != nil { log.Error("query cloudbrain model failed: %v", err) return @@ -397,17 +506,22 @@ func CloudbrainDownloadLogFile(ctx *context.Context) { } } if fileName != "" { + prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName) if err != nil { log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("Get minio get SignedUrl failed", err) return } + log.Info("fileName=" + fileName) http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) + } else { + log.Info("fileName is null.") + } } -func CloudbrainGetLog(ctx *context.Context) { +func CloudbrainGetLog(ctx *context.APIContext) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) if err != nil { @@ -415,12 +529,31 @@ func CloudbrainGetLog(ctx *context.Context) { ctx.ServerError(err.Error(), err) return } + if job.JobType == string(models.JobTypeModelSafety) { + if job.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(job.JobID) + existStr := "" + if err == nil && result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + existStr = taskRes.TaskStatuses[0].ExitDiagnostics + } + ctx.Data["existStr"] = existStr + log.Info("existStr=" + existStr) + } else { + ModelSafetyGetLog(ctx) + return + } + + } + lines := ctx.QueryInt("lines") baseLine := ctx.Query("base_line") order := ctx.Query("order") var result map[string]interface{} resultPath := "/model" - if job.JobType == string(models.JobTypeInference) { + if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { resultPath = "/result" } if baseLine == "" && order == "desc" { @@ -447,12 +580,19 @@ func CloudbrainGetLog(ctx *context.Context) { return } } + content := "" + if result["Content"] != nil { + content = result["Content"].(string) + } + if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 { + content = content + ctx.Data["existStr"].(string) + } re := map[string]interface{}{ "JobID": ID, "LogFileName": result["FileName"], "StartLine": result["StartLine"], "EndLine": result["EndLine"], - "Content": result["Content"], + "Content": content, "Lines": result["Lines"], "CanLogDownload": result["FileName"] != "", } @@ -485,7 +625,7 @@ func getAllLineFromFile(path string) int { } func getLastLogFromModelDir(jobName string, lines int, resultPath string) map[string]interface{} { - prefix := "/" + setting.CBCodePathPrefix + jobName + resultPath + prefix := setting.CBCodePathPrefix + jobName + resultPath files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") if err != nil { log.Error("query cloudbrain model failed: %v", err) @@ -546,7 +686,7 @@ func getLastLogFromModelDir(jobName string, lines int, resultPath string) map[st } func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath string) map[string]interface{} { - prefix := "/" + setting.CBCodePathPrefix + jobName + resultPath + prefix := setting.CBCodePathPrefix + jobName + resultPath files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") if err != nil { log.Error("query cloudbrain model failed: %v", err) @@ -577,8 +717,12 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s r := bufio.NewReader(reader) for i := 0; i < endLine; i++ { line, error := r.ReadString('\n') - log.Info("line=" + line) if error == io.EOF { + if i >= startLine { + fileEndLine = i + re = re + line + count++ + } log.Info("read file completed.") break } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 9d237d8e3..c665fe256 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -19,6 +19,7 @@ type DateCloudbrainNum struct { CloudOneJobTypeRes map[string]int `json:"cloudOneJobTypeRes"` CloudTwoJobTypeRes map[string]int `json:"cloudTwoJobTypeRes"` IntelligentNetJobTypeRes map[string]int `json:"intelligentNetJobTypeRes"` + CDCenterJobTypeRes map[string]int `json:"cDCenterJobTypeRes"` CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"` CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"` } @@ -53,20 +54,90 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { log.Error("Can not query todayCreatorCount.", err) return } + cloudbrainTypeCount, err := models.GetCloudbrainTypeCount() + log.Info("cloudbrainTypeCount:", cloudbrainTypeCount) + if err != nil { + log.Error("Can not query cloudbrainTypeCount.", err) + return + } + + cloudbrainTpyeDurationSum, err := models.GetCloudbrainTpyeDurationSum() + log.Info("cloudbrainTpyeDurationSum:", cloudbrainTpyeDurationSum) + if err != nil { + log.Error("Can not query cloudbrainTpyeDurationSum.", err) + return + } + + todayCloudbrainCount, err := models.GetTodayCloudbrainCount(beginTime, endTime) + log.Info("todayCloudbrainCount:", todayCloudbrainCount) + if err != nil { + log.Error("Can not query todayCloudbrainCount.", err) + return + } + + todayRunningCount, err := models.GetTodayRunningCount(beginTime, endTime) + log.Info("todayRunningCount:", todayRunningCount) + if err != nil { + log.Error("Can not query todayRunningCount.", err) + return + } + + todayWaitingCount, err := models.GetTodayWaitingCount(beginTime, endTime) + log.Info("todayWaittingCount:", todayWaitingCount) + if err != nil { + log.Error("Can not query todayWaitingCount.", err) + return + } + + todayCompletedCount := todayCloudbrainCount - todayRunningCount - todayWaitingCount + log.Info("todayCompletedCount:", todayCompletedCount) + creatorCount, err := models.GetCreatorCount() if err != nil { log.Error("Can not query creatorCount.", err) return } - todayStatusResult := make(map[string]int) - cloudBrainNum := make(map[int]int) - cloudBrainOneDuration := int64(0) - cloudBrainTwoDuration := int64(0) - intelligentNetDuration := int64(0) - todayNewJobCount := 0 + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "recordBeginTime": recordBeginTime, + "updateTime": now.Unix(), + "todayCreatorCount": todayCreatorCount, + "creatorCount": creatorCount, + "todayRunningCount": todayRunningCount, + "todayCompletedCount": todayCompletedCount, + "todayWaitingCount": todayWaitingCount, + "todayNewJobCount": todayCloudbrainCount, + "cloudbrainTypeCount": cloudbrainTypeCount, + }) +} +func GetOverviewDuration(ctx *context.Context) { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix + now := time.Now() + endTime := now page := 1 - pagesize := 1000 + pagesize := 10000 count := pagesize + worker_server_num := 1 + cardNum := 1 + durationAllSum := int64(0) + cardDuSum := int64(0) + + cloudBrainOneCardDuSum := int64(0) + cloudBrainTwoCardDuSum := int64(0) + c2NetCardDuSum := int64(0) + cDNetCardDuSum := int64(0) + + cloudBrainOneDuration := int64(0) + cloudBrainTwoDuration := int64(0) + c2NetDuration := int64(0) + cDCenterDuration := int64(0) + for count == pagesize && count != 0 { cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -81,72 +152,53 @@ func GetAllCloudbrainsOverview(ctx *context.Context) { ctx.ServerError("Get cloudbrains failed:", err) return } + models.LoadSpecs4CloudbrainInfo(cloudbrains) for _, cloudbrain := range cloudbrains { - if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { - todayNewJobCount += 1 - if _, ok := todayStatusResult[cloudbrain.Status]; !ok { - todayStatusResult[cloudbrain.Status] = 1 - } else { - todayStatusResult[cloudbrain.Status] += 1 - } + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber + } else { + worker_server_num = 1 } - - if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { - cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 + if cloudbrain.Cloudbrain.Spec == nil { + cardNum = 1 } else { - cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 + cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } - + duration := cloudbrain.Duration + durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration - } - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration + cloudBrainOneDuration += duration + cloudBrainOneCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration + cloudBrainTwoCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration + c2NetCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDNetCardDuSum += durationSum } - if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - intelligentNetDuration = intelligentNetDuration + cloudbrain.Cloudbrain.Duration - } - - } - count = len(cloudbrains) - page += 1 - } - statusNameList := []string{string(models.ModelArtsTrainJobCompleted), string(models.JobFailed), string(models.ModelArtsTrainJobInit), - string(models.JobRunning), string(models.ModelArtsStartFailed), string(models.JobStopped), string(models.JobSucceeded), - string(models.JobWaiting), string(models.ModelArtsTrainJobKilled)} - for _, v := range statusNameList { - if _, ok := todayStatusResult[v]; !ok { - todayStatusResult[v] = 0 - } - } - - cloudBrainTypeList := []int{0, 1, 2} - for _, v := range cloudBrainTypeList { - if _, ok := cloudBrainNum[v]; !ok { - cloudBrainNum[v] = 0 + durationAllSum += duration + cardDuSum += durationSum + count = len(cloudbrains) + page += 1 } } - - todayRunningCount := todayStatusResult[string(models.JobRunning)] - todayCompletedCount := todayStatusResult[string(models.ModelArtsTrainJobCompleted)] + todayStatusResult[string(models.JobFailed)] + - todayStatusResult[string(models.ModelArtsStartFailed)] + todayStatusResult[string(models.JobStopped)] + todayStatusResult[string(models.JobSucceeded)] + todayStatusResult[string(models.ModelArtsTrainJobKilled)] - todayWaitingCount := todayStatusResult[string(models.ModelArtsTrainJobInit)] + todayStatusResult[string(models.JobWaiting)] - ctx.JSON(http.StatusOK, map[string]interface{}{ - "recordBeginTime": recordBeginTime, - "updateTime": now.Unix(), - "cloudBrainNum": cloudBrainNum, - "cloudBrainOneDuration": cloudBrainOneDuration, - "cloudBrainTwoDuration": cloudBrainTwoDuration, - "intelligentNetDuration": intelligentNetDuration, - "todayCreatorCount": todayCreatorCount, - "creatorCount": creatorCount, - "todayRunningCount": todayRunningCount, - "todayCompletedCount": todayCompletedCount, - "todayWaitingCount": todayWaitingCount, - "todayNewJobCount": todayNewJobCount, + "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, + "cloudBrainTwoCardDuSum": cloudBrainTwoCardDuSum, + "c2NetCardDuSum": c2NetCardDuSum, + "cDNetCardDuSum": cDNetCardDuSum, + "cardDuSum": cardDuSum, + + "cloudBrainOneDuration": cloudBrainOneDuration, + "cloudBrainTwoDuration": cloudBrainTwoDuration, + "c2NetDuration": c2NetDuration, + "cDCenterDuration": cDCenterDuration, + "durationSum": durationAllSum, }) } @@ -500,6 +552,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { cloudOneJobTypeRes := make(map[string]int) cloudTwoJobTypeRes := make(map[string]int) intelligentNetJobTypeRes := make(map[string]int) + cDCenterJobTypeRes := make(map[string]int) cloudBrainPeriodNum := make(map[int]int) cloudBrainComputeResource := make(map[string]int) beginTimeTemp := beginTime.Unix() @@ -508,9 +561,9 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } page := 1 - pagesize := 1000 + pagesize := 10000 count := pagesize - //Each time a maximum of 1000 pieces of data are detected to the memory, batch processing + //Each time a maximum of 10000 pieces of data are detected to the memory, batch processing for count == pagesize && count != 0 { cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -548,6 +601,13 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { intelligentNetJobTypeRes[cloudbrain.JobType] += 1 } } + if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + if _, ok := cDCenterJobTypeRes[cloudbrain.JobType]; !ok { + cDCenterJobTypeRes[cloudbrain.JobType] = 1 + } else { + cDCenterJobTypeRes[cloudbrain.JobType] += 1 + } + } if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 @@ -566,7 +626,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), - string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} + string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -577,8 +637,11 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { if _, ok := intelligentNetJobTypeRes[v]; !ok { intelligentNetJobTypeRes[v] = 0 } + if _, ok := cDCenterJobTypeRes[v]; !ok { + cDCenterJobTypeRes[v] = 0 + } } - cloudBrainTypeList := []int{0, 1, 2} + cloudBrainTypeList := []int{0, 1, 2, 3} for _, v := range cloudBrainTypeList { if _, ok := cloudBrainPeriodNum[v]; !ok { cloudBrainPeriodNum[v] = 0 @@ -592,77 +655,30 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } } - cloudOneJobTypeRes["EVALUATION"] = cloudOneJobTypeRes[string(models.JobTypeBenchmark)] + cloudOneJobTypeRes[string(models.JobTypeSnn4imagenet)] + cloudOneJobTypeRes[string(models.JobTypeBrainScore)] - cloudTwoJobTypeRes["EVALUATION"] = cloudTwoJobTypeRes[string(models.JobTypeBenchmark)] + cloudTwoJobTypeRes[string(models.JobTypeSnn4imagenet)] + cloudTwoJobTypeRes[string(models.JobTypeBrainScore)] - intelligentNetJobTypeRes["EVALUATION"] = intelligentNetJobTypeRes[string(models.JobTypeBenchmark)] + intelligentNetJobTypeRes[string(models.JobTypeSnn4imagenet)] + intelligentNetJobTypeRes[string(models.JobTypeBrainScore)] + cloudOneJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[0] - cloudOneJobTypeRes[string(models.JobTypeTrain)] - cloudOneJobTypeRes[string(models.JobTypeInference)] - cloudOneJobTypeRes[string(models.JobTypeDebug)] + cloudTwoJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[1] - cloudTwoJobTypeRes[string(models.JobTypeTrain)] - cloudTwoJobTypeRes[string(models.JobTypeInference)] - cloudTwoJobTypeRes[string(models.JobTypeDebug)] + intelligentNetJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[2] - intelligentNetJobTypeRes[string(models.JobTypeTrain)] - intelligentNetJobTypeRes[string(models.JobTypeInference)] - intelligentNetJobTypeRes[string(models.JobTypeDebug)] + cDCenterJobTypeRes["EVALUATION"] = cloudBrainPeriodNum[3] - cDCenterJobTypeRes[string(models.JobTypeTrain)] - cDCenterJobTypeRes[string(models.JobTypeInference)] - cDCenterJobTypeRes[string(models.JobTypeDebug)] ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudOneJobTypeRes": cloudOneJobTypeRes, "cloudTwoJobTypeRes": cloudTwoJobTypeRes, "intelligentNetJobTypeRes": intelligentNetJobTypeRes, + "cDCenterJobTypeRes": cDCenterJobTypeRes, "cloudBrainPeriodNum": cloudBrainPeriodNum, "cloudBrainComputeResource": cloudBrainComputeResource, }) } func GetCloudbrainsStatusAnalysis(ctx *context.Context) { - recordCloudbrain, err := models.GetRecordBeginTime() + cloudbrainStatusCount, err := models.GetCloudbrainStatusCount() + log.Info("cloudbrainStatusCount:", cloudbrainStatusCount) if err != nil { - log.Error("Can not get recordCloudbrain", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + log.Error("Can not query cloudbrainStatusCount.", err) return } - recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix - endTime := time.Now() - cloudBrainStatusResult := make(map[string]int) - cloudBrainStatusAnalysis := make(map[string]int) - totalCount := 0 - - page := 1 - pagesize := 1000 - count := pagesize - for count == pagesize && count != 0 { - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: pagesize, - }, - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - for _, cloudbrain := range cloudbrains { - if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { - cloudBrainStatusResult[cloudbrain.Status] = 1 - } else { - cloudBrainStatusResult[cloudbrain.Status] += 1 - } - } - count = len(cloudbrains) - totalCount = totalCount + count - page += 1 - } - - var jobStatuses []string - jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), - string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), - string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) - jobStatusesCount := 0 - for _, v := range jobStatuses { - if _, ok := cloudBrainStatusResult[v]; !ok { - cloudBrainStatusAnalysis[v] = 0 - } else { - cloudBrainStatusAnalysis[v] = cloudBrainStatusResult[v] - } - jobStatusesCount = jobStatusesCount + cloudBrainStatusResult[v] - } - cloudBrainStatusAnalysis["OTHER"] = totalCount - jobStatusesCount ctx.JSON(http.StatusOK, map[string]interface{}{ - "cloudBrainStatusResult": cloudBrainStatusAnalysis, + "cloudbrainStatusCount": cloudbrainStatusCount, }) } @@ -693,7 +709,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } @@ -738,7 +754,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { tasks := []models.TaskDetail{} for i, task := range ciTasks { ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - var taskDetail models.TaskDetail taskDetail.ID = ciTasks[i].Cloudbrain.ID taskDetail.JobID = ciTasks[i].Cloudbrain.JobID @@ -758,11 +773,12 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias } - taskDetail.CardNum, taskDetail.CardType, _ = repo.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) + if ciTasks[i].Cloudbrain.WorkServerNumber >= 1 { + taskDetail.WorkServerNum = int64(ciTasks[i].Cloudbrain.WorkServerNumber) + } else { + taskDetail.WorkServerNum = 1 + } taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) - taskDetail.AiCenter = repo.GetCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) - taskDetail.FlavorName, _ = repo.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) - taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) if ciTasks[i].Cloudbrain.DeletedAt != nilTime || ciTasks[i].Repo == nil { @@ -787,17 +803,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { }) } -func getCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { - if task.Type == models.TypeCloudBrainOne { - return ctx.Tr("repo.cloudbrain1") - } else if task.Type == models.TypeCloudBrainTwo { - return ctx.Tr("repo.cloudbrain2") - } else if task.Type == models.TypeC2Net { - return task.AiCenter - } - return "" -} - func GetCloudbrainsCreateHoursData(ctx *context.Context) { recordCloudbrain, err := models.GetRecordBeginTime() if err != nil { @@ -958,11 +963,12 @@ func GetRunningTop(ctx *context.Context) { }) } -func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*models.CloudbrainInfo) (map[string]int, map[string]int, map[string]int, map[int]int, map[string]int) { +func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*models.CloudbrainInfo) (map[string]int, map[string]int, map[string]int, map[string]int, map[int]int, map[string]int) { cloudOneJobTypeRes := make(map[string]int) cloudTwoJobTypeRes := make(map[string]int) intelligentNetJobTypeRes := make(map[string]int) + cDCenterJobTypeRes := make(map[string]int) cloudBrainPeriodNum := make(map[int]int) cloudBrainComputeResource := make(map[string]int) for _, cloudbrain := range cloudbrains { @@ -1005,7 +1011,7 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m } jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} - cloudBrainTypeList := []int{0, 1, 2} + cloudBrainTypeList := []int{0, 1, 2, 3} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -1016,14 +1022,17 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m if _, ok := intelligentNetJobTypeRes[v]; !ok { intelligentNetJobTypeRes[v] = 0 } + if _, ok := cDCenterJobTypeRes[v]; !ok { + cDCenterJobTypeRes[v] = 0 + } } for _, v := range cloudBrainTypeList { if _, ok := cloudBrainPeriodNum[v]; !ok { cloudBrainPeriodNum[v] = 0 } } - cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] - return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource + cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] + cloudBrainPeriodNum[3] + return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource } func getDayCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { @@ -1040,12 +1049,13 @@ func getDayCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudbra } dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { - cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006/01/02"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CDCenterJobTypeRes: cDCenterJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) @@ -1075,12 +1085,13 @@ func getMonthCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudb return nil, err } for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { - cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cDCenterJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) monthCloudbrainNum = append(monthCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006/01"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CDCenterJobTypeRes: cDCenterJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) @@ -1113,7 +1124,7 @@ func getDayCloudbrainInfo(beginTime time.Time, endTime time.Time) ([]DateCloudbr dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { - _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) + _, _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) dayCloudbrainInfo = append(dayCloudbrainInfo, DateCloudbrainInfo{ Date: endTimeTemp.Format("2006/01/02"), CloudBrainPeriodNum: cloudBrainPeriodNum, @@ -1144,7 +1155,7 @@ func getMonthCloudbrainInfo(beginTime time.Time, endTime time.Time) ([]DateCloud dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) || (endTimeTemp.Before(beginTime) && beginTime.Before(endTime)) { - _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) + _, _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) dayCloudbrainInfo = append(dayCloudbrainInfo, DateCloudbrainInfo{ Date: endTimeTemp.Format("2006/01"), CloudBrainPeriodNum: cloudBrainPeriodNum, @@ -1205,6 +1216,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { log.Warn("Can not get cloud brain info", err) continue } + models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { for k, v := range allCloudbrainValues(row, record, ctx) { @@ -1235,10 +1247,9 @@ func allCloudbrainHeader(ctx *context.Context) map[string]string { "H1": ctx.Tr("cloudbrain.card_duration"), "I1": ctx.Tr("repo.modelarts.train_job.start_time"), "J1": ctx.Tr("repo.modelarts.train_job.end_time"), "K1": ctx.Tr("repo.modelarts.computing_resources"), "L1": ctx.Tr("cloudbrain.card_type"), - "M1": ctx.Tr("repo.grampus.train_job.ai_center"), "N1": ctx.Tr("cloudbrain.resource_specification"), - "O1": ctx.Tr("repo.cloudbrain_creator"), "P1": ctx.Tr("repo.repo_name"), "Q1": ctx.Tr("repo.cloudbrain_task_name"), - "R1": ctx.Tr("repo.modelarts.deletetime")} - + "M1": ctx.Tr("repo.modelarts.train_job.amount_of_compute_node"), "N1": ctx.Tr("repo.grampus.train_job.ai_center"), + "O1": ctx.Tr("cloudbrain.resource_specification"), "P1": ctx.Tr("repo.cloudbrain_creator"), "Q1": ctx.Tr("repo.repo_name"), + "R1": ctx.Tr("repo.cloudbrain_task_name"), "S1": ctx.Tr("repo.modelarts.deletetime")} } func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): getCloudbrainType(rs, ctx), getCellName("C", row): rs.Status, getCellName("D", row): rs.JobType, @@ -1246,9 +1257,16 @@ func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Contex getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): repo.GetCloudbrainCardDuration(rs.Cloudbrain), getCellName("I", row): getBrainStartTime(rs), getCellName("J", row): getBrainEndTime(rs), getCellName("K", row): rs.ComputeResource, getCellName("L", row): getCloudbrainCardType(rs), - getCellName("M", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), getCellName("N", row): getCloudbrainFlavorName(rs), - getCellName("O", row): rs.Name, getCellName("P", row): getBrainRepo(rs), - getCellName("Q", row): rs.JobName, getCellName("R", row): getBrainDeleteTime(rs), + getCellName("M", row): getWorkServerNum(rs), getCellName("N", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), + getCellName("O", row): getCloudbrainFlavorName(rs), getCellName("P", row): rs.Name, + getCellName("Q", row): getBrainRepo(rs), getCellName("R", row): rs.JobName, getCellName("S", row): getBrainDeleteTime(rs), + } +} +func getWorkServerNum(rs *models.CloudbrainInfo) string { + if rs.Cloudbrain.WorkServerNumber >= 1 { + return fmt.Sprint(rs.Cloudbrain.WorkServerNumber) + } else { + return "1" } } func getBrainRepo(rs *models.CloudbrainInfo) string { @@ -1287,11 +1305,14 @@ func getCloudbrainType(rs *models.CloudbrainInfo, ctx *context.Context) string { } } func getCloudbrainCardType(rs *models.CloudbrainInfo) string { - _, cardType, _ := repo.GetCloudbrainCardNumAndType(rs.Cloudbrain) - return cardType + if rs.Cloudbrain.Spec != nil { + return rs.Cloudbrain.Spec.AccCardType + } else { + return "" + } } func getCloudbrainFlavorName(rs *models.CloudbrainInfo) string { - flavorName, _ := repo.GetCloudbrainFlavorName(rs.Cloudbrain) + flavorName := repo.GetCloudbrainFlavorName(rs.Cloudbrain) return flavorName } diff --git a/routers/api/v1/repo/fork.go b/routers/api/v1/repo/fork.go index a753f192d..9416035e7 100644 --- a/routers/api/v1/repo/fork.go +++ b/routers/api/v1/repo/fork.go @@ -7,6 +7,7 @@ package repo import ( "fmt" + "html/template" "net/http" "code.gitea.io/gitea/models" @@ -118,7 +119,7 @@ func CreateFork(ctx *context.APIContext, form api.CreateForkOption) { forker = org } - fork, err := repo_service.ForkRepository(ctx.User, forker, repo, repo.Name, repo.Description, repo.Alias) + fork, err := repo_service.ForkRepository(ctx.User, forker, repo, repo.Name, template.HTMLEscapeString(repo.Description), repo.Alias) if err != nil { ctx.Error(http.StatusInternalServerError, "ForkRepository", err) return diff --git a/routers/modelapp/desensitization.go b/routers/modelapp/desensitization.go new file mode 100644 index 000000000..9de33526e --- /dev/null +++ b/routers/modelapp/desensitization.go @@ -0,0 +1,78 @@ +package modelapp + +import ( + "bytes" + "code.gitea.io/gitea/models" + "crypto/tls" + "image" + "image/png" + "net/http" + "strconv" + + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/base" + + "code.gitea.io/gitea/modules/context" + "github.com/go-resty/resty/v2" +) + +var restyClient *resty.Client +var tplExploreUpload base.TplName = "model/tuomin/upload" +var uploadUrl = "/extension/tuomin/upload" +var allowedContentType = []string{"image/jpeg", "image/jpg", "image/png"} + +func ProcessImageUI(ctx *context.Context) { + ctx.HTML(200, tplExploreUpload) +} + +func ProcessImage(ctx *context.Context) { + + file, header, err := ctx.GetFile("file") + if err != nil { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.get_file_fail"))) + return + } + defer file.Close() + + contentType := header.Header.Get("Content-Type") + + if !isInAllowedContentType(contentType) { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.content_type_unsupported"))) + return + } + + client := getRestyClient() + res, err := client.R().SetMultipartField( + "file", header.Filename, contentType, file).Post(setting.ModelApp.DesensitizationUrl + "?mode=" + strconv.Itoa(ctx.QueryInt("mode"))) + if err != nil { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.process_image_fail"))) + return + } + image, _, err := image.Decode(bytes.NewReader(res.Body())) + if err != nil { + ctx.JSON(http.StatusBadRequest,models.BaseErrorMessage(ctx.Tr("model_app.process_image_fail"))) + return + } + + png.Encode(ctx.Resp, image) + return + +} + +func isInAllowedContentType(contentType string) bool { + for _, allowType := range allowedContentType { + if allowType == contentType { + return true + } + } + return false +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + return restyClient +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 957028fc6..5b358b83b 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "regexp" "strings" "code.gitea.io/gitea/models" @@ -85,6 +86,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio accuracy["Precision"] = "" accuracyJson, _ := json.Marshal(accuracy) log.Info("accuracyJson=" + string(accuracyJson)) + aiTask.ContainerIp = "" aiTaskJson, _ := json.Marshal(aiTask) model := &models.AiModelManage{ @@ -635,6 +637,7 @@ func ShowSingleModel(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range models { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -644,6 +647,13 @@ func ShowSingleModel(ctx *context.Context) { ctx.JSON(http.StatusOK, models) } +func removeIpInfo(model *models.AiModelManage) { + reg, _ := regexp.Compile(`[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}`) + taskInfo := model.TrainTaskInfo + taskInfo = reg.ReplaceAllString(taskInfo, "") + model.TrainTaskInfo = taskInfo +} + func queryUserName(intSlice []int64) map[int64]*models.User { keys := make(map[int64]string) uniqueElements := []int64{} @@ -677,6 +687,7 @@ func ShowOneVersionOtherModel(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range aimodels { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -793,6 +804,7 @@ func ShowModelPageInfo(ctx *context.Context) { userNameMap := queryUserName(userIds) for _, model := range modelResult { + removeIpInfo(model) value := userNameMap[model.UserId] if value != nil { model.UserName = value.Name @@ -866,6 +878,7 @@ func QueryModelListForPredict(ctx *context.Context) { nameMap := make(map[string][]*models.AiModelManage) for _, model := range modelResult { + removeIpInfo(model) if _, value := nameMap[model.Name]; !value { models := make([]*models.AiModelManage, 0) models = append(models, model) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go new file mode 100644 index 000000000..5102a6722 --- /dev/null +++ b/routers/repo/aisafety.go @@ -0,0 +1,1049 @@ +package repo + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/aisafety" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward/point/account" + uuid "github.com/satori/go.uuid" +) + +const ( + tplModelSafetyTestCreateGrampusGpu = "repo/modelsafety/newgrampusgpu" + tplModelSafetyTestCreateGrampusNpu = "repo/modelsafety/newgrampusnpu" + tplModelSafetyTestCreateGpu = "repo/modelsafety/newgpu" + tplModelSafetyTestCreateNpu = "repo/modelsafety/newnpu" + tplModelSafetyTestShow = "repo/modelsafety/show" +) + +func CloudBrainAiSafetyCreateTest(ctx *context.Context) { + log.Info("start to create CloudBrainAiSafetyCreate") + uuid := uuid.NewV4() + id := uuid.String() + seriaNoParas := ctx.Query("serialNo") + fileName := ctx.Query("fileName") + + //if jobType == string(models.JobTypeBenchmark) { + req := aisafety.TaskReq{ + UnionId: id, + EvalName: "test1", + EvalContent: "test1", + TLPath: "test1", + Indicators: []string{"ACC", "ASS"}, + CDName: "CIFAR10_1000_FGSM", + BDName: "CIFAR10_1000基础数据集", + } + aisafety.GetAlgorithmList() + if seriaNoParas != "" { + aisafety.GetTaskStatus(seriaNoParas) + } else { + jsonStr, err := getJsonContent("http://192.168.207.34:8065/Test_zap1234/openi_aisafety/raw/branch/master/result/" + fileName) + serialNo, err := aisafety.CreateSafetyTask(req, jsonStr) + if err == nil { + log.Info("serialNo=" + serialNo) + time.Sleep(time.Duration(2) * time.Second) + aisafety.GetTaskStatus(serialNo) + } else { + log.Info("CreateSafetyTask error," + err.Error()) + } + } +} + +func GetAiSafetyTaskByJob(job *models.Cloudbrain) { + if job == nil { + log.Error("GetCloudbrainByJobID failed") + return + } + syncAiSafetyTaskStatus(job) +} + +func GetAiSafetyTaskTmpl(ctx *context.Context) { + ctx.Data["id"] = ctx.Params(":id") + ctx.Data["PageIsCloudBrain"] = true + ctx.HTML(200, tplModelSafetyTestShow) +} + +func GetAiSafetyTask(ctx *context.Context) { + var ID = ctx.Params(":id") + job, err := models.GetCloudbrainByIDWithDeleted(ID) + if err != nil { + log.Error("GetCloudbrainByJobID failed:" + err.Error()) + return + } + syncAiSafetyTaskStatus(job) + job, err = models.GetCloudbrainByIDWithDeleted(ID) + job.BenchmarkType = "安全评测" + job.BenchmarkTypeName = "Image Classification" + job.CanModify = cloudbrain.CanModifyJob(ctx, job) + job.CanDel = cloudbrain.CanDeleteJob(ctx, job) + if job.Parameters == "{\"parameter\":[]}" { + job.Parameters = "" + } + s, err := resource.GetCloudbrainSpec(job.ID) + if err == nil { + job.Spec = s + } + user, err := models.GetUserByID(job.UserID) + if err == nil { + tmpUser := &models.User{ + Name: user.Name, + } + job.User = tmpUser + } + + ctx.JSON(200, job) +} + +func StopAiSafetyTask(ctx *context.Context) { + log.Info("start to stop the task.") + var ID = ctx.Params(":id") + task, err := models.GetCloudbrainByIDWithDeleted(ID) + result := make(map[string]interface{}) + result["result_code"] = "-1" + if err != nil { + log.Info("query task error.err=" + err.Error()) + log.Error("GetCloudbrainByJobID failed:" + err.Error()) + result["msg"] = "No such task." + ctx.JSON(200, result) + return + } + if isTaskNotFinished(task.Status) { + if task.Type == models.TypeCloudBrainTwo { + log.Info("start to stop model arts task.") + _, err := modelarts.StopTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Info("stop failed.err=" + err.Error()) + } + task.Status = string(models.JobStopped) + if task.EndTime == 0 { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_success_update_status_fail" + ctx.JSON(200, result) + return + } + //queryTaskStatusFromCloudbrainTwo(job) + } else if task.Type == models.TypeCloudBrainOne { + if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) { + log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Already_stopped" + ctx.JSON(200, result) + return + } + err := cloudbrain.StopJob(task.JobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_failed" + ctx.JSON(200, result) + return + } + task.Status = string(models.JobStopped) + if task.EndTime == 0 { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_success_update_status_fail" + ctx.JSON(200, result) + return + } + } + } else { + if task.Status == string(models.ModelSafetyTesting) { + //修改为Failed + task.Status = string(models.JobStopped) + if task.EndTime == 0 { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) + result["msg"] = "cloudbrain.Stopped_success_update_status_fail" + ctx.JSON(200, result) + return + } + } else { + log.Info("The job is finished. status=" + task.Status) + } + } + result["result_code"] = "0" + result["msg"] = "succeed" + ctx.JSON(200, result) + +} + +func DelAiSafetyTask(ctx *context.Context) { + var ID = ctx.Params(":id") + task, err := models.GetCloudbrainByIDWithDeleted(ID) + + if err != nil { + log.Error("GetCloudbrainByJobID failed:" + err.Error()) + ctx.ServerError("No such task.", err) + return + } + if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) { + log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) + ctx.ServerError("the job("+task.JobName+") has not been stopped", nil) + return + } + if task.Type == models.TypeCloudBrainOne { + DeleteCloudbrainJobStorage(task.JobName, models.TypeCloudBrainOne) + } + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError(err.Error(), err) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") +} + +func syncAiSafetyTaskStatus(job *models.Cloudbrain) { + log.Info("start to query safety task status.") + if isTaskNotFinished(job.Status) { + if job.Type == models.TypeCloudBrainTwo { + queryTaskStatusFromCloudbrainTwo(job) + } else if job.Type == models.TypeCloudBrainOne { + queryTaskStatusFromCloudbrain(job) + } + } else { + if job.Status == string(models.ModelSafetyTesting) { + queryTaskStatusFromModelSafetyTestServer(job) + } else { + log.Info("The job is finished. status=" + job.Status) + } + } +} + +func TimerHandleModelSafetyTestTask() { + log.Info("start to TimerHandleModelSafetyTestTask") + tasks, err := models.GetModelSafetyTestTask() + if err == nil { + if tasks != nil && len(tasks) > 0 { + for _, job := range tasks { + syncAiSafetyTaskStatus(job) + } + } else { + log.Info("query running model safety test task 0.") + } + } else { + log.Info("query running model safety test task err." + err.Error()) + } +} + +func queryTaskStatusFromCloudbrainTwo(job *models.Cloudbrain) { + log.Info("The task not finished,name=" + job.DisplayJobName) + result, err := modelarts.GetTrainJob(job.JobID, strconv.FormatInt(job.VersionID, 10)) + if err != nil { + log.Info("query train job error." + err.Error()) + return + } + + job.Status = modelarts.TransTrainJobStatus(result.IntStatus) + job.Duration = result.Duration / 1000 + job.TrainJobDuration = result.TrainJobDuration + + if job.StartTime == 0 && result.StartTime > 0 { + job.StartTime = timeutil.TimeStamp(result.StartTime / 1000) + } + job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) + if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { + job.EndTime = job.StartTime.Add(job.Duration) + } + job.CorrectCreateUnix() + + if job.Status != string(models.ModelArtsTrainJobCompleted) { + log.Info("CloudbrainTwo task status=" + job.Status) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } else { + log.Info("start to deal ModelSafetyTesting, task status=" + job.Status) + job.Status = string(models.ModelSafetyTesting) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + //send msg to beihang + sendNPUInferenceResultToTest(job) + } + +} + +func queryTaskStatusFromCloudbrain(job *models.Cloudbrain) { + + log.Info("The task not finished,name=" + job.DisplayJobName) + jobResult, err := cloudbrain.GetJob(job.JobID) + + result, err := models.ConvertToJobResultPayload(jobResult.Payload) + if err != nil { + log.Error("ConvertToJobResultPayload failed:", err) + return + } + job.Status = result.JobStatus.State + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + job.Status = taskRes.TaskStatuses[0].State + } + models.ParseAndSetDurationFromCloudBrainOne(result, job) + //updateCloudBrainOneJobTime(job) + log.Info("cloud brain one job status=" + job.Status) + if result.JobStatus.State != string(models.JobSucceeded) { + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } else { + // + job.Status = string(models.ModelSafetyTesting) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + //send msg to beihang + sendGPUInferenceResultToTest(job) + } +} + +func queryTaskStatusFromModelSafetyTestServer(job *models.Cloudbrain) { + result, err := aisafety.GetTaskStatus(job.PreVersionName) + if err == nil { + if result.Code == "0" { + if result.Data.Status == 1 { + log.Info("The task is running....") + } else { + if result.Data.Code == 0 { + job.ResultJson = result.Data.StandardJson + job.Status = string(models.JobSucceeded) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } else { + job.ResultJson = result.Data.Msg + job.Status = string(models.JobFailed) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + } + } else { + log.Info("The task is failed.") + job.Status = string(models.JobFailed) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + } else { + log.Info("The task not found.....") + } +} + +func getAisafetyTaskReq(job *models.Cloudbrain) aisafety.TaskReq { + datasetname := job.DatasetName + datasetnames := strings.Split(datasetname, ";") + indicator := job.LabelName + EvalContent := "test1" + if job.Description != "" { + EvalContent = job.Description + } + req := aisafety.TaskReq{ + UnionId: job.JobID, + EvalName: job.DisplayJobName, + EvalContent: EvalContent, + TLPath: "test1", + Indicators: strings.Split(indicator, ";"), + CDName: strings.Split(datasetnames[1], ".")[0], + BDName: strings.Split(datasetnames[0], ".")[0] + "基础数据集", + } + log.Info("CDName=" + req.CDName) + log.Info("BDName=" + req.BDName) + return req +} + +func sendGPUInferenceResultToTest(job *models.Cloudbrain) { + log.Info("send sendGPUInferenceResultToTest") + req := getAisafetyTaskReq(job) + resultDir := "/result" + prefix := setting.CBCodePathPrefix + job.JobName + resultDir + files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") + if err != nil { + log.Error("query cloudbrain one model failed: %v", err) + return + } + jsonContent := "" + for _, file := range files { + if strings.HasSuffix(file.FileName, "result.json") { + path := storage.GetMinioPath(job.JobName+resultDir+"/", file.FileName) + log.Info("path=" + path) + reader, err := os.Open(path) + defer reader.Close() + if err == nil { + r := bufio.NewReader(reader) + for { + line, error := r.ReadString('\n') + jsonContent += line + if error == io.EOF { + log.Info("read file completed.") + break + } + if error != nil { + log.Info("read file error." + error.Error()) + break + } + } + } + break + } + } + if jsonContent != "" { + sendHttpReqToBeihang(job, jsonContent, req) + } else { + updateJobFailed(job, "推理生成的Json数据为空，无法进行评测。") + } +} + +func sendNPUInferenceResultToTest(job *models.Cloudbrain) { + log.Info("start to sendNPUInferenceResultToTest") + req := getAisafetyTaskReq(job) + jsonContent := "" + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + resultPath := modelarts.JobPath + job.JobName + modelarts.ResultPath + VersionOutputPath + "/result.json" + resultPath = resultPath[1:] + log.Info("bucket=" + setting.Bucket + " resultPath=" + resultPath) + body, err := storage.ObsDownloadAFile(setting.Bucket, resultPath) + if err != nil { + log.Info("ObsDownloadAFile error." + err.Error() + " resultPath=" + resultPath) + } else { + defer body.Close() + var data []byte + p := make([]byte, 4096) + var readErr error + var readCount int + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + data = append(data, p[:readCount]...) + } + if readErr != nil || readCount == 0 { + break + } + } + jsonContent = string(data) + } + + if jsonContent != "" { + sendHttpReqToBeihang(job, jsonContent, req) + } else { + updateJobFailed(job, "推理生成的Json数据为空，无法进行评测。") + } +} +func updateJobFailed(job *models.Cloudbrain, msg string) { + log.Info("The json is null. so set it failed.") + //update task failed. + job.Status = string(models.ModelArtsTrainJobFailed) + job.ResultJson = msg + err := models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } +} +func sendHttpReqToBeihang(job *models.Cloudbrain, jsonContent string, req aisafety.TaskReq) { + log.Info("start to send beihang ...") + serialNo, err := aisafety.CreateSafetyTask(req, jsonContent) + if err == nil { + //update serial no to db + job.PreVersionName = serialNo + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } +} + +func isTaskNotFinished(status string) bool { + if status == string(models.ModelArtsTrainJobRunning) || status == string(models.ModelArtsTrainJobWaiting) { + return true + } + if status == string(models.JobWaiting) || status == string(models.JobRunning) { + return true + } + + if status == string(models.ModelArtsTrainJobUnknown) || status == string(models.ModelArtsTrainJobInit) { + return true + } + if status == string(models.ModelArtsTrainJobImageCreating) || status == string(models.ModelArtsTrainJobSubmitTrying) { + return true + } + return false +} + +func AiSafetyCreateForGetGPU(ctx *context.Context) { + t := time.Now() + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["IsCreate"] = true + ctx.Data["type"] = models.TypeCloudBrainOne + ctx.Data["compute_resource"] = models.GPUResource + ctx.Data["datasetType"] = models.TypeCloudBrainOne + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.GPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.GPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.GPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.GPUCombatDataSetUUID + log.Info("GPUBaseDataSetName=" + setting.ModelSafetyTest.GPUBaseDataSetName) + log.Info("GPUBaseDataSetUUID=" + setting.ModelSafetyTest.GPUBaseDataSetUUID) + log.Info("GPUCombatDataSetName=" + setting.ModelSafetyTest.GPUCombatDataSetName) + log.Info("GPUCombatDataSetUUID=" + setting.ModelSafetyTest.GPUCombatDataSetUUID) + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["display_job_name"] = displayJobName + prepareCloudbrainOneSpecs(ctx) + queuesDetail, _ := cloudbrain.GetQueuesDetail() + if queuesDetail != nil { + ctx.Data["QueuesDetail"] = queuesDetail + reqPara, _ := json.Marshal(queuesDetail) + log.Warn("The GPU WaitCount json:", string(reqPara)) + } else { + log.Info("The GPU WaitCount not get") + } + ctx.HTML(200, tplModelSafetyTestCreateGpu) +} + +func AiSafetyCreateForGetNPU(ctx *context.Context) { + t := time.Now() + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["IsCreate"] = true + ctx.Data["type"] = models.TypeCloudBrainTwo + ctx.Data["compute_resource"] = models.NPUResource + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["display_job_name"] = displayJobName + ctx.Data["datasetType"] = models.TypeCloudBrainTwo + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.NPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.NPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.NPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.NPUCombatDataSetUUID + + log.Info("NPUBaseDataSetName=" + setting.ModelSafetyTest.NPUBaseDataSetName) + log.Info("NPUBaseDataSetUUID=" + setting.ModelSafetyTest.NPUBaseDataSetUUID) + log.Info("NPUCombatDataSetName=" + setting.ModelSafetyTest.NPUCombatDataSetName) + log.Info("NPUCombatDataSetUUID=" + setting.ModelSafetyTest.NPUCombatDataSetUUID) + var resourcePools modelarts.ResourcePool + if err := json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err := json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engine_versions"] = versionInfos.Version + + prepareCloudbrainTwoInferenceSpecs(ctx) + waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") + ctx.Data["WaitCount"] = waitCount + log.Info("The NPU WaitCount is " + fmt.Sprint(waitCount)) + ctx.HTML(200, tplModelSafetyTestCreateNpu) +} + +func AiSafetyCreateForPost(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + displayJobName := ctx.Query("display_job_name") + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + + taskType := ctx.QueryInt("type") + description := ctx.Query("description") + ctx.Data["type"] = taskType + ctx.Data["displayJobName"] = displayJobName + ctx.Data["description"] = description + + repo := ctx.Repo.Repository + + tpname := tplCloudBrainModelSafetyNewNpu + if taskType == models.TypeCloudBrainOne { + tpname = tplCloudBrainModelSafetyNewGpu + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeModelSafety), displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr("the job name did already exist", tpname, nil) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tpname, nil) + return + } + } + + if !jobNamePattern.MatchString(jobName) { + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpname, nil) + return + } + + count, err := models.GetModelSafetyCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tpname, nil) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpname, nil) + return + } + } + BootFile := ctx.Query("boot_file") + BootFile = strings.TrimSpace(BootFile) + bootFileExist, err := ctx.Repo.FileExists(BootFile, cloudbrain.DefaultBranchName) + if err != nil || !bootFileExist { + log.Error("Get bootfile error:", err) + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpname, nil) + return + } + if taskType == models.TypeCloudBrainTwo { + err = createForNPU(ctx, jobName) + } else if taskType == models.TypeCloudBrainOne { + err = createForGPU(ctx, jobName) + } + if err != nil { + modelSafetyNewDataPrepare(ctx) + ctx.RenderWithErr(err.Error(), tpname, nil) + } else { + log.Info("to redirect...") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") + } +} + +func createForNPU(ctx *context.Context, jobName string) error { + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + BootFile := ctx.Query("boot_file") + BootFile = strings.TrimSpace(BootFile) + displayJobName := ctx.Query("display_job_name") + description := ctx.Query("description") + + srcDataset := ctx.Query("src_dataset") //uuid + combatDataset := ctx.Query("combat_dataset") //uuid + evaluationIndex := ctx.Query("evaluation_index") + Params := ctx.Query("run_para_list") + specId := ctx.QueryInt64("spec_id") + + engineID := ctx.QueryInt("engine_id") + log.Info("engine_id=" + fmt.Sprint(engineID)) + poolID := ctx.Query("pool_id") + repo := ctx.Repo.Repository + + trainUrl := ctx.Query("train_url") + modelName := ctx.Query("model_name") + modelVersion := ctx.Query("model_version") + ckptName := ctx.Query("ckpt_name") + ckptUrl := "/" + trainUrl + ckptName + log.Info("ckpt url:" + ckptUrl) + + FlavorName := ctx.Query("flaver_names") + EngineName := ctx.Query("engine_names") + + isLatestVersion := modelarts.IsLatestVersion + VersionCount := modelarts.VersionCountOne + + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + resultObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.ResultPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" + log.Info("ckpt url:" + ckptUrl) + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobTypeInference, + ComputeResource: models.NPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainTwo}) + if err != nil || spec == nil { + + //ctx.RenderWithErr("Resource specification not available", tplCloudBrainModelSafetyNewNpu, nil) + return errors.New("Resource specification not available") + } + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) + return errors.New(ctx.Tr("points.insufficient_points_balance")) + } + + //todo: del the codeLocalPath + _, err = ioutil.ReadDir(codeLocalPath) + if err == nil { + os.RemoveAll(codeLocalPath) + } + + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(cloudbrain.DefaultBranchName) + + if err := downloadCode(repo, codeLocalPath, cloudbrain.DefaultBranchName); err != nil { + log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) + return errors.New(ctx.Tr("cloudbrain.load_code_failed")) + } + + //todo: upload code (send to file_server todo this work?) + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.ResultPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_result: %s (%v)", repo.FullName(), err) + return errors.New("Failed to obsMkdir_result") + } + + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) + return errors.New("Failed to obsMkdir_log") + } + + if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) + return errors.New(ctx.Tr("cloudbrain.load_code_failed")) + } + + var parameters models.Parameters + param := make([]models.Parameter, 0) + param = append(param, models.Parameter{ + Label: modelarts.ResultUrl, + Value: "s3:/" + resultObsPath, + }, models.Parameter{ + Label: modelarts.CkptUrl, + Value: "s3:/" + ckptUrl, + }) + uuid := srcDataset + ";" + combatDataset + datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid) + if err != nil { + return err + } + dataPath := dataUrl + jsondatas, err := json.Marshal(datasUrlList) + if err != nil { + log.Error("Failed to Marshal: %v", err) + return err + } + if isMultiDataset { + param = append(param, models.Parameter{ + Label: modelarts.MultiDataUrl, + Value: string(jsondatas), + }) + } + + existDeviceTarget := false + if len(Params) != 0 { + err := json.Unmarshal([]byte(Params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", Params, err) + return errors.New("运行参数错误") + } + + for _, parameter := range parameters.Parameter { + if parameter.Label == modelarts.DeviceTarget { + existDeviceTarget = true + } + if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl { + param = append(param, models.Parameter{ + Label: parameter.Label, + Value: parameter.Value, + }) + } + } + } + if !existDeviceTarget { + param = append(param, models.Parameter{ + Label: modelarts.DeviceTarget, + Value: modelarts.Ascend, + }) + } + + req := &modelarts.GenerateInferenceJobReq{ + JobName: jobName, + DisplayJobName: displayJobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + BootFile, + BootFile: BootFile, + TrainUrl: trainUrl, + WorkServerNumber: 1, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Uuid: uuid, + Parameters: param, //modelarts train parameters + CommitID: commitID, + BranchName: cloudbrain.DefaultBranchName, + Params: Params, + FlavorName: FlavorName, + EngineName: EngineName, + LabelName: evaluationIndex, + IsLatestVersion: isLatestVersion, + VersionCount: VersionCount, + TotalVersionCount: modelarts.TotalVersionCount, + ModelName: modelName, + ModelVersion: modelVersion, + CkptName: ckptName, + ResultUrl: resultObsPath, + Spec: spec, + DatasetName: datasetNames, + JobType: string(models.JobTypeModelSafety), + } + + err = modelarts.GenerateInferenceJob(ctx, req) + if err != nil { + log.Error("GenerateTrainJob failed:%v", err.Error()) + return err + } + return nil +} + +func createForGPU(ctx *context.Context, jobName string) error { + BootFile := ctx.Query("boot_file") + BootFile = strings.TrimSpace(BootFile) + displayJobName := ctx.Query("display_job_name") + description := ctx.Query("description") + image := strings.TrimSpace(ctx.Query("image")) + srcDataset := ctx.Query("src_dataset") //uuid + combatDataset := ctx.Query("combat_dataset") //uuid + evaluationIndex := ctx.Query("evaluation_index") + Params := ctx.Query("run_para_list") + specId := ctx.QueryInt64("spec_id") + TrainUrl := ctx.Query("train_url") + CkptName := ctx.Query("ckpt_name") + modelName := ctx.Query("model_name") + modelVersion := ctx.Query("model_version") + + ckptUrl := setting.Attachment.Minio.RealPath + TrainUrl + CkptName + log.Info("ckpt url:" + ckptUrl) + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobTypeBenchmark, + ComputeResource: models.GPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainOne}) + if err != nil || spec == nil { + return errors.New("Resource specification not available") + } + + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) + return errors.New(ctx.Tr("points.insufficient_points_balance")) + } + + repo := ctx.Repo.Repository + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + os.RemoveAll(codePath) + + if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { + log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) + return errors.New("system error") + } + + err = uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + if err != nil { + log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) + return errors.New("system error") + } + + uuid := srcDataset + ";" + combatDataset + datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) + log.Info("uuid=" + uuid) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return errors.New(ctx.Tr("cloudbrain.error.dataset_select")) + } + command, err := getGpuModelSafetyCommand(BootFile, Params, CkptName, displayJobName) + if err != nil { + log.Error("Get Command failed: %v", err, ctx.Data["MsgID"]) + return errors.New(ctx.Tr("cloudbrain.error.dataset_select")) + } + log.Info("Command=" + command) + + req := cloudbrain.GenerateCloudBrainTaskReq{ + Ctx: ctx, + DisplayJobName: displayJobName, + JobName: jobName, + Image: image, + Command: command, + Uuids: uuid, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, + CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + ModelPath: setting.Attachment.Minio.RealPath + TrainUrl, + BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), + Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + JobType: string(models.JobTypeModelSafety), + Description: description, + BranchName: cloudbrain.DefaultBranchName, + BootFile: BootFile, + Params: Params, + CommitID: "", + ModelName: modelName, + ModelVersion: modelVersion, + CkptName: CkptName, + ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), + Spec: spec, + LabelName: evaluationIndex, + } + + err = cloudbrain.GenerateTask(req) + if err != nil { + return err + } + return nil +} + +func getGpuModelSafetyCommand(BootFile string, params string, CkptName string, DisplayJobName string) (string, error) { + var command string + bootFile := strings.TrimSpace(BootFile) + + if !strings.HasSuffix(bootFile, ".py") { + log.Error("bootFile(%s) format error", bootFile) + return command, errors.New("bootFile format error") + } + + var parameters models.Parameters + var param string + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + return command, err + } + + for _, parameter := range parameters.Parameter { + param += " --" + parameter.Label + "=" + parameter.Value + } + } + + param += " --ckpt_url=" + cloudbrain.ModelMountPath + "/" + CkptName + + command += "python /code/" + bootFile + param + " > " + cloudbrain.ResultPath + "/" + DisplayJobName + "-" + cloudbrain.LogFile + + return command, nil +} + +func modelSafetyNewDataPrepare(ctx *context.Context) error { + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["type"] = ctx.QueryInt("type") + ctx.Data["boot_file"] = ctx.Query("boot_file") + ctx.Data["display_job_name"] = ctx.Query("display_job_name") + ctx.Data["description"] = ctx.Query("description") + ctx.Data["image"] = strings.TrimSpace(ctx.Query("image")) + ctx.Data["src_dataset"] = ctx.Query("src_dataset") //uuid + ctx.Data["combat_dataset"] = ctx.Query("combat_dataset") //uuid + ctx.Data["evaluationIndex"] = ctx.Query("evaluation_index") + ctx.Data["run_para_list"] = ctx.Query("run_para_list") + ctx.Data["spec_id"] = ctx.QueryInt64("spec_id") + ctx.Data["train_url"] = ctx.Query("train_url") + ctx.Data["ckpt_name"] = ctx.Query("ckpt_name") + + ctx.Data["train_url"] = ctx.Query("train_url") + ctx.Data["ckpt_name"] = ctx.Query("ckpt_name") + ctx.Data["model_name"] = ctx.Query("model_name") + ctx.Data["model_version"] = ctx.Query("model_version") + + if ctx.QueryInt("type") == models.TypeCloudBrainOne { + ctx.Data["type"] = models.TypeCloudBrainOne + ctx.Data["compute_resource"] = models.GPUResource + ctx.Data["datasetType"] = models.TypeCloudBrainOne + + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.GPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.GPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.GPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.GPUCombatDataSetUUID + prepareCloudbrainOneSpecs(ctx) + + queuesDetail, _ := cloudbrain.GetQueuesDetail() + if queuesDetail != nil { + ctx.Data["QueuesDetail"] = queuesDetail + } + + } else { + ctx.Data["engine_id"] = ctx.QueryInt("engine_id") + ctx.Data["pool_id"] = ctx.Query("pool_id") + + ctx.Data["type"] = models.TypeCloudBrainTwo + ctx.Data["compute_resource"] = models.NPUResource + ctx.Data["datasetType"] = models.TypeCloudBrainTwo + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.NPUBaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.NPUBaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.NPUCombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.NPUCombatDataSetUUID + + var engines modelarts.Engine + if err := json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engine_versions"] = versionInfos.Version + prepareCloudbrainTwoInferenceSpecs(ctx) + + waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") + ctx.Data["WaitCount"] = waitCount + } + + return nil +} + +func getJsonContent(url string) (string, error) { + + resp, err := http.Get(url) + if err != nil || resp.StatusCode != 200 { + log.Info("Get organizations url error=" + err.Error()) + return "", err + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + return "", err + } + str := string(bytes) + //log.Info("json str =" + str) + + return str, nil +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 73aa9b791..a2ea7d51b 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -51,6 +51,9 @@ const ( tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new" tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show" + tplCloudBrainModelSafetyNewGpu base.TplName = "repo/modelsafety/newgpu" + tplCloudBrainModelSafetyNewNpu base.TplName = "repo/modelsafety/newnpu" + tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit" tplCloudBrainImageEdit base.TplName = "repo/cloudbrain/image/edit" @@ -140,8 +143,11 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled ctx.Data["datasetType"] = models.TypeCloudBrainOne - - ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode") + defaultMode := ctx.Query("benchmarkMode") + if defaultMode == "" { + defaultMode = "alogrithm" + } + ctx.Data["benchmarkMode"] = defaultMode if ctx.Cloudbrain != nil { ctx.Data["branch_name"] = ctx.Cloudbrain.BranchName @@ -734,7 +740,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo } else { task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id")) } - + if task.JobType == string(models.JobTypeModelSafety) { + GetAiSafetyTaskTmpl(ctx) + return + } if err != nil { log.Info("error:" + err.Error()) ctx.NotFound(ctx.Req.URL.RequestURI(), nil) @@ -760,8 +769,8 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State + task.ContainerIp = "" task.ContainerID = taskRes.TaskStatuses[0].ContainerID - task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) if task.DeletedAt.IsZero() { //normal record @@ -1147,6 +1156,8 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain } else if models.IsErrorImageCommitting(err) { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing"))) + } else if isOver20GError(err) { + ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_over_20g"))) } else { ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail"))) } @@ -1156,6 +1167,10 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain ctx.JSON(200, models.BaseOKMessage) } +func isOver20GError(err error) bool { + return strings.Contains(err.Error(), "over max image size 20GB") +} + func checkTopics(Topics string) ([]string, string) { var topics = make([]string, 0) var topicsStr = strings.TrimSpace(Topics) @@ -1821,6 +1836,9 @@ func SyncCloudbrainStatus() { } for _, task := range cloudBrains { + if task.JobType == string(models.JobTypeModelSafety) { + continue + } if task.Type == models.TypeCloudBrainOne { result, err := cloudbrain.GetJob(task.JobID) if err != nil { @@ -2106,14 +2124,14 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, - Type: models.TypeCloudBrainOne, + Type: -1, JobTypes: jobTypes, }) if err != nil { @@ -2160,6 +2178,10 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } } } + if task.JobType == string(models.JobTypeModelSafety) { + ciTasks[i].BenchmarkType = "安全评测" + ciTasks[i].BenchmarkTypeName = "Image Classification" + } } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) @@ -2813,16 +2835,11 @@ func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { } func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { - if task.Type == models.TypeCloudBrainOne { - return ctx.Tr("repo.cloudbrain1") - } else if task.Type == models.TypeCloudBrainTwo { - return ctx.Tr("repo.cloudbrain2") - } else if task.Type == models.TypeCDCenter { - return ctx.Tr("repo.cdCenter") - } else if task.Type == models.TypeC2Net { - return getCutStringAiCenterByAiCenter(task.AiCenter) + if task.Spec != nil { + return task.Spec.AiCenterName + } else { + return "" } - return "" } func getCutStringAiCenterByAiCenter(aiCenter string) string { if aiCenter == "" { @@ -2841,8 +2858,24 @@ func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { return "" } func GetCloudbrainCardDuration(task models.Cloudbrain) string { - cardNum, _, _ := GetCloudbrainCardNumAndType(task) - cardDuration := models.ConvertDurationToStr(int64(cardNum) * task.Duration) + cardNum := int(0) + spec, err := resource.GetCloudbrainSpec(task.ID) + if err != nil { + log.Info("error:" + err.Error()) + return "" + } + if spec != nil { + cardNum = spec.AccCardsNum + } else { + cardNum = 1 + } + var workServerNumber int64 + if task.WorkServerNumber >= 1 { + workServerNumber = int64(task.WorkServerNumber) + } else { + workServerNumber = 1 + } + cardDuration := models.ConvertDurationToStr(workServerNumber * int64(cardNum) * task.Duration) return cardDuration } func GetCloudbrainWaitTime(task models.Cloudbrain) string { @@ -2869,114 +2902,12 @@ func GetCloudbrainWaitTime(task models.Cloudbrain) string { } return waitTime } - -func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) { - if !models.SpecsMapInitFlag { - models.InitCloudbrainOneResourceSpecMap() - } - if !models.GpuInfosMapInitFlag { - models.InitCloudbrainOneGpuInfoMap() - } - flavorName, err := GetCloudbrainFlavorName(task) - if err != nil { - return 0, "", nil - } - return getCardNumAndTypeByFlavorname(flavorName) -} - -func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { - if FlavorName == "" { - return 0, "", nil - } else { - var beginIndex = strings.Index(FlavorName, ":") - var lastIndex = strings.LastIndex(FlavorName, ":") - var endIndex = strings.Index(FlavorName, "*") - if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) { - cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) - return 0, "", err - } - cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) - return cardNum, cardType, err - } - return 0, "", nil - } -} - -func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { - if task.Type == models.TypeCloudBrainOne { - resourceSpec, gpuInfo, err := getCloudBrainOneResourceSpec(task) - if err != nil { - log.Info("getCloudBrainOneResourceSpec err:", err) - return "", err - } else { - if resourceSpec == nil || gpuInfo == nil { - err := errors.New("resourceSpec or gpuInfo is nil") - return "", err - } else { - CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(resourceSpec.GpuNum) + "*Nvidia-" + gpuInfo.Value + - " | CPU:" + strconv.Itoa(resourceSpec.CpuNum) + "核" + strconv.Itoa(resourceSpec.MemMiB) + "MB" - return CloudbrainOneFlavorName, nil - } - } - } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net || task.Type == models.TypeCDCenter) && task.FlavorName != "" { - replaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") - return replaceFlavorName, nil - } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter) && task.FlavorName == "" && task.FlavorCode != "" { - cloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode) - return cloudbrainTwoFlavorName, nil - } else if task.Type == models.TypeCloudBrainTwo && task.JobType == string(models.JobTypeDebug) && task.FlavorName == "" && task.FlavorCode == "" { - tasks, err := models.GetModelartsReDebugTaskByJobId(task.JobID) - if err != nil { - return "", err - } - if len(tasks) >= 1 { - return getFlavorNameByFlavorCode(tasks[0].FlavorCode), nil - } - return "", nil - } - return "", nil -} - -func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, *models.GpuInfo, error) { - gpuQueueDefault := "openidebug" - if task.GpuQueue != "" { - gpuQueueDefault = task.GpuQueue - } - if task.ResourceSpecId >= 0 { - if task.JobType == string(models.JobTypeTrain) { - if models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId] != nil { - return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[gpuQueueDefault], nil - } else { - return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil - } - } else if task.JobType == string(models.JobTypeDebug) { - if models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId] != nil { - return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[gpuQueueDefault], nil - } else { - return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil - } - } else if task.JobType == string(models.JobTypeInference) { - return models.CloudbrainInferenceResourceSpecsMap[task.ResourceSpecId], models.CloudbrainInferenceGpuInfosMap[gpuQueueDefault], nil - } else if task.JobType == string(models.JobTypeBenchmark) || task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { - return models.CloudbrainBenchmarkResourceSpecsMap[task.ResourceSpecId], models.CloudbrainBenchmarkGpuInfosMap[gpuQueueDefault], nil - } +func GetCloudbrainFlavorName(task models.Cloudbrain) string { + if task.Spec != nil { + flavorName := task.Spec.ComputeResource + ":" + fmt.Sprint(task.Spec.AccCardsNum) + "*" + task.Spec.AccCardType + + ",内存:" + strconv.FormatInt(int64(task.Spec.MemGiB), 10) + "GB,共享内存:" + strconv.FormatInt(int64(task.Spec.ShareMemGiB), 10) + "GB" + return flavorName } else { - err := errors.New("ResourceSpecId is null") - return nil, nil, err - } - return nil, nil, nil -} -func getFlavorNameByFlavorCode(flavorCode string) string { - index := strings.LastIndex(flavorCode, ".") - cardNum, err := strconv.Atoi(strings.TrimSpace(flavorCode[index+1 : len(flavorCode)])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) return "" } - cloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) + - "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + - "核" + strconv.Itoa(cardNum*256) + "GB" - return cloudbrainTwoFlavorName } diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 40edc4767..b350343db 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -303,7 +303,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo } if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { ctx.Redirect(ctx.Repo.RepoLink + "/src/branch/" + util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(form.TreePath)) } @@ -475,7 +475,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { ctx.Flash.Success(ctx.Tr("repo.editor.file_delete_success", ctx.Repo.TreePath)) if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { treePath := filepath.Dir(ctx.Repo.TreePath) if treePath == "." { @@ -686,7 +686,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } if form.CommitChoice == frmCommitChoiceNewBranch && ctx.Repo.Repository.UnitEnabled(models.UnitTypePullRequests) { - ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + ctx.Repo.BranchName + "..." + form.NewBranchName) + ctx.Redirect(ctx.Repo.RepoLink + "/compare/" + util.PathEscapeSegments(ctx.Repo.BranchName) + "..." + util.PathEscapeSegments(form.NewBranchName)) } else { ctx.Redirect(ctx.Repo.RepoLink + "/src/branch/" + util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(form.TreePath)) } diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index abf64281a..b78bdebd3 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -713,6 +713,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain DatasetNames: datasetNames, DatasetInfos: datasetInfos, Spec: spec, + CodeName: strings.ToLower(repo.Name), } if form.ModelName != "" { //使用预训练模型训练 req.ModelName = form.ModelName @@ -837,6 +838,7 @@ func GrampusTrainJobShow(ctx *context.Context) { ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } + task.ContainerIp = "" if task.DeletedAt.IsZero() { //normal record result, err := grampus.GetJob(task.JobID) @@ -976,8 +978,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" - command += commandDownload + //no need to download code & dataset by internet } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) @@ -986,8 +987,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //unzip code & dataset if processorType == grampus.ProcessorTypeNPU { - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" - command += commandUnzip + //no need to process } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand @@ -1024,7 +1024,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo var commandCode string if processorType == grampus.ProcessorTypeNPU { - commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" + commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py /tmp/log/train.log" + paramCode + ";" } else if processorType == grampus.ProcessorTypeGPU { if pretrainModelFileName != "" { paramCode += " --ckpt_url" + "=" + workDir + "pretrainmodel/" + pretrainModelFileName diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 5487231a2..6e44b3cd2 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -985,7 +985,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) //isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath - codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" // dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" @@ -1108,8 +1108,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - // parentDir := VersionOutputPath + "/" - if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + parentDir := VersionOutputPath + "/" + if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) @@ -1795,7 +1795,7 @@ func TrainJobShow(ctx *context.Context) { datasetList = append(datasetList, GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)) VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) - + VersionListTasks[i].ContainerIp = "" //add spec s, err := resource.GetCloudbrainSpec(task.Cloudbrain.ID) if err != nil { @@ -2200,6 +2200,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ResultUrl: resultObsPath, Spec: spec, DatasetName: datasetNames, + JobType: string(models.JobTypeInference), } err = modelarts.GenerateInferenceJob(ctx, req) @@ -2737,7 +2738,7 @@ func TrainJobDownloadLogFile(ctx *context.Context) { return } ctx.Resp.Header().Set("Cache-Control", "max-age=0") - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) } func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) { var isMultiDataset bool diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 03d2d832a..2c8c2f45b 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -12,6 +12,7 @@ import ( "path" "regexp" "strings" + "text/template" "unicode/utf8" "code.gitea.io/gitea/modules/validation" @@ -212,7 +213,7 @@ func CreatePost(ctx *context.Context, form auth.CreateRepoForm) { opts := models.GenerateRepoOptions{ Name: form.RepoName, Alias: form.Alias, - Description: form.Description, + Description: template.HTMLEscapeString(form.Description), Private: form.Private, GitContent: form.GitContent, Topics: form.Topics, diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 11efdf275..5fcfb2287 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -8,6 +8,7 @@ package repo import ( "errors" "fmt" + "html/template" "io/ioutil" "net/url" "regexp" @@ -129,7 +130,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { // In case it's just a case change. repo.Name = newRepoName repo.LowerName = strings.ToLower(newRepoName) - repo.Description = form.Description + repo.Description = template.HTMLEscapeString(form.Description) repo.Website = form.Website repo.IsTemplate = form.Template repo.Alias = newAlias diff --git a/routers/repo/setting_protected_branch.go b/routers/repo/setting_protected_branch.go index ab0fd77ee..f1ea17528 100644 --- a/routers/repo/setting_protected_branch.go +++ b/routers/repo/setting_protected_branch.go @@ -5,6 +5,7 @@ package repo import ( + "code.gitea.io/gitea/modules/util" "fmt" "strings" "time" @@ -192,7 +193,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) } if f.RequiredApprovals < 0 { ctx.Flash.Error(ctx.Tr("repo.settings.protected_branch_required_approvals_min")) - ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, branch)) + ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, util.PathEscapeSegments(branch))) } var whitelistUsers, whitelistTeams, mergeWhitelistUsers, mergeWhitelistTeams, approvalsWhitelistUsers, approvalsWhitelistTeams []int64 @@ -263,7 +264,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) return } ctx.Flash.Success(ctx.Tr("repo.settings.update_protect_branch_success", branch)) - ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, branch)) + ctx.Redirect(fmt.Sprintf("%s/settings/branches/%s", ctx.Repo.RepoLink, util.PathEscapeSegments(branch))) } else { if protectBranch != nil { if err := ctx.Repo.Repository.DeleteProtectedBranch(protectBranch.ID); err != nil { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 66a357c79..9a523ea48 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,15 +6,17 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - "code.gitea.io/gitea/services/reward" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/modelapp" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/modules/slideimage" "code.gitea.io/gitea/routers/image" @@ -346,6 +348,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/user/login/kanban", user.SignInPostAPI) m.Get("/home/term", routers.HomeTerm) m.Get("/home/privacy", routers.HomePrivacy) + m.Get("/extension/tuomin/upload", modelapp.ProcessImageUI) + m.Post("/extension/tuomin/upload", reqSignIn, modelapp.ProcessImage) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") @@ -1198,7 +1202,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/modelmanage", func() { m.Post("/create_model", repo.SaveModel) - m.Post("/create_model_convert", reqRepoModelManageWriter, repo.SaveModelConvert) + m.Post("/create_model_convert", reqWechatBind, reqRepoModelManageWriter, repo.SaveModelConvert) m.Post("/create_new_model", repo.SaveNewNameModel) m.Delete("/delete_model", repo.DeleteModel) m.Post("/delete_model_convert/:id", repo.DeleteModelConvert) @@ -1225,6 +1229,18 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/downloadall", repo.DownloadMultiModelFile) }, context.RepoRef()) + m.Group("/modelsafety", func() { + m.Group("/:id", func() { + m.Get("/show", reqRepoCloudBrainWriter, repo.GetAiSafetyTaskTmpl) + m.Get("", reqRepoCloudBrainWriter, repo.GetAiSafetyTask) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.StopAiSafetyTask) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.DelAiSafetyTask) + }) + m.Get("/create_gpu", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.AiSafetyCreateForGetGPU) + m.Get("/create_npu", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.AiSafetyCreateForGetNPU) + m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.AiSafetyCreateForPost) + }, context.RepoRef()) + m.Group("/debugjob", func() { m.Get("", reqRepoCloudBrainReader, repo.DebugJobIndex) }, context.RepoRef()) diff --git a/routers/user/home.go b/routers/user/home.go index 78e6c00e9..1a20c26e2 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -23,7 +23,6 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/routers/repo" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" @@ -778,7 +777,7 @@ func Cloudbrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } @@ -841,9 +840,6 @@ func Cloudbrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) - } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index 14951dd69..97d968954 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -89,7 +89,7 @@
{{$JobID := '0'}} - {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}} + {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}} {{$JobID = .Cloudbrain.ID}} {{else}} {{$JobID = .JobID}} @@ -124,7 +124,7 @@ {{.DisplayJobName}} - {{else if eq .JobType "BENCHMARK"}} + {{else if eq .JobType "BENCHMARK" "MODELSAFETY"}} @@ -135,14 +135,13 @@
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} + {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -185,6 +183,17 @@ var spanEl = document.querySelector('.card_type_{{.DisplayJobName}}_{{$JobID}}'); spanEl.setAttribute('title', cardType); spanEl.innerText = cardType; + + var cluster = spec.Cluster || '--'; + var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--'; + spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', cluster); + spanEl.innerText = clusterName; + + var aiCenter = spec.AiCenterName || '--'; + spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); + spanEl.setAttribute('title', aiCenter); + spanEl.innerText = aiCenter; })(); @@ -205,11 +214,12 @@
- {{.JobName}} @@ -238,8 +248,19 @@
{{end}} - +
+ {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.stop"}} + +
+ {{else}} {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
{{$.CsrfTokenHtml}} @@ -258,6 +279,7 @@ {{$.i18n.Tr "repo.stop"}} {{end}} + {{end}}
{{if eq .JobType "TRAIN"}} @@ -268,17 +290,30 @@
{{end}} - - {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.delete"}} - - + {{if eq .JobType "MODELSAFETY"}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{else}} +
+ {{$.CsrfTokenHtml}} + + {{$.i18n.Tr "repo.delete"}} + +
+ {{end}}
diff --git a/templates/admin/cloudbrain/search.tmpl b/templates/admin/cloudbrain/search.tmpl index 72077b8a2..3562672a8 100644 --- a/templates/admin/cloudbrain/search.tmpl +++ b/templates/admin/cloudbrain/search.tmpl @@ -42,6 +42,7 @@ BENCHMARK SNN4IMAGENET BRAINSCORE + MODELSAFETY
diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 3de7b25a0..f9908f6f0 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -32,9 +32,15 @@
- {{.i18n.Tr "custom.head.project"}} {{.i18n.Tr "custom.head.dataset"}} +
+ {{.i18n.Tr "repo.model_manager"}} + +
+ 模型体验 +
+
+ {{else if eq .benchmarkMode "aisafety"}} +
+ {{.CsrfTokenHtml}} + + +
+ +
+ {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} + {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} + 模型安全评测 +
+
+ +
+
+ + {{template "custom/task_wait_count" .}} +
+
+ + + {{.i18n.Tr "repo.cloudbrain_jobname_err"}} +
+
+ + +
+ + +
+
+ +
+ + +
+ +
+ + + {{.i18n.Tr "cloudbrain.view_sample"}} +
+ + +
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+
+
+ {{end}} diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 3c71140bf..a54989df4 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -253,7 +253,7 @@
{{if and ($.canDownload) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 45b6501fd..67548696f 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -265,7 +265,7 @@
{{if and ($.canDownload) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}},EngineName:{{.EngineName}},ComputeResource:{{.ComputeResource}},Type:{{.Type}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index f3929ec60..6683fc94a 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -207,6 +207,18 @@ padding-left: 1rem; padding-top: 0.5rem; } + .dataset_nowrap_two_line{ + word-wrap: break-word; + word-break: break-all; + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + line-clamp: 2; + -webkit-line-clamp: 2; + text-overflow: -o-ellipsis-lastline; + max-height: 50px; + }
{{template "repo/header" .}} @@ -437,15 +449,14 @@ {{range $.datasetDownload}} - - + {{if eq .IsDelete true}} {{.DatasetName}}({{$.i18n.Tr "dataset.file_deleted"}}) {{else}} {{.DatasetName}} {{end}} - {{.DatasetDownloadLink}} +
{{.DatasetDownloadLink}}
{{$.i18n.Tr "dataset.download_copy"}} {{end}} diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 8e2810317..0e1e8eedb 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -261,29 +261,25 @@
- -
- + {{if .WorkNode}} + {{range .WorkNode}} - {{if $.work_server_number}} - {{if eq . $.work_server_number }} - - {{else}} - - {{end}} - {{else}} - - {{end}} - {{end}} + {{if $.work_server_number}} + {{if eq . $.work_server_number }} + + {{else}} + + {{end}} + {{else}} + + {{end}} + {{end}} - {{else}} - - {{end}} + {{else}} + + {{end}}
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 0c9bb3ba4..e95a1233d 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -249,7 +249,7 @@ {{$.CsrfTokenHtml}} {{if and (.CanModify) (ne .Status "WAITING") ($.Permission.CanWrite $.UnitTypeModelManage) }} {{$.i18n.Tr "repo.modelarts.create_model"}} + onclick="showcreate({DisplayJobName:{{.DisplayJobName}},JobName:{{.JobName}},JobID:{{.JobID}},VersionName:{{.VersionName}},EngineID:{{.EngineID}}})">{{$.i18n.Tr "repo.modelarts.create_model"}} {{else}} {{$.i18n.Tr "repo.modelarts.create_model"}} {{end}} @@ -783,7 +783,6 @@ } let dirKey="isOnlyDir--:&"; function loadSelectedModelFile(trainJob){ - console.log("trainJob=" + trainJob); $('#choice_file').dropdown('clear') $("#model-file").empty() if(trainJob ==null || trainJob ==""){ @@ -893,11 +892,11 @@ $('#JobName').val(obj.DisplayJobName).addClass('model_disabled') $('input[name="JobId"]').val(obj.JobID) $('input[name="VersionName"]').val(obj.VersionName).addClass('model_disabled') - if(obj.EngineID ==122 || obj.EngineID ==35 || obj.EngineID ==-1){ + if(obj.EngineID ==122 || obj.EngineID ==35 || obj.EngineID ==-1 || obj.EngineID ==37){ $('input[name="Engine_name"]').val("MindSpore").addClass('model_disabled'); $('input[name="Engine"]').val(2); } - if(obj.EngineID ==121){ + if(obj.EngineID ==121 || obj.EngineID ==38){ $('input[name="Engine_name"]').val("TensorFlow").addClass('model_disabled'); $('input[name="Engine"]').val(1); } diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 3c4ac7f69..92eefca2e 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -581,10 +581,10 @@ if(versionList[0].Engine == engineOption){ return "selected=\"selected\""; }else{ - if(versionList[0].Engine==122 && engineOption==2){ + if((versionList[0].Engine==122 || versionList[0].Engine==37) && engineOption==2){ return "selected=\"selected\""; } - if(versionList[0].Engine==121 && engineOption==1){ + if((versionList[0].Engine==121 || versionList[0].Engine==38) && engineOption==1){ return "selected=\"selected\""; } } diff --git a/templates/repo/modelmanage/showinfo.tmpl b/templates/repo/modelmanage/showinfo.tmpl index e18c6bd99..4cc0305ed 100644 --- a/templates/repo/modelmanage/showinfo.tmpl +++ b/templates/repo/modelmanage/showinfo.tmpl @@ -244,9 +244,9 @@ function loadInfo(){ function getEngineName(model){ if(model.Engine == 0){ return "PyTorch"; - }else if(model.Engine == 1 || model.Engine == 121){ + }else if(model.Engine == 1 || model.Engine == 121 || model.Engine == 38){ return "TensorFlow"; - }else if(model.Engine == 2 || model.Engine == 122 || model.Engine == 35){ + }else if(model.Engine == 2 || model.Engine == 122 || model.Engine == 35 || model.Engine == 37){ return "MindSpore"; }else if(model.Engine == 3){ return "Other"; diff --git a/templates/repo/modelsafety/indicators_descr.tmpl b/templates/repo/modelsafety/indicators_descr.tmpl new file mode 100644 index 000000000..a8027659a --- /dev/null +++ b/templates/repo/modelsafety/indicators_descr.tmpl @@ -0,0 +1,63 @@ + + + + + diff --git a/templates/repo/modelsafety/new.tmpl b/templates/repo/modelsafety/new.tmpl new file mode 100644 index 000000000..07bcd311d --- /dev/null +++ b/templates/repo/modelsafety/new.tmpl @@ -0,0 +1,469 @@ +{{template "base/head" .}} + +{{template "custom/global_mask" .}} +
+ {{template "repo/header" .}} +
+ {{$Grampus := (or (eq (index (SubJumpablePath .Link) 1) "create_grampus_gpu") (eq (index (SubJumpablePath .Link) 1) "create_grampus_npu"))}} + + {{template "base/alert" .}} +

+ {{.i18n.Tr "repo.modelarts.evaluate_job.new_job"}} +

+
+ +
+ {{.CsrfTokenHtml}} + + + + +
+ +
+ {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} + {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} + {{.i18n.Tr "modelsafety.model_security_evaluation"}} +
+
+ +
+ +
+ + + + + + CPU/GPU + + + + + + + Ascend NPU +
+
+
+
+ + {{template "custom/task_wait_count" .}} +
+
+ +
+
+ + {{.i18n.Tr "modelsafety.new_model_security_evaluation_tips"}} +
+
+
+
+ + + {{.i18n.Tr "repo.cloudbrain_jobname_err"}} +
+
+ + +
+ +
+
+ + +
+ +
+ +
+
+ +
+ + + +
+ + {{if not $Grampus}} + {{if eq .datasetType 0}} +
+ + {{else}} +
+ +
+ +
+
+ +
+
+ {{end}} + {{else}} + {{if eq .datasetType 0}} +
+ {{else}} +
+ + +
+ {{end}} + {{end}} + +
+ + + + + + {{.i18n.Tr "cloudbrain.view_sample"}} +
+
+ +
+ +
+
+ +
+ +
+
+
+
+ + + + {{template "repo/modelsafety/indicators_descr".}} +
+
+ + + {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} + + +
+
+
+ + + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}} +
+
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+
+
+
+
+
+{{template "base/footer" .}} + + + + diff --git a/templates/repo/modelsafety/newgpu.tmpl b/templates/repo/modelsafety/newgpu.tmpl new file mode 100644 index 000000000..d41c8f816 --- /dev/null +++ b/templates/repo/modelsafety/newgpu.tmpl @@ -0,0 +1 @@ +{{template "repo/modelsafety/new".}} diff --git a/templates/repo/modelsafety/newnpu.tmpl b/templates/repo/modelsafety/newnpu.tmpl new file mode 100644 index 000000000..d41c8f816 --- /dev/null +++ b/templates/repo/modelsafety/newnpu.tmpl @@ -0,0 +1 @@ +{{template "repo/modelsafety/new".}} diff --git a/templates/repo/modelsafety/show.tmpl b/templates/repo/modelsafety/show.tmpl new file mode 100644 index 000000000..6954f6f6b --- /dev/null +++ b/templates/repo/modelsafety/show.tmpl @@ -0,0 +1,893 @@ +{{template "base/head" .}} + +
+
+
+
+
+
+
+
+
+
+ {{template "repo/header" .}} +
+

+
+ + {{$.i18n.Tr "repo.cloudbrain"}} + +
/
+ + {{$.i18n.Tr "repo.modelarts.evaluate_job"}} + +
/
+
+
+

+
+ +
+
+
+ + + +
+ + + + {{$.i18n.Tr "repo.modelarts.status"}}： + + + + + + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： + +
+ + +
+
+
+
+
+
+ {{$.i18n.Tr "repo.modelarts.train_job.config"}} + {{$.i18n.Tr "repo.modelarts.log"}} + {{.i18n.Tr "modelsafety.evaluation_result"}} +
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{$.i18n.Tr "repo.cloudbrain_task"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.status"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.createtime"}} + +
+ +
+
+ {{$.i18n.Tr "repo.cloudbrain.time.starttime"}} + +
+
+ {{$.i18n.Tr "repo.cloudbrain.time.endtime"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}} + +
+
+ {{$.i18n.Tr "cloudbrain.mirror"}} + {{$.i18n.Tr "repo.modelarts.train_job.AI_driver"}} + +
+ + +
+
+ {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_scenes"}} + +
+
+ {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}} + +
+
+ {{.i18n.Tr "modelsafety.base_data_set"}} + +
+
+ {{.i18n.Tr "modelsafety.combat_data_set"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}} + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{$.i18n.Tr "repo.modelarts.code_version"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.description"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.resource_type"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.standard"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.model_name"}} + +
+
+ {{$.i18n.Tr "repo.modelconvert.modelversion"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.infer_job_model_file"}} + +
+
+ {{$.i18n.Tr "repo.cloudbrain_creator"}} + +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.start_file"}} + +
+
+ {{.i18n.Tr "modelsafety.evaluation_indicators"}} + +
+
+
+
+
+ +
+
+
+ +
+ + + + + + + +
+
+
+
+ + + 

+                             

+                      
                         
+                    

+                    
                      
+                      
+                      

+                        {{template "repo/modelsafety/indicators_descr".}}
+                      

+                      

+                        
{{.i18n.Tr "modelsafety.no_data"}}

+                      

+                    

+                

+            

+        

+        {{template "base/paginate" .}}
+    

+    
+    

+        

+            

+                 {{$.i18n.Tr "cloudbrain.delete_task"}}
+            

+
+            

+                
{{$.i18n.Tr "cloudbrain.task_delete_confirm"}}

+            

+            

+                

+                     {{$.i18n.Tr "cloudbrain.operate_cancel"}}
+                

+                

+                     {{$.i18n.Tr "cloudbrain.operate_confirm"}}
+                

+            

+        

+    

+

+{{template "base/footer" .}}
+
+
+
+
diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl
index 17b40315c..5b7ab6aca 100755
--- a/templates/user/dashboard/cloudbrains.tmpl
+++ b/templates/user/dashboard/cloudbrains.tmpl
@@ -15,11 +15,11 @@
         data-all-compute="{{.i18n.Tr "admin.cloudbrain.all_computing_resources"}}"
         data-all-status="{{.i18n.Tr "admin.cloudbrain.all_status"}}">

     {{template "admin/cloudbrain/search_dashboard" .}}
-    

+    

         {{template "base/alert" .}}
-        

+        

             

-                

+                

                     
                     

                         
@@ -71,7 +71,7 @@
                             

                                 
                                 {{$JobID := '0'}}
-                                {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}}
+                                {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}}
                                 {{$JobID = .Cloudbrain.ID}}
                                 {{else}}
                                 {{$JobID = .JobID}}
@@ -85,7 +85,7 @@
                                         {{.DisplayJobName}}
                                     
-                                    {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}
+                                    {{else if (eq .JobType "SNN4IMAGENET" "BRAINSCORE")}}
                                     
@@ -106,7 +106,7 @@
                                         {{.DisplayJobName}}
                                     
-                                    {{else if eq .JobType "BENCHMARK"}}
+                                    {{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}
                                     
@@ -117,14 +117,13 @@
                                 

                                 
                                 

-                                    {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
+                                    {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
                                 

                                 
                                 

                                     
                                         
                                 

-                                    {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
+                                    {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
                                 

                                 
                                 

@@ -169,6 +167,17 @@
                                         var spanEl = document.querySelector('.card_type_{{.DisplayJobName}}_{{$JobID}}');
                                         spanEl.setAttribute('title', cardType);
                                         spanEl.innerText = cardType;
+                                        
+                                        var cluster = spec.Cluster || '--';
+                                        var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--';
+                                        spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}');
+                                        spanEl.setAttribute('title', cluster);
+                                        spanEl.innerText = clusterName;
+
+                                        var aiCenter = spec.AiCenterName || '--';
+                                        spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}');
+                                        spanEl.setAttribute('title', aiCenter);
+                                        spanEl.innerText = aiCenter;
                                     })();
                                 
                                 
@@ -202,23 +211,35 @@
                                     {{end}}
                                     
                                     

-                                        {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
-                                        

-                                            {{$.CsrfTokenHtml}}
+                                        {{if eq .JobType "MODELSAFETY"}}
+                                            
+                                                {{$.CsrfTokenHtml}}
+                                                
+                                                    {{$.i18n.Tr "repo.stop"}}
+                                                
+                                            

+                                        {{else}}
+                                            {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
+                                            

+                                                {{$.CsrfTokenHtml}}
+                                                
+                                                    {{$.i18n.Tr "repo.stop"}}
+                                                
+                                            

+                                            {{else}}
                                             
+                                                class='ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "STOPPED" "SUCCEEDED" "CREATE_FAILED"}}disabled {{else}} blue {{end}}button'
+                                                data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 1}}modelarts/inference-job{{else}}cloudbrain/train-job{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}modelarts/train-job{{else if eq .Cloudbrain.Type 0}}cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}grampus/train-job{{end}}{{end}}'
+                                                data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
                                                 {{$.i18n.Tr "repo.stop"}}
                                             
-                                        
-                                        {{else}}
-                                        
-                                            {{$.i18n.Tr "repo.stop"}}
-                                        
+                                            {{end}}
                                         {{end}}
                                     

                                     {{if eq .JobType "BENCHMARK"}}
@@ -228,7 +249,6 @@
                                             {{$.i18n.Tr "repo.score"}}
                                         
                                     

-
                                     {{end}}
                                     
                                     {{if eq .JobType "TRAIN"}}
@@ -239,6 +259,18 @@
                                     

                                     {{end}}
                                     
+                                    {{if eq .JobType "MODELSAFETY"}}
+                                    

+                                        {{$.CsrfTokenHtml}}
+                                        
+                                            {{$.i18n.Tr "repo.delete"}}
+                                        
+                                    

+                                    {{else}}
                                     

@@ -250,6 +282,8 @@
                                             {{$.i18n.Tr "repo.delete"}}
                                         
                                     

+                                    {{end}}
+                                    
                                 

                             

                         

@@ -377,19 +411,18 @@
                         

 
                         {{end}}
-                        {{end}}
-                        

-                            

-                                
-                                
-                            

-                        

+                        {{end}}                        
                     

-
                 

             

+        
        
+    

+    

+        

+            
+            
         

     

     
diff --git a/web_src/js/components/dataset/selectDataset.vue b/web_src/js/components/dataset/selectDataset.vue
index 77d54033f..057298d13 100755
--- a/web_src/js/components/dataset/selectDataset.vue
+++ b/web_src/js/components/dataset/selectDataset.vue
@@ -1011,7 +1011,8 @@ export default {
     if (
       location.href.indexOf("benchmark") !== -1 ||
       location.href.indexOf("train-job") !== -1 ||
-      location.href.indexOf("inference") !== -1
+      location.href.indexOf("inference") !== -1 || 
+      location.href.indexOf("modelsafety") !== -1
     ) {
       this.benchmarkNew = true;
     }
diff --git a/web_src/js/components/images/selectGrampusImages.vue b/web_src/js/components/images/selectGrampusImages.vue
index f7af01e9f..eced8cf88 100755
--- a/web_src/js/components/images/selectGrampusImages.vue
+++ b/web_src/js/components/images/selectGrampusImages.vue
@@ -242,7 +242,8 @@ export default {
     this.getImageListPublic();
     if (
       location.href.indexOf("benchmark") !== -1 ||
-      location.href.indexOf("train-job") !== -1
+      location.href.indexOf("train-job") !== -1 ||
+      location.href.indexOf("modelsafety") !== -1
     ) {
       this.benchmarkNew = true;
     }
diff --git a/web_src/js/components/images/selectImages.vue b/web_src/js/components/images/selectImages.vue
index add4553d0..ca2ab3138 100755
--- a/web_src/js/components/images/selectImages.vue
+++ b/web_src/js/components/images/selectImages.vue
@@ -47,6 +47,7 @@
           type="text"
           :placeholder="i18n.image_search_placeholder"
           v-model="search"
+          @keydown.enter.stop.prevent="searchName"
         />
       

       
@@ -505,7 +506,8 @@ export default {
     if (
       location.href.indexOf("train-job") !== -1 ||
       location.href.indexOf("inference-job") !== -1 ||
-      location.href.indexOf("benchmark") !== -1
+      location.href.indexOf("benchmark") !== -1 ||
+      location.href.indexOf("modelsafety") !== -1
     ) {
       this.benchmarkNew = true;
     }
diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index f451789d1..3aafcc1a6 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -148,6 +148,10 @@ export default async function initCloudrainSow() {
           $(`#${version_name}-log-down`)
             .removeClass("ti-download-file")
             .addClass("disabled");
+        } else {
+          $(`#${version_name}-log-down`)
+            .addClass("ti-download-file")
+            .removeClass("disabled");
         }
         $(`#log${version_name} input[name=end_line]`).val(data.EndLine); //如果变动就改变所对应的值
         $(`#log${version_name} input[name=start_line]`).val(data.StartLine);
diff --git a/web_src/js/features/cloudrbanin.js b/web_src/js/features/cloudrbanin.js
index 2df780233..698523d11 100644
--- a/web_src/js/features/cloudrbanin.js
+++ b/web_src/js/features/cloudrbanin.js
@@ -8,6 +8,7 @@ export default async function initCloudrain() {
   function loadJobStatus() {
     $(".job-status").each((index, job) => {
       const ID = job.dataset.jobid;
+      if (!ID) return;
       const repoPath = job.dataset.repopath;
       // const computeResource = job.dataset.resource
       const versionname = job.dataset.version;
@@ -144,6 +145,7 @@ export default async function initCloudrain() {
   function loadShowJobStatus() {
     $(".ui.accordion.border-according").each((index, job) => {
       const jobID = job.dataset.jobid;
+      if (!jobID) return;
       const repoPath = job.dataset.repopath;
       const versionname = job.dataset.version;
       // ['IMAGE_FAILED','SUBMIT_FAILED','DELETE_FAILED','KILLED','COMPLETED','FAILED','CANCELED','LOST','START_FAILED']
@@ -152,7 +154,7 @@ export default async function initCloudrain() {
       //     || job.textContent.trim() == 'CANCELED' || job.textContent.trim() == 'LOST') {
       //     return
       // }
-      let status = $(`#${versionname}-status-span`).text();
+      let status = $(`#${versionname}-status-span`).text().trim();
 
       if (
         [
diff --git a/web_src/js/index.js b/web_src/js/index.js
index b796d020a..9a1f04296 100755
--- a/web_src/js/index.js
+++ b/web_src/js/index.js
@@ -65,6 +65,7 @@ const { AppSubUrl, StaticUrlPrefix, csrf } = window.config;
 Object.defineProperty(Vue.prototype, "$echarts", {
   value: echarts,
 });
+window.echarts = echarts;
 
 function htmlEncode(text) {
   return jQuery("
").text(text).html();
diff --git a/web_src/js/standalone/cloudbrainNew.js b/web_src/js/standalone/cloudbrainNew.js
index 170344f7a..bea652dfd 100644
--- a/web_src/js/standalone/cloudbrainNew.js
+++ b/web_src/js/standalone/cloudbrainNew.js
@@ -89,7 +89,7 @@
           identifier: "job_name",
           rules: [
             {
-              type: "regExp[/^[a-zA-Z0-9-_]{1,36}$/]",
+              type: "regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]",
             },
           ],
         },
@@ -97,7 +97,7 @@
           identifier: "display_job_name",
           rules: [
             {
-              type: "regExp[/^[a-zA-Z0-9-_]{1,36}$/]",
+              type: "regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]",
             },
           ],
         },
@@ -121,6 +121,7 @@
       },
       onFailure: function (e) {
         isValidate = false;
+        createFlag = false;
         return false;
       },
     });
diff --git a/web_src/vuepages/apis/modules/desensitization.js b/web_src/vuepages/apis/modules/desensitization.js
new file mode 100644
index 000000000..2d0ab7b97
--- /dev/null
+++ b/web_src/vuepages/apis/modules/desensitization.js
@@ -0,0 +1,13 @@
+import service from "../service";
+
+// 数据脱敏接口
+export const postImgDesensitization = (params, data) => {
+  return service({
+    url: "/extension/tuomin/upload",
+    headers: { Accept: "image/png" },
+    responseType: "blob",
+    method: "POST",
+    params: params,
+    data: data,
+  });
+};
diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js
index c60b90f91..9258d1656 100644
--- a/web_src/vuepages/langs/config/en-US.js
+++ b/web_src/vuepages/langs/config/en-US.js
@@ -176,6 +176,22 @@ const en = {
     Activated: 'Activated',
     notActive: 'Not active',
   },
+  tranformImageFailed:'Picture desensitization failed',
+  originPicture:'Origin picture',
+  desensitizationPicture:'Desensitization picture',
+  desensitizationObject:'Desensitization object',
+  example:'Example',
+  startDesensitization:'Start desensitization',
+  all:'All',
+  onlyFace:'Only face',
+  onlyLicensePlate:'Only license plate',
+  dragThePictureHere:'Drag the picture here',
+  or:'or',
+  clickUpload:'Click upload',
+  dataDesensitizationModelExperience:'Data desensitization model experience',
+  dataDesensitizationModelDesc:'Use AI technology to desensitize the face and license plate number in the picture. For more information about this model, please visit the project',
+  limitFilesUpload:'Only jpg/jpeg/png files can be uploaded',
+  limitSizeUpload:'The size of the uploaded file cannot exceed 20M!',
 }
 
 export default en;
diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js
index 2b451e2db..c9d9f7a8c 100644
--- a/web_src/vuepages/langs/config/zh-CN.js
+++ b/web_src/vuepages/langs/config/zh-CN.js
@@ -176,6 +176,22 @@ const zh = {
     Activated: '已激活',
     notActive: '未激活',
   },
+  tranformImageFailed:'图片脱敏失败',
+  originPicture:'原始图片',
+  desensitizationPicture:'脱敏图片',
+  desensitizationObject:'脱敏对象',
+  example:'示例',
+  startDesensitization:'开始处理',
+  all:'全部',
+  onlyFace:'仅人脸',
+  onlyLicensePlate:'仅车牌',
+  dragThePictureHere:'拖动图片到这里',
+  or:'或',
+  clickUpload:'点击上传',
+  dataDesensitizationModelExperience:'数据脱敏模型体验',
+  dataDesensitizationModelDesc:'利用人工智能AI技术，把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目',
+  limitFilesUpload:'只能上传 jpg/jpeg/png 格式的文件',
+  limitSizeUpload:'上传文件大小不能超过 20M !',
 }
 
 export default zh;
diff --git a/web_src/vuepages/pages/model/tuomin/index.vue b/web_src/vuepages/pages/model/tuomin/index.vue
new file mode 100644
index 000000000..6c7813977
--- /dev/null
+++ b/web_src/vuepages/pages/model/tuomin/index.vue
@@ -0,0 +1,380 @@
+
+
+
+
+
diff --git a/web_src/vuepages/pages/model/tuomin/vp-model-tuomin.js b/web_src/vuepages/pages/model/tuomin/vp-model-tuomin.js
new file mode 100644
index 000000000..48e542500
--- /dev/null
+++ b/web_src/vuepages/pages/model/tuomin/vp-model-tuomin.js
@@ -0,0 +1,17 @@
+import Vue from 'vue';
+import ElementUI from 'element-ui';
+import 'element-ui/lib/theme-chalk/index.css';
+import localeEn from 'element-ui/lib/locale/lang/en';
+import localeZh from 'element-ui/lib/locale/lang/zh-CN';
+import { i18n, lang } from '~/langs';
+import App from './index.vue';
+
+Vue.use(ElementUI, {
+  locale: lang === 'zh-CN' ? localeZh : localeEn,
+  size: 'small',
+});
+
+new Vue({
+  i18n,
+  render: (h) => h(App),
+}).$mount('#__vue-root');
diff --git a/web_src/vuepages/pages/resources/specification/index.vue b/web_src/vuepages/pages/resources/specification/index.vue
index 58d95d74f..4dc2657f3 100644
--- a/web_src/vuepages/pages/resources/specification/index.vue
+++ b/web_src/vuepages/pages/resources/specification/index.vue
@@ -21,16 +21,16 @@
       

     

     

-      

-        
-          
-          
+      

+        
+          
+          
           
-          
           
-          
           
           
           
-          
            {
     if (record.SourceType === 'ADMIN_OPERATE') {
       out.remark = record.Remark;
     } else if (record.SourceType === 'ACCOMPLISH_TASK') {
+      if (record.Action?.Cloudbrain?.JobType === 'MODELSAFETY') {
+        record.Action.Cloudbrain.oJobType = 'MODELSAFETY';
+        record.Action.Cloudbrain.JobType = 'BENCHMARK';
+      }
       switch (record?.Action?.TaskType) {
         case 'CreatePublicRepo': // 创建公开项目 - 创建了项目OpenI/aiforge
           out.remark = `${i18n.t('createdRepository')}${record.Action.ShortRepoFullDisplayName}`;