diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go
index a88da8fe5..d9adda2dc 100644
--- a/models/ai_model_manage.go
+++ b/models/ai_model_manage.go
@@ -88,7 +88,7 @@ type AiModelQueryOptions struct {
}
func (a *AiModelConvert) IsGpuTrainTask() bool {
- if a.SrcEngine == 0 || a.SrcEngine == 1 {
+ if a.SrcEngine == 0 || a.SrcEngine == 1 || a.SrcEngine == 4 || a.SrcEngine == 6 {
return true
}
return false
diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 1eaeffcd3..989d0905b 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -1070,6 +1070,12 @@ type CreateInferenceJobParams struct {
InfConfig InfConfig `json:"config"`
WorkspaceID string `json:"workspace_id"`
}
+type CreateInfUserImageParams struct {
+ JobName string `json:"job_name"`
+ Description string `json:"job_desc"`
+ Config InfUserImageConfig `json:"config"`
+ WorkspaceID string `json:"workspace_id"`
+}
type InfConfig struct {
WorkServerNum int `json:"worker_server_num"`
@@ -1084,6 +1090,21 @@ type InfConfig struct {
PoolID string `json:"pool_id"`
}
+type InfUserImageConfig struct {
+ WorkServerNum int `json:"worker_server_num"`
+ AppUrl string `json:"app_url"` //训练作业的代码目录
+ BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下
+ Parameter []Parameter `json:"parameter"`
+ DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
+ EngineID int64 `json:"engine_id"`
+ LogUrl string `json:"log_url"`
+ CreateVersion bool `json:"create_version"`
+ Flavor Flavor `json:"flavor"`
+ PoolID string `json:"pool_id"`
+ UserImageUrl string `json:"user_image_url"`
+ UserCommand string `json:"user_command"`
+}
+
type CreateTrainJobVersionParams struct {
Description string `json:"job_desc"`
Config TrainJobVersionConfig `json:"config"`
@@ -2024,7 +2045,7 @@ func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) {
}
func GetModelSafetyCountByUserID(userID int64) (int, error) {
- count, err := x.In("status", JobWaiting, JobRunning,ModelArtsTrainJobInit,ModelArtsTrainJobImageCreating,ModelArtsTrainJobSubmitTrying,ModelArtsTrainJobScaling,ModelArtsTrainJobCheckInit,ModelArtsTrainJobCheckRunning,ModelArtsTrainJobCheckRunningCompleted).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain))
+ count, err := x.In("status", JobWaiting, JobRunning, ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain))
return int(count), err
}
@@ -2260,9 +2281,9 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er
}
sess.Limit(opts.PageSize, start)
}
- sess.OrderBy("cloudbrain.created_unix DESC")
+ // sess.OrderBy("cloudbrain.created_unix DESC")
cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum)
- if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond).
+ if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond).
Find(&cloudbrains); err != nil {
return nil, 0, fmt.Errorf("Find: %v", err)
}
diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go
index c32e4b0fd..49a4d603e 100644
--- a/models/cloudbrain_spec.go
+++ b/models/cloudbrain_spec.go
@@ -9,7 +9,7 @@ type CloudbrainSpec struct {
SpecId int64 `xorm:"index"`
SourceSpecId string
AccCardsNum int
- AccCardType string
+ AccCardType string `xorm:"index"`
CpuCores int
MemGiB float32
GPUMemGiB float32
@@ -19,7 +19,7 @@ type CloudbrainSpec struct {
QueueId int64
QueueCode string
Cluster string
- AiCenterCode string
+ AiCenterCode string `xorm:"index"`
AiCenterName string
IsExclusive bool
ExclusiveOrg string
diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go
index 48df111a0..19e55fb6d 100644
--- a/models/cloudbrain_static.go
+++ b/models/cloudbrain_static.go
@@ -1,6 +1,7 @@
package models
import (
+ "fmt"
"strconv"
"time"
@@ -38,6 +39,60 @@ type TaskDetail struct {
Spec *Specification `json:"Spec"`
}
+type CloudbrainDurationStatistic struct {
+ ID int64 `xorm:"pk autoincr"`
+ Cluster string
+ AiCenterCode string
+ AiCenterName string
+ ComputeResource string
+ AccCardType string
+
+ DateTime string
+ DayTime string
+ HourTime int
+ CardsUseDuration int
+ CardsTotalDuration int
+ CardsTotalNum int
+
+ DeletedUnix timeutil.TimeStamp `xorm:"deleted"`
+ CreatedUnix timeutil.TimeStamp `xorm:"created"`
+ UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
+}
+type DurationStatisticOptions struct {
+ BeginTime time.Time
+ EndTime time.Time
+ AiCenterCode string
+}
+
+type DurationRateStatistic struct {
+ AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"`
+ AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"`
+ UsageRate map[string]float64 `json:"UsageRate"`
+}
+type ResourceDetail struct {
+ QueueCode string
+ Cluster string `xorm:"notnull"`
+ AiCenterCode string
+ AiCenterName string
+ ComputeResource string
+ AccCardType string
+ CardsTotalNum int
+ IsAutomaticSync bool
+}
+
+type DateUsageStatistic struct {
+ Date string `json:"date"`
+ UsageDuration int `json:"usageDuration"`
+ TotalDuration int `json:"totalDuration"`
+ UsageRate float64 `json:"usageRate"`
+}
+
+type HourTimeStatistic struct {
+ HourTimeUsageDuration map[string]int `json:"hourTimeUsageDuration"`
+ HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"`
+ HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"`
+}
+
func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(distinct user_id) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
@@ -199,3 +254,121 @@ func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string
}
return dateHourMap, nil
}
+
+func GetCloudbrainRunning() ([]*CloudbrainInfo, error) {
+ sess := x.NewSession()
+ defer sess.Close()
+ var cond = builder.NewCond()
+ cond = cond.And(
+ builder.Eq{"cloudbrain.status": string(JobRunning)},
+ )
+ sess.OrderBy("cloudbrain.created_unix ASC")
+ cloudbrains := make([]*CloudbrainInfo, 0, 10)
+ if err := sess.Table(&Cloudbrain{}).Where(cond).
+ Find(&cloudbrains); err != nil {
+ log.Info("find error.")
+ }
+ return cloudbrains, nil
+}
+
+func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) {
+ sess := x.NewSession()
+ defer sess.Close()
+ var cond = builder.NewCond()
+ cond = cond.And(
+ builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}),
+ )
+ cond = cond.Or(
+ builder.Eq{"cloudbrain.status": string(JobRunning)},
+ )
+ sess.OrderBy("cloudbrain.created_unix ASC")
+ cloudbrains := make([]*CloudbrainInfo, 0, 10)
+ if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond).
+ Find(&cloudbrains); err != nil {
+ log.Info("find error.")
+ }
+ return cloudbrains, nil
+}
+
+func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*CloudbrainSpec, error) {
+ sess := x.NewSession()
+ defer sess.Close()
+ var cond = builder.NewCond()
+ cond = cond.And(
+ builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}),
+ )
+ cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10)
+ if err := sess.Table(&CloudbrainSpec{}).Where(cond).
+ Find(&cloudbrainSpecs); err != nil {
+ log.Info("find error.")
+ }
+ return cloudbrainSpecs, nil
+}
+
+func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) {
+ return xStatistic.Insert(cloudbrainDurationStatistic)
+}
+
+func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string) error {
+ sess := xStatistic.NewSession()
+ defer sess.Close()
+ if err := sess.Begin(); err != nil {
+ return fmt.Errorf("Begin: %v", err)
+ }
+
+ if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ?", date, hour, aiCenterCode, accCardType).Delete(&CloudbrainDurationStatistic{}); err != nil {
+ return fmt.Errorf("Delete: %v", err)
+ }
+
+ if err := sess.Commit(); err != nil {
+ sess.Close()
+ return fmt.Errorf("Commit: %v", err)
+ }
+
+ sess.Close()
+ return nil
+}
+
+func GetCanUseCardInfo() ([]*ResourceQueue, error) {
+ sess := x.NewSession()
+ defer sess.Close()
+ sess.OrderBy("resource_queue.id ASC")
+ ResourceQueues := make([]*ResourceQueue, 0, 10)
+ if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil {
+ log.Info("find error.")
+ }
+ return ResourceQueues, nil
+}
+
+func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) {
+ sess := xStatistic.NewSession()
+ defer sess.Close()
+ var cond = builder.NewCond()
+ if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 {
+ cond = cond.And(
+ builder.And(builder.Gte{"cloudbrain_duration_statistic.created_unix": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_unix": opts.EndTime.Unix()}),
+ )
+ }
+ if opts.AiCenterCode != "" {
+ cond = cond.And(
+ builder.Eq{"cloudbrain_duration_statistic.ai_center_code": opts.AiCenterCode},
+ )
+ }
+ CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10)
+ if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond).
+ Find(&CloudbrainDurationStatistics); err != nil {
+ log.Info("find error.")
+ }
+ return CloudbrainDurationStatistics, nil
+}
+
+func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) {
+ sess := xStatistic.NewSession()
+ defer sess.Close()
+ sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1")
+ CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0)
+ if err := sess.Table(&CloudbrainDurationStatistic{}).Find(&CloudbrainDurationStatistics); err != nil {
+ log.Info("find error.")
+ }
+ return CloudbrainDurationStatistics, nil
+}
diff --git a/models/models.go b/models/models.go
index ff64bfad2..b2b7a6472 100755
--- a/models/models.go
+++ b/models/models.go
@@ -183,6 +183,7 @@ func init() {
new(UserMetrics),
new(UserAnalysisPara),
new(Invitation),
+ new(CloudbrainDurationStatistic),
)
gonicNames := []string{"SSL", "UID"}
diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go
index 8dbc8d1ed..aac4f87c3 100755
--- a/modules/cron/tasks_basic.go
+++ b/modules/cron/tasks_basic.go
@@ -266,6 +266,17 @@ func registerSyncModelArtsTempJobs() {
})
}
+func registerHandleCloudbrainDurationStatistic() {
+ RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{
+ Enabled: true,
+ RunAtStart: false,
+ Schedule: "1 0 * * * ?",
+ }, func(ctx context.Context, _ *models.User, _ Config) error {
+ repo.CloudbrainDurationStatisticHour()
+ return nil
+ })
+}
+
func initBasicTasks() {
registerUpdateMirrorTask()
registerRepoHealthCheck()
@@ -293,4 +304,5 @@ func initBasicTasks() {
registerCloudbrainPointDeductTask()
registerHandleModelSafetyTask()
+ registerHandleCloudbrainDurationStatistic()
}
diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go
index 06521993e..567f6d620 100755
--- a/modules/modelarts/modelarts.go
+++ b/modules/modelarts/modelarts.go
@@ -143,6 +143,8 @@ type GenerateInferenceJobReq struct {
Spec *models.Specification
DatasetName string
JobType string
+ UserImageUrl string
+ UserCommand string
}
type VersionInfo struct {
@@ -682,26 +684,51 @@ func GetOutputPathByCount(TotalVersionCount int) (VersionOutputPath string) {
func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (err error) {
createTime := timeutil.TimeStampNow()
- jobResult, err := createInferenceJob(models.CreateInferenceJobParams{
- JobName: req.JobName,
- Description: req.Description,
- InfConfig: models.InfConfig{
- WorkServerNum: req.WorkServerNumber,
- AppUrl: req.CodeObsPath,
- BootFileUrl: req.BootFileUrl,
- DataUrl: req.DataUrl,
- EngineID: req.EngineID,
- // TrainUrl: req.TrainUrl,
- LogUrl: req.LogUrl,
- PoolID: req.PoolID,
- CreateVersion: true,
- Flavor: models.Flavor{
- Code: req.Spec.SourceSpecId,
+ var jobResult *models.CreateTrainJobResult
+ var createErr error
+ if req.EngineID < 0 {
+ jobResult, createErr = createInferenceJobUserImage(models.CreateInfUserImageParams{
+ JobName: req.JobName,
+ Description: req.Description,
+ Config: models.InfUserImageConfig{
+ WorkServerNum: req.WorkServerNumber,
+ AppUrl: req.CodeObsPath,
+ BootFileUrl: req.BootFileUrl,
+ DataUrl: req.DataUrl,
+ // TrainUrl: req.TrainUrl,
+ LogUrl: req.LogUrl,
+ PoolID: req.PoolID,
+ CreateVersion: true,
+ Flavor: models.Flavor{
+ Code: req.Spec.SourceSpecId,
+ },
+ Parameter: req.Parameters,
+ UserImageUrl: req.UserImageUrl,
+ UserCommand: req.UserCommand,
},
- Parameter: req.Parameters,
- },
- })
- if err != nil {
+ })
+ } else {
+ jobResult, createErr = createInferenceJob(models.CreateInferenceJobParams{
+ JobName: req.JobName,
+ Description: req.Description,
+ InfConfig: models.InfConfig{
+ WorkServerNum: req.WorkServerNumber,
+ AppUrl: req.CodeObsPath,
+ BootFileUrl: req.BootFileUrl,
+ DataUrl: req.DataUrl,
+ EngineID: req.EngineID,
+ // TrainUrl: req.TrainUrl,
+ LogUrl: req.LogUrl,
+ PoolID: req.PoolID,
+ CreateVersion: true,
+ Flavor: models.Flavor{
+ Code: req.Spec.SourceSpecId,
+ },
+ Parameter: req.Parameters,
+ },
+ })
+ }
+ if createErr != nil {
log.Error("createInferenceJob failed: %v", err.Error())
if strings.HasPrefix(err.Error(), UnknownErrorPrefix) {
log.Info("(%s)unknown error, set temp status", req.DisplayJobName)
diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go
index fd1c467f3..c38300606 100755
--- a/modules/modelarts/resty.go
+++ b/modules/modelarts/resty.go
@@ -1197,6 +1197,66 @@ sendjob:
return &result, nil
}
+func createInferenceJobUserImage(createJobParams models.CreateInfUserImageParams) (*models.CreateTrainJobResult, error) {
+ checkSetting()
+ client := getRestyClient()
+ var result models.CreateTrainJobResult
+
+ retry := 0
+
+sendjob:
+ res, err := client.R().
+ SetHeader("Content-Type", "application/json").
+ SetAuthToken(TOKEN).
+ SetBody(createJobParams).
+ SetResult(&result).
+ Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob)
+
+ if err != nil {
+ return nil, fmt.Errorf("resty create train-job: %s", err)
+ }
+
+ req, _ := json.Marshal(createJobParams)
+ log.Info("%s", req)
+
+ if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
+ retry++
+ _ = getToken()
+ goto sendjob
+ }
+
+ if res.StatusCode() != http.StatusOK {
+ var temp models.ErrorResult
+ if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
+ log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
+ return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
+ }
+ log.Error("createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+ bootFileErrorMsg := "Invalid OBS path '" + createJobParams.Config.BootFileUrl + "'."
+ dataSetErrorMsg := "Invalid OBS path '" + createJobParams.Config.DataUrl + "'."
+ if temp.ErrorMsg == bootFileErrorMsg {
+ log.Error("启动文件错误！createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+ return &result, fmt.Errorf("启动文件错误！")
+ }
+ if temp.ErrorMsg == dataSetErrorMsg {
+ log.Error("数据集错误！createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+ return &result, fmt.Errorf("数据集错误！")
+ }
+ if res.StatusCode() == http.StatusBadGateway {
+ return &result, fmt.Errorf(UnknownErrorPrefix+"createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+ } else {
+ return &result, fmt.Errorf("createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+ }
+ }
+
+ if !result.IsSuccess {
+ log.Error("createInferenceJobUserImage failed(%s): %s", result.ErrorCode, result.ErrorMsg)
+ return &result, fmt.Errorf("createInferenceJobUserImage failed(%s): %s", result.ErrorCode, result.ErrorMsg)
+ }
+
+ return &result, nil
+}
+
func createNotebook2(createJobParams models.CreateNotebook2Params) (*models.CreateNotebookResult, error) {
checkSetting()
client := getRestyClient()
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 4e9e74303..052cdd3c5 100755
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -702,8 +702,12 @@ var (
GPU_PYTORCH_IMAGE string
GpuQueue string
GPU_TENSORFLOW_IMAGE string
+ GPU_PADDLE_IMAGE string
+ GPU_MXNET_IMAGE string
NPU_MINDSPORE_16_IMAGE string
PytorchOnnxBootFile string
+ PaddleOnnxBootFile string
+ MXnetOnnxBootFile string
PytorchTrTBootFile string
MindsporeBootFile string
TensorFlowNpuBootFile string
@@ -1597,6 +1601,10 @@ func getModelConvertConfig() {
ModelConvert.NPU_PoolID = sec.Key("NPU_PoolID").MustString("pool7908321a")
ModelConvert.NPU_MINDSPORE_IMAGE_ID = sec.Key("NPU_MINDSPORE_IMAGE_ID").MustInt(121)
ModelConvert.NPU_TENSORFLOW_IMAGE_ID = sec.Key("NPU_TENSORFLOW_IMAGE_ID").MustInt(35)
+ ModelConvert.GPU_PADDLE_IMAGE = sec.Key("GPU_PADDLE_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:paddle2.3.0_gpu_cuda11.2_cudnn8")
+ ModelConvert.GPU_MXNET_IMAGE = sec.Key("GPU_MXNET_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:mxnet191cu_cuda102_py37")
+ ModelConvert.PaddleOnnxBootFile = sec.Key("PaddleOnnxBootFile").MustString("convert_paddle.py")
+ ModelConvert.MXnetOnnxBootFile = sec.Key("MXnetOnnxBootFile").MustString("convert_mxnet.py")
}
func getModelAppConfig() {
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index ba75009b1..f9ed861fa 100755
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -1218,7 +1218,7 @@ cloudbrain.benchmark.evaluate_test=Test Script
cloudbrain.benchmark.types={"type":[{"id":1,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=detection","first":"Target detection","second":[{"id":1,"value":"None","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"yangzhx","repo_name":"detection_benchmark_script"}]},{"id":2,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=reid","first":"Target re-identification","second":[{"id":1,"value":"Vehicle re-identification","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"},{"id":2,"value":"Image-based person re-identification","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"}]},{"id":3,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=tracking","first":"Multi-target tracking","second":[{"id":1,"value":"None","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"lix07","repo_name":"MOT_benchmark_script"}]}]}
cloudbrain.morethanonejob=You already have a running or waiting task, create it after that task is over.
cloudbrain.morethanonejob1=You have created a similar task that is waiting or running, please wait for the task to finish before creating it.
-cloudbrain.morethanonejob2=You can view all your Cloud Brain tasks in Personal Center > Cloud Brain Tasks .
+cloudbrain.morethanonejob2=You can view all your Cloud Brain tasks in Personal Center > Cloud Brain Tasks .
modelarts.infer_job_model = Model
modelarts.infer_job_model_file = Model File
@@ -1228,6 +1228,9 @@ modelarts.infer_job.select_model = Select Model
modelarts.infer_job.boot_file_helper=The startup file is the entry file for your program execution and must end in.py.Such as inference.py, main.py, example/inference.py, case/main.py.
modelarts.infer_job.tooltip = The model has been deleted and cannot be viewed.
modelarts.download_log=Download log file
+modelarts.log_file = Log file
+modelarts.fullscreen_log_file = View in full screen
+modelarts.exit_full_screen = Exit fullscreen
modelarts.no_node_right = The value of 'Amount of Compute Node' is wrong, you have no right to use the current value of 'Amount of Compute Node'.
@@ -3219,7 +3222,7 @@ view_sample = View sample
inference_output_path_rule = The inference output path is stored in the run parameter result_url.
model_file_path_rule=The model file location is stored in the run parameter ckpt_url
model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt]
-model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb]
+model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json]
delete_task = Delete task
task_delete_confirm = Are you sure you want to delete this task? Once this task is deleted, it cannot be recovered.
operate_confirm = confirm
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index 5d5c54a1c..766bd3b85 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -1232,7 +1232,7 @@ cloudbrain.benchmark.types={"type":[{"id":1,"rank_link":"https://git.openi.org.c
cloudbrain.benchmark.model.types={"type":[{"id":1,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=detection","first":"目标检测","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"yangzhx","repo_name":"detection_benchmark_script"}]},{"id":2,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=reid","first":"目标重识别","second":[{"id":1,"value":"车辆重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"},{"id":2,"value":"基于图像的行人重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"}]},{"id":3,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=tracking","first":"多目标跟踪","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"lix07","repo_name":"MOT_benchmark_script"}]}]}
cloudbrain.morethanonejob=您已经创建了一个正在等待或运行中的同类任务，请等待任务结束再创建。
cloudbrain.morethanonejob1=您已经有 同类任务 正在等待或运行中，请等待任务结束再创建；
-cloudbrain.morethanonejob2=可以在 “个人中心 > 云脑任务 ” 查看您所有的云脑任务。
+cloudbrain.morethanonejob2=可以在 “个人中心 > 云脑任务 ” 查看您所有的云脑任务。
modelarts.infer_job_model = 模型名称
modelarts.infer_job_model_file = 模型文件
@@ -1242,6 +1242,9 @@ modelarts.infer_job.select_model = 选择模型
modelarts.infer_job.boot_file_helper=启动文件是您程序执行的入口文件，必须是以.py结尾的文件。比如inference.py、main.py、example/inference.py、case/main.py。
modelarts.infer_job.tooltip = 该模型已删除，无法查看。
modelarts.download_log=下载日志文件
+modelarts.log_file=日志文件
+modelarts.fullscreen_log_file=全屏查看
+modelarts.exit_full_screen=退出全屏
modelarts.no_node_right = 计算节点数的值配置错误，您没有权限使用当前配置的计算节点数。
@@ -3239,7 +3242,7 @@ view_sample = 查看样例
inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。
model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。
model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt]
-model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb]
+model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json]
delete_task = 删除任务
task_delete_confirm = 你确认删除该任务么？此任务一旦删除不可恢复。
operate_confirm = 确定操作
diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index 8e1d725ed..c464f252c 100755
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -599,6 +599,11 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData)
m.Get("/waitting_top_data", repo.GetWaittingTop)
m.Get("/running_top_data", repo.GetRunningTop)
+
+ m.Get("/overview_resource", repo.GetCloudbrainResourceOverview)
+ m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic)
+ m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail)
+ m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest)
})
}, operationReq)
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 37a9e0147..a8b9bcbac 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -578,6 +578,7 @@ func CloudbrainGetLog(ctx *context.APIContext) {
endLine += 1
}
}
+
result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)
if result == nil {
log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])
@@ -724,7 +725,6 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s
line, error := r.ReadString('\n')
if error == io.EOF {
if i >= startLine {
- fileEndLine = i
re = re + line
count++
}
diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go
index 935006476..fdc8d64bb 100755
--- a/routers/api/v1/repo/cloudbrain_dashboard.go
+++ b/routers/api/v1/repo/cloudbrain_dashboard.go
@@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
"net/url"
+ "strconv"
"strings"
"time"
@@ -120,9 +121,6 @@ func GetOverviewDuration(ctx *context.Context) {
recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix
now := time.Now()
endTime := now
- page := 1
- pagesize := 1000
- count := pagesize
worker_server_num := 1
cardNum := 1
durationAllSum := int64(0)
@@ -138,54 +136,46 @@ func GetOverviewDuration(ctx *context.Context) {
c2NetDuration := int64(0)
cDCenterDuration := int64(0)
- for count == pagesize && count != 0 {
- cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{
- ListOptions: models.ListOptions{
- Page: page,
- PageSize: pagesize,
- },
- Type: models.TypeCloudBrainAll,
- BeginTimeUnix: int64(recordBeginTime),
- EndTimeUnix: endTime.Unix(),
- })
- if err != nil {
- ctx.ServerError("Get cloudbrains failed:", err)
- return
- }
- models.LoadSpecs4CloudbrainInfo(cloudbrains)
-
- for _, cloudbrain := range cloudbrains {
- if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
- worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber
- } else {
- worker_server_num = 1
- }
- if cloudbrain.Cloudbrain.Spec == nil {
- cardNum = 1
- } else {
- cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
- }
- duration := cloudbrain.Duration
- durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum)
- if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
- cloudBrainOneDuration += duration
- cloudBrainOneCardDuSum += durationSum
- } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
- cloudBrainTwoDuration += duration
- cloudBrainTwoCardDuSum += durationSum
- } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
- c2NetDuration += duration
- c2NetCardDuSum += durationSum
- } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
- cDCenterDuration += duration
- cDNetCardDuSum += durationSum
- }
+ cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{
+ Type: models.TypeCloudBrainAll,
+ BeginTimeUnix: int64(recordBeginTime),
+ EndTimeUnix: endTime.Unix(),
+ })
+ if err != nil {
+ ctx.ServerError("Get cloudbrains failed:", err)
+ return
+ }
+ models.LoadSpecs4CloudbrainInfo(cloudbrains)
- durationAllSum += duration
- cardDuSum += durationSum
- count = len(cloudbrains)
- page += 1
+ for _, cloudbrain := range cloudbrains {
+ if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
+ worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber
+ } else {
+ worker_server_num = 1
}
+ if cloudbrain.Cloudbrain.Spec == nil {
+ cardNum = 1
+ } else {
+ cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
+ }
+ duration := cloudbrain.Duration
+ durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum)
+ if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
+ cloudBrainOneDuration += duration
+ cloudBrainOneCardDuSum += durationSum
+ } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
+ cloudBrainTwoDuration += duration
+ cloudBrainTwoCardDuSum += durationSum
+ } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
+ c2NetDuration += duration
+ c2NetCardDuSum += durationSum
+ } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
+ cDCenterDuration += duration
+ cDNetCardDuSum += durationSum
+ }
+
+ durationAllSum += duration
+ cardDuSum += durationSum
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"cloudBrainOneCardDuSum": cloudBrainOneCardDuSum,
@@ -532,6 +522,21 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int
}
+func getPageDateCloudbrainDuration(dateUsageStatistic []models.DateUsageStatistic, page int, pagesize int) []models.DateUsageStatistic {
+ begin := (page - 1) * pagesize
+ end := (page) * pagesize
+
+ if begin > len(dateUsageStatistic)-1 {
+ return nil
+ }
+ if end > len(dateUsageStatistic)-1 {
+ return dateUsageStatistic[begin:]
+ } else {
+ return dateUsageStatistic[begin:end]
+ }
+
+}
+
func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
queryType := ctx.QueryTrim("type")
beginTimeStr := ctx.QueryTrim("beginTime")
@@ -545,7 +550,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0)
beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime)
if err != nil {
- log.Error("Parameter is wrong", err)
+ log.Error("getCloudbrainTimePeroid error:", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
return
}
@@ -1403,3 +1408,424 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t
return beginTime, endTime, nil
}
+
+func GetCloudbrainResourceOverview(ctx *context.Context) {
+ recordCloudbrainDuration, err := models.GetDurationRecordBeginTime()
+ if err != nil {
+ log.Error("Can not get GetDurationRecordBeginTime", err)
+ return
+ }
+ recordBeginTime := recordCloudbrainDuration[0].CreatedUnix
+ recordUpdateTime := time.Now().Unix()
+ resourceQueues, err := models.GetCanUseCardInfo()
+ if err != nil {
+ log.Info("GetCanUseCardInfo err: %v", err)
+ return
+ }
+ OpenIResourceDetail := []models.ResourceDetail{}
+ C2NetResourceDetail := []models.ResourceDetail{}
+ for _, resourceQueue := range resourceQueues {
+ if resourceQueue.Cluster == models.OpenICluster {
+ var resourceDetail models.ResourceDetail
+ resourceDetail.QueueCode = resourceQueue.QueueCode
+ resourceDetail.Cluster = resourceQueue.Cluster
+ resourceDetail.AiCenterCode = resourceQueue.AiCenterCode
+ resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode
+ resourceDetail.ComputeResource = resourceQueue.ComputeResource
+ resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")"
+ resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum
+ resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync
+ OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail)
+ }
+ if resourceQueue.Cluster == models.C2NetCluster {
+ var resourceDetail models.ResourceDetail
+ resourceDetail.QueueCode = resourceQueue.QueueCode
+ resourceDetail.Cluster = resourceQueue.Cluster
+ resourceDetail.AiCenterCode = resourceQueue.AiCenterCode
+ resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode
+ resourceDetail.ComputeResource = resourceQueue.ComputeResource
+ resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")"
+ resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum
+ resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync
+ C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail)
+ }
+ }
+ openIResourceNum := make(map[string]map[string]int)
+
+ for _, openIResourceDetail := range OpenIResourceDetail {
+ if _, ok := openIResourceNum[openIResourceDetail.AiCenterName]; !ok {
+ openIResourceNum[openIResourceDetail.AiCenterName] = make(map[string]int)
+ }
+ if _, ok := openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType]; !ok {
+ openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum
+ } else {
+ openIResourceNum[openIResourceDetail.AiCenterName][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum
+ }
+ }
+
+ c2NetResourceNum := make(map[string]map[string]int)
+ for _, c2NetResourceDetail := range C2NetResourceDetail {
+ if _, ok := c2NetResourceNum[c2NetResourceDetail.AiCenterName]; !ok {
+ c2NetResourceNum[c2NetResourceDetail.AiCenterName] = make(map[string]int)
+ }
+ if _, ok := c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType]; !ok {
+ c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType] = c2NetResourceDetail.CardsTotalNum
+ } else {
+ c2NetResourceNum[c2NetResourceDetail.AiCenterName][c2NetResourceDetail.AccCardType] += c2NetResourceDetail.CardsTotalNum
+ }
+
+ }
+
+ ctx.JSON(http.StatusOK, map[string]interface{}{
+ "openI": openIResourceNum,
+ "c2Net": c2NetResourceNum,
+ "recordUpdateTime": recordUpdateTime,
+ "recordBeginTime": recordBeginTime,
+ })
+}
+
+func GetCloudbrainResourceUsageDetail(ctx *context.Context) {
+ aiCenterCode := ctx.QueryTrim("aiCenterCode")
+ if aiCenterCode == "" {
+ aiCenterCode = models.AICenterOfCloudBrainOne
+ }
+ beginTime, endTime := getBeginAndEndTime(ctx)
+ dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode)
+ if err != nil {
+ log.Error("Can not query dayCloudbrainDuration.", err)
+ return
+ }
+ hourCloudbrainDuration, err := getHourCloudbrainDuration(beginTime, endTime, aiCenterCode)
+ if err != nil {
+ log.Error("Can not query hourCloudbrainDuration.", err)
+ return
+ }
+ page := ctx.QueryInt("page")
+ if page <= 0 {
+ page = 1
+ }
+ pagesize := ctx.QueryInt("pagesize")
+ if pagesize <= 0 {
+ pagesize = 36500
+ }
+ pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize)
+ ctx.JSON(http.StatusOK, map[string]interface{}{
+ "totalCount": count,
+ "pageDateCloudbrainDuration": pageDateCloudbrainDuration,
+ "hourCloudbrainDuration": hourCloudbrainDuration,
+ })
+}
+
+func GetDurationRateStatistic(ctx *context.Context) {
+ beginTime, endTime := getBeginAndEndTime(ctx)
+ OpenIDurationRate, C2NetDurationRate, totalUsageRate := getDurationStatistic(beginTime, endTime)
+
+ ctx.JSON(http.StatusOK, map[string]interface{}{
+ "openIDurationRate": OpenIDurationRate,
+ "c2NetDurationRate": C2NetDurationRate,
+ "totalUsageRate": totalUsageRate,
+ })
+
+}
+
+func CloudbrainDurationStatisticForTest(ctx *context.Context) {
+ repo.CloudbrainDurationStatisticHour()
+ ctx.JSON(http.StatusOK, map[string]interface{}{
+ "message": 0,
+ })
+}
+
+func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) {
+ queryType := ctx.QueryTrim("type")
+ now := time.Now()
+ beginTimeStr := ctx.QueryTrim("beginTime")
+ endTimeStr := ctx.QueryTrim("endTime")
+
+ var beginTime time.Time
+ var endTime time.Time
+ var err error
+ if queryType != "" {
+ if queryType == "all" {
+ recordCloudbrainDuration, err := models.GetDurationRecordBeginTime()
+ if err != nil {
+ log.Error("Can not get GetDurationRecordBeginTime", err)
+ ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
+ return beginTime, endTime
+ }
+ brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime()
+ beginTime = brainRecordBeginTime
+ endTime = now
+ } else if queryType == "today" {
+ beginTime = now.AddDate(0, 0, 0)
+ beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+ endTime = now
+
+ } else if queryType == "yesterday" {
+ beginTime = now.AddDate(0, 0, -1)
+ beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+ endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+ } else if queryType == "last_7day" {
+ beginTime = now.AddDate(0, 0, -6)
+ beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+ endTime = now
+ } else if queryType == "last_30day" {
+ beginTime = now.AddDate(0, 0, -29)
+ beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+ endTime = now
+ } else if queryType == "current_month" {
+ endTime = now
+ beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location())
+
+ } else if queryType == "current_year" {
+ endTime = now
+ beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location())
+ } else if queryType == "last_month" {
+ lastMonthTime := now.AddDate(0, -1, 0)
+ beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location())
+ endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
+ }
+
+ } else {
+ if beginTimeStr == "" || endTimeStr == "" {
+ //如果查询类型和开始时间结束时间都未设置，按queryType=all处理
+ recordCloudbrainDuration, err := models.GetDurationRecordBeginTime()
+ if err != nil {
+ log.Error("Can not get recordCloudbrain", err)
+ ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
+ return beginTime, endTime
+ }
+ brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime()
+ beginTime = brainRecordBeginTime
+ endTime = now
+ } else {
+ beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local)
+ if err != nil {
+ log.Error("Can not ParseInLocation.", err)
+ ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error"))
+ return beginTime, endTime
+ }
+ endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local)
+ if err != nil {
+ log.Error("Can not ParseInLocation.", err)
+ ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error"))
+ return beginTime, endTime
+ }
+ if endTime.After(time.Now()) {
+ endTime = time.Now()
+ }
+ }
+
+ }
+ return beginTime, endTime
+}
+
+func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (int, int, float64) {
+ totalDuration := int(0)
+ usageDuration := int(0)
+ usageRate := float64(0)
+
+ for _, cloudbrainStatistic := range cloudbrainStatistics {
+ if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() {
+ totalDuration += cloudbrainStatistic.CardsTotalDuration
+ usageDuration += cloudbrainStatistic.CardsUseDuration
+ }
+ }
+ if totalDuration == 0 || usageDuration == 0 {
+ usageRate = 0
+ } else {
+ usageRate = float64(usageDuration) / float64(totalDuration)
+ }
+
+ return totalDuration, usageDuration, usageRate
+}
+
+func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float64) {
+ OpenITotalDuration := make(map[string]int)
+ OpenIUsageDuration := make(map[string]int)
+ OpenIUsageRate := make(map[string]float64)
+
+ C2NetTotalDuration := make(map[string]int)
+ C2NetUsageDuration := make(map[string]int)
+ OpenIDurationRate := models.DurationRateStatistic{}
+ C2NetDurationRate := models.DurationRateStatistic{}
+ cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
+ BeginTime: beginTime,
+ EndTime: endTime,
+ })
+ if err != nil {
+ log.Error("GetCardDurationStatistics error:", err)
+ return OpenIDurationRate, C2NetDurationRate, 0
+ }
+ for _, cloudbrainStatistic := range cardDurationStatistics {
+ if cloudbrainStatistic.Cluster == models.OpenICluster {
+ if _, ok := OpenITotalDuration[cloudbrainStatistic.AiCenterName]; !ok {
+ OpenITotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration
+ } else {
+ OpenITotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration
+ }
+ if _, ok := OpenIUsageDuration[cloudbrainStatistic.AiCenterName]; !ok {
+ OpenIUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration
+ } else {
+ OpenIUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration
+ }
+ }
+ if cloudbrainStatistic.Cluster == models.C2NetCluster {
+ if _, ok := C2NetTotalDuration[cloudbrainStatistic.AiCenterName]; !ok {
+ C2NetTotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration
+ } else {
+ C2NetTotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration
+ }
+ if _, ok := C2NetUsageDuration[cloudbrainStatistic.AiCenterName]; !ok {
+ C2NetUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration
+ } else {
+ C2NetUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration
+ }
+ }
+ }
+ ResourceAiCenterRes, err := models.GetResourceAiCenters()
+ if err != nil {
+ log.Error("Can not get ResourceAiCenterRes.", err)
+ return OpenIDurationRate, C2NetDurationRate, 0
+ }
+ for _, v := range ResourceAiCenterRes {
+ if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) {
+ if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok {
+ OpenIUsageDuration[v.AiCenterName] = 0
+ }
+ if _, ok := OpenITotalDuration[v.AiCenterName]; !ok {
+ OpenITotalDuration[v.AiCenterName] = 0
+ }
+ } else {
+ if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok {
+ C2NetUsageDuration[v.AiCenterName] = 0
+ }
+ }
+ }
+ totalCanUse := float64(0)
+ totalUse := float64(0)
+ totalUsageRate := float64(0)
+ for k, v := range OpenITotalDuration {
+ for i, j := range OpenIUsageDuration {
+ if k == i {
+ OpenIUsageRate[k] = float64(j) / float64(v)
+ }
+ }
+ }
+ for _, v := range OpenITotalDuration {
+ totalCanUse += float64(v)
+ }
+ for _, v := range OpenIUsageRate {
+ totalUse += float64(v)
+ }
+ if totalCanUse == 0 || totalUse == 0 {
+ totalUsageRate = 0
+ } else {
+ totalUsageRate = totalUse / totalCanUse
+ }
+
+ OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration
+ OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration
+ OpenIDurationRate.UsageRate = OpenIUsageRate
+ C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration
+ C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration
+ return OpenIDurationRate, C2NetDurationRate, totalUsageRate
+}
+
+func cutString(str string, lens int) string {
+ if len(str) < lens {
+ return str
+ }
+ return str[:lens]
+}
+
+func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) {
+ now := time.Now()
+ endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location())
+ if endTimeTemp.Equal(endTime) {
+ endTimeTemp = endTimeTemp.AddDate(0, 0, -1)
+ }
+ cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
+ BeginTime: beginTime,
+ EndTime: endTime,
+ AiCenterCode: aiCenterCode,
+ })
+ if err != nil {
+ log.Error("GetCardDurationStatistics error:", err)
+ return nil, 0, err
+ }
+
+ dayCloudbrainInfo := make([]models.DateUsageStatistic, 0)
+ count := 0
+ for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) {
+ TotalDuration, UsageDuration, UsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics)
+ dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateUsageStatistic{
+ Date: endTimeTemp.Format("2006/01/02"),
+ UsageDuration: UsageDuration,
+ TotalDuration: TotalDuration,
+ UsageRate: UsageRate,
+ })
+ endTime = endTimeTemp
+ endTimeTemp = endTimeTemp.AddDate(0, 0, -1)
+ if endTimeTemp.Before(beginTime) && beginTime.Before(endTime) {
+ endTimeTemp = beginTime
+ }
+ count += 1
+ }
+ return dayCloudbrainInfo, count, nil
+}
+
+func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) (models.HourTimeStatistic, error) {
+ hourTimeTotalDuration := make(map[string]int)
+ hourTimeUsageDuration := make(map[string]int)
+ hourTimeUsageRate := make(map[string]float64)
+ hourTimeStatistic := models.HourTimeStatistic{}
+
+ cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
+ BeginTime: beginTime,
+ EndTime: endTime,
+ })
+ if err != nil {
+ log.Error("GetCardDurationStatistics error:", err)
+ return hourTimeStatistic, err
+ }
+ for _, cloudbrainStatistic := range cardDurationStatistics {
+ if cloudbrainStatistic.AiCenterCode == aiCenterCode {
+ if _, ok := hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok {
+ hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration
+ } else {
+ hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration
+ }
+ if _, ok := hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok {
+ hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsUseDuration
+ } else {
+ hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsUseDuration
+ }
+ }
+ }
+ hourTimeList := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"}
+ for _, v := range hourTimeList {
+ if _, ok := hourTimeUsageDuration[v]; !ok {
+ hourTimeUsageDuration[v] = 0
+ }
+ if _, ok := hourTimeTotalDuration[v]; !ok {
+ hourTimeTotalDuration[v] = 0
+ }
+ }
+
+ for k, v := range hourTimeTotalDuration {
+ for i, j := range hourTimeUsageDuration {
+ if k == i {
+ if v == 0 || j == 0 {
+ hourTimeUsageRate[k] = 0
+ } else {
+ hourTimeUsageRate[k] = float64(j) / float64(v)
+ }
+ }
+ }
+ }
+
+ hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration
+ hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration
+ hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate
+ return hourTimeStatistic, nil
+}
diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go
index 9a5874956..bd6a01072 100644
--- a/routers/repo/ai_model_convert.go
+++ b/routers/repo/ai_model_convert.go
@@ -29,7 +29,9 @@ const (
tplModelConvertInfo = "repo/modelmanage/convertshowinfo"
PYTORCH_ENGINE = 0
TENSORFLOW_ENGINE = 1
- MINDSPORE_ENGIN = 2
+ MINDSPORE_ENGINE = 2
+ PADDLE_ENGINE = 4
+ MXNET_ENGINE = 6
ModelMountPath = "/model"
CodeMountPath = "/code"
DataSetMountPath = "/dataset"
@@ -395,6 +397,20 @@ func createGpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context
deleteLocalDir(relatetiveModelPath)
dataActualPath = setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/dataset"
}
+ } else if modelConvert.SrcEngine == PADDLE_ENGINE {
+ IMAGE_URL = setting.ModelConvert.GPU_PADDLE_IMAGE
+ if modelConvert.DestFormat == CONVERT_FORMAT_ONNX {
+ command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PaddleOnnxBootFile)
+ } else {
+ return errors.New("Not support the format.")
+ }
+ } else if modelConvert.SrcEngine == MXNET_ENGINE {
+ IMAGE_URL = setting.ModelConvert.GPU_MXNET_IMAGE
+ if modelConvert.DestFormat == CONVERT_FORMAT_ONNX {
+ command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.MXnetOnnxBootFile)
+ } else {
+ return errors.New("Not support the format.")
+ }
}
log.Info("dataActualPath=" + dataActualPath)
diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go
new file mode 100644
index 000000000..3814c2daf
--- /dev/null
+++ b/routers/repo/cloudbrain_statistic.go
@@ -0,0 +1,169 @@
+package repo
+
+import (
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/timeutil"
+)
+
+func CloudbrainDurationStatisticHour() {
+
+ dateTime := time.Now().Format("2006-01-02 15:04:05")
+ dayTime := time.Now().Format("2006-01-02")
+ now := time.Now()
+
+ currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
+
+ m, _ := time.ParseDuration("-1h")
+ beginTime := currentTime.Add(m).Unix()
+ endTime := currentTime.Unix()
+ hourTime := currentTime.Add(m).Hour()
+
+ ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime)
+ if err != nil {
+ log.Info("GetCloudbrainByTime err: %v", err)
+ return
+ }
+ specMap := make(map[string]*models.Specification)
+ models.LoadSpecs4CloudbrainInfo(ciTasks)
+ for _, cloudbrain := range ciTasks {
+ if _, ok := specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
+ if cloudbrain.Cloudbrain.Spec != nil {
+ specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec
+ }
+ }
+ }
+
+ cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime)
+
+ resourceQueues, err := models.GetCanUseCardInfo()
+ if err != nil {
+ log.Info("GetCanUseCardInfo err: %v", err)
+ return
+ }
+ cardsTotalDurationMap := make(map[string]int)
+ for _, resourceQueue := range resourceQueues {
+ cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterName+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
+ }
+
+ for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo {
+ for cardType, cardDuration := range CardTypeInfo {
+ spec := specMap[centerCode+"/"+cardType]
+ if spec != nil {
+ if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil {
+ log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
+ return
+ }
+ if _, ok := cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource]; !ok {
+ cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource] = 0
+ }
+ cloudbrainDurationStat := models.CloudbrainDurationStatistic{
+ DateTime: dateTime,
+ DayTime: dayTime,
+ HourTime: hourTime,
+ Cluster: spec.Cluster,
+ AiCenterName: spec.AiCenterName,
+ AiCenterCode: centerCode,
+ AccCardType: cardType,
+ ComputeResource: spec.ComputeResource,
+ CardsUseDuration: cardDuration,
+ CardsTotalDuration: cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource],
+ CreatedUnix: timeutil.TimeStampNow(),
+ }
+ if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
+ log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
+ }
+ delete(cardsTotalDurationMap, spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource)
+ }
+ }
+ }
+
+ for key, cardsTotalDuration := range cardsTotalDurationMap {
+ if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[2], strings.Split(key, "/")[3]); err != nil {
+ log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
+ return
+ }
+ cloudbrainDurationStat := models.CloudbrainDurationStatistic{
+ DateTime: dateTime,
+ DayTime: dayTime,
+ HourTime: hourTime,
+ Cluster: strings.Split(key, "/")[0],
+ AiCenterName: strings.Split(key, "/")[1],
+ AiCenterCode: strings.Split(key, "/")[2],
+ AccCardType: strings.Split(key, "/")[3],
+ ComputeResource: strings.Split(key, "/")[4],
+ CardsUseDuration: 0,
+ CardsTotalDuration: cardsTotalDuration,
+ CreatedUnix: timeutil.TimeStampNow(),
+ }
+ if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
+ log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
+ }
+ }
+
+ log.Info("finish summary cloudbrainDurationStat")
+}
+
+func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int {
+ var WorkServerNumber int
+ var AccCardsNum int
+ cloudBrainCenterCodeAndCardType := make(map[string]map[string]int)
+ for _, cloudbrain := range ciTasks {
+
+ if cloudbrain.Cloudbrain.StartTime == 0 {
+ cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
+ }
+ if cloudbrain.Cloudbrain.EndTime == 0 {
+ cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix
+ }
+ if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
+ WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber
+ } else {
+ WorkServerNumber = 1
+ }
+ if cloudbrain.Cloudbrain.Spec == nil {
+ AccCardsNum = 1
+ } else {
+ AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
+ }
+ if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode]; !ok {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode] = make(map[string]int)
+ }
+
+ if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) {
+ if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
+ if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
+ } else {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
+ }
+ } else {
+ if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
+ } else {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
+ }
+ }
+ } else {
+ if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
+ if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
+ } else {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
+ }
+ } else {
+ if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
+ } else {
+ cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
+ }
+ }
+
+ }
+ }
+
+ return cloudBrainCenterCodeAndCardType
+}
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 07c1fcd3e..be59b0f3f 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -1312,6 +1312,36 @@ func getUserCommand(engineId int, req *modelarts.GenerateTrainJobReq) (string, s
return userCommand, userImageUrl
}
+func getInfJobUserCommand(engineId int, req *modelarts.GenerateInferenceJobReq) (string, string) {
+ userImageUrl := ""
+ userCommand := ""
+ if engineId < 0 {
+ tmpCodeObsPath := strings.Trim(req.CodeObsPath, "/")
+ tmpCodeObsPaths := strings.Split(tmpCodeObsPath, "/")
+ lastCodeDir := "code"
+ if len(tmpCodeObsPaths) > 0 {
+ lastCodeDir = tmpCodeObsPaths[len(tmpCodeObsPaths)-1]
+ }
+ userCommand = "/bin/bash /home/work/run_train.sh 's3://" + req.CodeObsPath + "' '" + lastCodeDir + "/" + req.BootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + req.DataUrl + "' --'train_url'='s3://" + req.TrainUrl + "'"
+ var versionInfos modelarts.VersionInfo
+ if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil {
+ log.Info("json parse err." + err.Error())
+ } else {
+ for _, engine := range versionInfos.Version {
+ if engine.ID == engineId {
+ userImageUrl = engine.Url
+ break
+ }
+ }
+ }
+ for _, param := range req.Parameters {
+ userCommand += " --'" + param.Label + "'='" + param.Value + "'"
+ }
+ return userCommand, userImageUrl
+ }
+ return userCommand, userImageUrl
+}
+
func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
ctx.Data["PageIsTrainJob"] = true
var jobID = ctx.Params(":jobid")
@@ -2171,6 +2201,10 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
JobType: string(models.JobTypeInference),
}
+ userCommand, userImageUrl := getInfJobUserCommand(engineID, req)
+ req.UserCommand = userCommand
+ req.UserImageUrl = userImageUrl
+
err = modelarts.GenerateInferenceJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error())
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index 4fd2246bd..fd8b274e6 100755
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -6,17 +6,18 @@ package routes
import (
"bytes"
- "code.gitea.io/gitea/routers/badge"
- "code.gitea.io/gitea/routers/reward/point"
- "code.gitea.io/gitea/routers/task"
- badge_service "code.gitea.io/gitea/services/badge"
- "code.gitea.io/gitea/services/reward"
"encoding/gob"
"net/http"
"path"
"text/template"
"time"
+ "code.gitea.io/gitea/routers/badge"
+ "code.gitea.io/gitea/routers/reward/point"
+ "code.gitea.io/gitea/routers/task"
+ badge_service "code.gitea.io/gitea/services/badge"
+ "code.gitea.io/gitea/services/reward"
+
"code.gitea.io/gitea/routers/modelapp"
"code.gitea.io/gitea/modules/slideimage"
diff --git a/templates/custom/max_log.tmpl b/templates/custom/max_log.tmpl
new file mode 100644
index 000000000..978274f4e
--- /dev/null
+++ b/templates/custom/max_log.tmpl
@@ -0,0 +1,40 @@
+
+
\ No newline at end of file
diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl
index 84d16d00c..de68a3241 100755
--- a/templates/repo/cloudbrain/benchmark/show.tmpl
+++ b/templates/repo/cloudbrain/benchmark/show.tmpl
@@ -524,14 +524,17 @@
+ {{template "custom/max_log" .}}
{{end}} {{template "base/paginate" .}}
diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index 546f16848..fe127b89d 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -535,14 +535,17 @@
-
+ {{template "custom/max_log" .}}
{{end}}
diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl
index a54989df4..ac08f5f5b 100644
--- a/templates/repo/cloudbrain/trainjob/show.tmpl
+++ b/templates/repo/cloudbrain/trainjob/show.tmpl
@@ -524,14 +524,17 @@
+ {{template "custom/max_log" .}}
{{end}} {{template "base/paginate" .}}
diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl
index 67c415488..0813cf477 100755
--- a/templates/repo/grampus/trainjob/show.tmpl
+++ b/templates/repo/grampus/trainjob/show.tmpl
@@ -520,14 +520,17 @@
+ {{template "custom/max_log" .}}
{{end}} {{template "base/paginate" .}}
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl
index 7d671ae4c..98240fbd4 100644
--- a/templates/repo/modelarts/inferencejob/show.tmpl
+++ b/templates/repo/modelarts/inferencejob/show.tmpl
@@ -460,14 +460,17 @@ td, th {
-
+
@@ -486,8 +489,12 @@ td, th {
-
@@ -510,12 +517,8 @@ td, th {
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow{{else if eq .SrcEngine 2}}MindSpore {{end}}
+ {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow {{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle {{else if eq .SrcEngine 6}}MXNet {{end}}
{{if eq .DestFormat 0}}ONNX {{else if eq .DestFormat 1}}TensorRT {{end}}
@@ -532,7 +532,7 @@
}
}
function isModel(filename){
- var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb"];
+ var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel",".pdiparams",".params",".json"];
for(var i =0; i
PyTorch";
html +="TensorFlow ";
html +="MindSpore ";
+ html +="PaddlePaddle ";
+ html +="MXNet ";
$('#SrcEngine').html(html);
srcEngineChanged();
}
diff --git a/templates/repo/modelmanage/convertshowinfo.tmpl b/templates/repo/modelmanage/convertshowinfo.tmpl
index 70aa301d5..f4db2237c 100644
--- a/templates/repo/modelmanage/convertshowinfo.tmpl
+++ b/templates/repo/modelmanage/convertshowinfo.tmpl
@@ -355,7 +355,7 @@ td, th {
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}Tensorflow{{else if eq .SrcEngine 2}}MindSpore {{end}}
+ {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}Tensorflow{{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle{{else if eq .SrcEngine 6}} MXNet{{end}}
diff --git a/templates/repo/modelsafety/show.tmpl b/templates/repo/modelsafety/show.tmpl
index 6954f6f6b..1f0d2d3aa 100644
--- a/templates/repo/modelsafety/show.tmpl
+++ b/templates/repo/modelsafety/show.tmpl
@@ -467,13 +467,17 @@
@@ -571,6 +579,7 @@
+ {{template "custom/max_log" .}}
{{template "base/paginate" .}}
diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index 3aafcc1a6..d3338523d 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -14,8 +14,8 @@ export default async function initCloudrainSow() {
};
}
- function logScroll(version_name, repoPath, ID) {
- let container = document.querySelector(`#log${version_name}`);
+ function logScroll(version_name, repoPath, ID, max = "", lines = 60) {
+ let container = document.querySelector(`#log${max}${version_name}`);
let scrollTop = container.scrollTop;
let scrollHeight = container.scrollHeight;
let clientHeight = container.clientHeight;
@@ -26,46 +26,90 @@ export default async function initCloudrainSow() {
parseInt(scrollTop) + clientHeight - 1 == scrollHeight) &&
scrollLeft === 0
) {
- let end_line = $(`#log${version_name} input[name=end_line]`).val();
+ let end_line = $(`#log${version_name} input[name=end_line${max}]`).val();
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css({
+ "background-color": "#fff",
+ display: "block",
+ });
$.get(
- `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${end_line}&lines=50&order=desc`,
+ `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${end_line}&lines=${lines}&order=desc`,
(data) => {
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
+ "display",
+ "none"
+ );
if (data.Lines == 0) {
- $(`.message${version_name} #header`).text("您已翻阅至日志底部");
- $(`.message${version_name}`).css("display", "block");
- setTimeout(function () {
- $(`.message${version_name}`).css("display", "none");
- }, 1000);
+ if (max) {
+ $("body").toast({
+ class: "info",
+ message: `您已翻阅至日志底部,请稍后再试！`,
+ });
+ } else {
+ $(`.message${version_name} #header`).text("您已翻阅至日志底部");
+ $(`.message${version_name}`).css("display", "block");
+ setTimeout(function () {
+ $(`.message${version_name}`).css("display", "none");
+ }, 1000);
+ }
} else {
if (end_line === data.EndLine) {
return;
} else {
- $(`#log${version_name} input[name=end_line]`).val(data.EndLine);
- $(`#log${version_name}`).append("" + data.Content);
+ $(`#log${version_name} input[name=end_line${max}]`).val(
+ data.EndLine
+ );
+ $(`#log${max}${version_name}`).append("" + data.Content);
}
}
}
).fail(function (err) {
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
+ "display",
+ "none"
+ );
console.log(err);
});
}
if (scrollTop == 0 && scrollLeft == 0) {
- let start_line = $(`#log${version_name} input[name=start_line]`).val();
+ let start_line = $(
+ `#log${version_name} input[name=start_line${max}]`
+ ).val();
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css({
+ "background-color": "#fff",
+ display: "block",
+ });
$.get(
- `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${start_line}&lines=50&order=asc`,
+ `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${start_line}&lines=${lines}&order=asc`,
(data) => {
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
+ "display",
+ "none"
+ );
if (data.Lines == 0) {
- $(`.message${version_name} #header`).text("您已翻阅至日志顶部");
- $(`.message${version_name}`).css("display", "block");
- setTimeout(function () {
- $(`.message${version_name}`).css("display", "none");
- }, 1000);
+ if (max) {
+ $("body").toast({
+ class: "info",
+ message: `您已翻阅至日志顶部,请稍后再试！`,
+ });
+ } else {
+ $(`.message${version_name} #header`).text("您已翻阅至日志顶部");
+ $(`.message${version_name}`).css("display", "block");
+ setTimeout(function () {
+ $(`.message${version_name}`).css("display", "none");
+ }, 1000);
+ }
} else {
- $(`#log${version_name} input[name=start_line]`).val(data.StartLine); //如果变动就改变所对应的值
- $(`#log${version_name}`).prepend("" + data.Content);
+ $(`#log${version_name} input[name=start_line${max}]`).val(
+ data.StartLine
+ ); //如果变动就改变所对应的值
+ $(`#log${max}${version_name}`).prepend("" + data.Content);
}
}
).fail(function (err) {
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
+ "display",
+ "none"
+ );
console.log(err);
});
}
@@ -96,51 +140,71 @@ export default async function initCloudrainSow() {
}, 1);
}
- $(".log_top").click(function () {
- // let logContentDom = document.querySelector('.log')
- // if(!logContentDom)
- // return
- // let version_name = $('.log_top').data('version')
+ function logTop(e) {
+ let max = e.currentTarget.getAttribute("data-max") || "";
+ let lines = !!max ? 100 : 60;
let version_name = $(this).data("version");
- let logContentDom = document.querySelector(`#log${version_name}`);
+ let logContentDom = document.querySelector(`#log${max}${version_name}`);
let ID = $(`#accordion${version_name}`).data("jobid");
let repoPath = $(`#accordion${version_name}`).data("repopath");
- $(`#log_file${version_name}`).siblings("pre").remove();
- $(`#log${version_name} .ui.inverted.active.dimmer`).css("display", "block");
+ $(`#log_file${max}${version_name}`).siblings("pre").remove();
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css({
+ "background-color": "#fff",
+ display: "block",
+ });
$.get(
- `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=50&order=asc`,
+ `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=${lines}&order=asc`,
(data) => {
- $(".ui.inverted.active.dimmer").css("display", "none");
- $(`#log${version_name} input[name=end_line]`).val(data.EndLine); //如果变动就改变所对应的值
- $(`#log${version_name} input[name=start_line]`).val(data.StartLine);
- $(`#log${version_name}`).prepend("" + data.Content);
- $(`.message${version_name} #header`).text("您已翻阅至日志顶部");
- $(`.message${version_name}`).css("display", "block");
- setTimeout(function () {
- $(`.message${version_name}`).css("display", "none");
- }, 1000);
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
+ "display",
+ "none"
+ );
+ $(`#log${version_name} input[name=end_line${max}]`).val(data.EndLine); //如果变动就改变所对应的值
+ $(`#log${version_name} input[name=start_line${max}]`).val(
+ data.StartLine
+ );
+ $(`#log${max}${version_name}`).prepend("" + data.Content);
+ if (data.Lines == 0) {
+ if (max) {
+ $("body").toast({
+ class: "info",
+ message: `您已翻阅至日志顶部,请稍后再试！`,
+ });
+ } else {
+ $(`.message${version_name} #header`).text("您已翻阅至日志顶部");
+ $(`.message${version_name}`).css("display", "block");
+ setTimeout(function () {
+ $(`.message${version_name}`).css("display", "none");
+ }, 1000);
+ }
+ }
scrollAnimation(logContentDom, logContentDom.scrollTop, 0);
}
).fail((err) => {
- $(`#log${version_name} .ui.inverted.active.dimmer`).css(
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
"display",
"none"
);
throw err;
});
- });
- $(".log_bottom").click(function (e) {
+ }
+ function logBottom(e) {
+ let max = e.currentTarget.getAttribute("data-max") || "";
+ let lines = !!max ? 100 : 60;
let version_name = $(this).data("version");
- let logContentDom = document.querySelector(`#log${version_name}`);
+ let logContentDom = document.querySelector(`#log${max}${version_name}`);
let ID = $(`#accordion${version_name}`).data("jobid");
let repoPath = $(`#accordion${version_name}`).data("repopath");
- $(`#log_file${version_name}`).siblings("pre").remove();
- let end_line = $(`#log${version_name} input[name=end_line]`).val();
- $(`#log${version_name} .ui.inverted.active.dimmer`).css("display", "block");
+ $(`#log_file${max}${version_name}`).siblings("pre").remove();
+ let end_line = $(`#log${version_name} input[name=end_line${max}]`).val();
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css({
+ "background-color": "#fff",
+ display: "block",
+ });
$.get(
- `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=50&order=desc`,
+ `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=${lines}&order=desc`,
(data) => {
- $(`#log${version_name} .ui.inverted.active.dimmer`).css(
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
"display",
"none"
);
@@ -153,39 +217,71 @@ export default async function initCloudrainSow() {
.addClass("ti-download-file")
.removeClass("disabled");
}
- $(`#log${version_name} input[name=end_line]`).val(data.EndLine); //如果变动就改变所对应的值
- $(`#log${version_name} input[name=start_line]`).val(data.StartLine);
- $(`#log${version_name}`).append("" + data.Content);
+ $(`#log${version_name} input[name=end_line${max}]`).val(data.EndLine); //如果变动就改变所对应的值
+ if ($(this)[0].hasAttribute("data-tab")) {
+ $(`#log${version_name} input[name=end_line-max]`).val(data.EndLine);
+ $(`#log${version_name} input[name=start_line-max]`).val(
+ data.StartLine
+ );
+ $(`#log${version_name} input[name=start_line-max-copy]`).val(
+ data.StartLine
+ );
+ }
+ $(`#log${version_name} input[name=start_line${max}]`).val(
+ data.StartLine
+ );
+ $(`#log${max}${version_name}`).append("" + data.Content);
$.get(
- `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${data.EndLine}&lines=50&order=desc`,
+ `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${data.EndLine}&lines=${lines}&order=desc`,
(data) => {
- $(".ui.inverted.active.dimmer").css("display", "none");
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
+ "display",
+ "none"
+ );
if (data.Lines == 0) {
- $(`.message${version_name} #header`).text("您已翻阅至日志底部");
- $(`.message${version_name}`).css("display", "block");
- setTimeout(function () {
- $(`.message${version_name}`).css("display", "none");
- }, 1000);
+ if (max) {
+ $("body").toast({
+ class: "info",
+ message: `您已翻阅至日志底部,请稍后再试！`,
+ });
+ } else {
+ $(`.message${version_name} #header`).text("您已翻阅至日志底部");
+ $(`.message${version_name}`).css("display", "block");
+ setTimeout(function () {
+ $(`.message${version_name}`).css("display", "none");
+ }, 1000);
+ }
} else {
if (end_line === data.EndLine || end_line === "") {
return;
} else {
- $(`#log${version_name} input[name=end_line]`).val(data.EndLine);
- $(`#log${version_name}`).append("" + data.Content);
+ $(`#log${version_name} input[name=end_line${max}]`).val(
+ data.EndLine
+ );
+ if ($(this)[0].hasAttribute("data-tab")) {
+ $(`#log${version_name} input[name=end_line-max]`).val(
+ data.EndLine
+ );
+ }
+ $(`#log${max}${version_name}`).append("" + data.Content);
}
}
}
).fail(function (err) {
- $(`#log${version_name} .ui.inverted.active.dimmer`).css(
+ $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
"display",
"none"
);
console.log(err);
});
+
+ let test = $(`#log_file${version_name}`).nextAll();
+ $(`#log${version_name} input[name=init_log]`).val(test[0].innerHTML);
+
scrollAnimation(
logContentDom,
logContentDom.scrollTop + 1,
- logContentDom.scrollHeight - logContentDom.clientHeight
+ logContentDom.scrollHeight - logContentDom.clientHeight - 10
);
}
).fail((err) => {
@@ -195,6 +291,68 @@ export default async function initCloudrainSow() {
);
throw err;
});
+ }
+ $(".log_top").click(logTop);
+ $(".log_bottom").click(logBottom);
+
+ // $(".log-scroll-max").scroll();
+
+ $(".full-log-dialog").click(function () {
+ let version_name = $(this).data("version");
+ let log_type = $(this).data("log-type") || "";
+ let logContentDom = document.querySelector(`#log-max${version_name}`);
+ $(`.ui.modal.max-full-log${version_name}`)
+ .modal({
+ closable: false,
+ onShow: function () {
+ $(".ui.dimmer.modals").css({
+ "background-color": "rgb(136, 136, 136,0.7)",
+ });
+ $(".log-scroll-max .ui.inverted.active.dimmer").css(
+ "display",
+ "none"
+ );
+ $(".file-info #log-file-title").text(
+ $(".full-log-dialog").data("log")
+ );
+ $(".file-info #log-file-exit").text(
+ $(".full-log-dialog").data("exit")
+ );
+ },
+ onVisible: function () {
+ $(`#log-max${version_name}`).append(
+ "" + $(`#log${version_name} input[name=init_log]`).val()
+ );
+ scrollAnimation(
+ logContentDom,
+ logContentDom.scrollTop + 1,
+ logContentDom.scrollHeight - logContentDom.clientHeight
+ );
+ if (log_type !== "c2Net") {
+ $(".log-scroll-max").bind("scroll", function () {
+ let version_name = $(this).data("version");
+ let ID = $(`#accordion${version_name}`).data("jobid");
+ let repoPath = $(`#accordion${version_name}`).data("repopath");
+ fn(version_name, repoPath, ID, "-max", 100);
+ });
+ }
+
+ $(".log_bottom-max").bind("click", logBottom);
+ $(".log_top-max").bind("click", logTop);
+ },
+ onHide: function () {
+ let startLine = $(
+ `#log${version_name} input[name=start_line-max-copy]`
+ ).val();
+ $(`#log_file-max${version_name}`).siblings("pre").remove();
+ $(`#log${version_name} input[name=start_line-max]`).val(startLine);
+
+ $(".log-scroll-max").unbind("scroll");
+ $(".log_bottom-max").unbind("click");
+ $(".log_top-max").unbind("click");
+ },
+ })
+ .modal("show");
});
function loadLog(version_name) {
@@ -245,7 +403,6 @@ export default async function initCloudrainSow() {
.modal({
onApprove: function () {
$.post(url, { version_name: version_name }, (data) => {
- console.log(data);
if (data.StatusOK === 0) {
if (data.VersionListCount === 0) {
location.href = `/${repoPath}`;