From c1a63edd2cfd3516ca8a4e9d89472fac076a4a3f Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 20 Oct 2022 10:37:13 +0800 Subject: [PATCH 1/8] add --- modules/setting/setting.go | 8 ++++++++ routers/repo/ai_model_convert.go | 18 +++++++++++++++++- templates/repo/modelmanage/convertIndex.tmpl | 6 ++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c6afae05a..89a0fd2d6 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -694,8 +694,12 @@ var ( GPU_PYTORCH_IMAGE string GpuQueue string GPU_TENSORFLOW_IMAGE string + GPU_PADDLE_IMAGE string + GPU_MXNET_IMAGE string NPU_MINDSPORE_16_IMAGE string PytorchOnnxBootFile string + PaddleOnnxBootFile string + MXnetOnnxBootFile string PytorchTrTBootFile string MindsporeBootFile string TensorFlowNpuBootFile string @@ -1576,6 +1580,10 @@ func getModelConvertConfig() { ModelConvert.NPU_PoolID = sec.Key("NPU_PoolID").MustString("pool7908321a") ModelConvert.NPU_MINDSPORE_IMAGE_ID = sec.Key("NPU_MINDSPORE_IMAGE_ID").MustInt(121) ModelConvert.NPU_TENSORFLOW_IMAGE_ID = sec.Key("NPU_TENSORFLOW_IMAGE_ID").MustInt(35) + ModelConvert.GPU_PADDLE_IMAGE = sec.Key("GPU_PADDLE_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:paddle2.3.0_gpu_cuda11.2_cudnn8") + ModelConvert.GPU_MXNET_IMAGE = sec.Key("GPU_MXNET_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:mxnet191cu_cuda102_py37") + ModelConvert.PaddleOnnxBootFile = sec.Key("PaddleOnnxBootFile").MustString("convert_paddle.py") + ModelConvert.MXnetOnnxBootFile = sec.Key("MXnetOnnxBootFile").MustString("convert_mxnet.py") } func getModelAppConfig() { diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 9a5874956..bd6a01072 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -29,7 +29,9 @@ const ( tplModelConvertInfo = "repo/modelmanage/convertshowinfo" PYTORCH_ENGINE = 0 TENSORFLOW_ENGINE = 1 - MINDSPORE_ENGIN = 2 + MINDSPORE_ENGINE = 2 + PADDLE_ENGINE = 4 + MXNET_ENGINE = 6 ModelMountPath = "/model" CodeMountPath = "/code" DataSetMountPath = "/dataset" @@ -395,6 +397,20 @@ func createGpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context deleteLocalDir(relatetiveModelPath) dataActualPath = setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/dataset" } + } else if modelConvert.SrcEngine == PADDLE_ENGINE { + IMAGE_URL = setting.ModelConvert.GPU_PADDLE_IMAGE + if modelConvert.DestFormat == CONVERT_FORMAT_ONNX { + command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PaddleOnnxBootFile) + } else { + return errors.New("Not support the format.") + } + } else if modelConvert.SrcEngine == MXNET_ENGINE { + IMAGE_URL = setting.ModelConvert.GPU_MXNET_IMAGE + if modelConvert.DestFormat == CONVERT_FORMAT_ONNX { + command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.MXnetOnnxBootFile) + } else { + return errors.New("Not support the format.") + } } log.Info("dataActualPath=" + dataActualPath) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 92eefca2e..26e79b04c 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -103,7 +103,7 @@
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow{{else if eq .SrcEngine 2}}MindSpore {{end}} + {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow {{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle {{else if eq .SrcEngine 6}}MXNet {{end}}
{{if eq .DestFormat 0}}ONNX {{else if eq .DestFormat 1}}TensorRT {{end}} @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdparams",".params",".json"]; for(var i =0; iPyTorch"; html +=""; html +=""; + html +=""; + html +=""; $('#SrcEngine').html(html); srcEngineChanged(); } From 3695fd996c94557a72e78a1f22055e82afbe0955 Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 21 Oct 2022 16:40:38 +0800 Subject: [PATCH 2/8] added --- models/ai_model_manage.go | 2 +- routers/repo/ai_model_convert.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a88da8fe5..d9adda2dc 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -88,7 +88,7 @@ type AiModelQueryOptions struct { } func (a *AiModelConvert) IsGpuTrainTask() bool { - if a.SrcEngine == 0 || a.SrcEngine == 1 { + if a.SrcEngine == 0 || a.SrcEngine == 1 || a.SrcEngine == 4 || a.SrcEngine == 6 { return true } return false diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index bd6a01072..13b02f9d0 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/set ting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" uuid "github.com/satori/go.uuid" From e1a454d5704835a3a78802eb2b8ffac74eaee892 Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 21 Oct 2022 16:43:59 +0800 Subject: [PATCH 3/8] add --- routers/repo/ai_model_convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 13b02f9d0..bd6a01072 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/set ting" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" uuid "github.com/satori/go.uuid" From 674f4701c998cbd6f81e31d28c9f4a424f9fd31a Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 10:57:32 +0800 Subject: [PATCH 4/8] add convert --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 26e79b04c..1802b3fdd 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdparams",".params",".json"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdiparams",".params",".json"]; for(var i =0; i Date: Tue, 25 Oct 2022 16:11:34 +0800 Subject: [PATCH 5/8] modified --- options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 773a338c1..ecd2bb038 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3216,7 +3216,7 @@ view_sample = View sample inference_output_path_rule = The inference output path is stored in the run parameter result_url. model_file_path_rule=The model file location is stored in the run parameter ckpt_url model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] -model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] +model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json] delete_task = Delete task task_delete_confirm = Are you sure you want to delete this task? Once this task is deleted, it cannot be recovered. operate_confirm = confirm diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8ba4d252d..74e46c6ea 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3234,7 +3234,7 @@ view_sample = 查看样例 inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。 model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。 model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] -model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] +model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json] delete_task = 删除任务 task_delete_confirm = 你确认删除该任务么？此任务一旦删除不可恢复。 operate_confirm = 确定操作 From 3dc3273f53afd2b6c03a7adebd60477ea73a3035 Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 16:16:54 +0800 Subject: [PATCH 6/8] modified --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 1802b3fdd..05b5306c8 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdiparams",".params",".json"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel",".pdiparams",".params",".json"]; for(var i =0; i Date: Thu, 27 Oct 2022 15:31:25 +0800 Subject: [PATCH 7/8] add paddle&mxnet --- templates/repo/modelmanage/convertshowinfo.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertshowinfo.tmpl b/templates/repo/modelmanage/convertshowinfo.tmpl index 70aa301d5..34387b90c 100644 --- a/templates/repo/modelmanage/convertshowinfo.tmpl +++ b/templates/repo/modelmanage/convertshowinfo.tmpl @@ -355,7 +355,7 @@ td, th {
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}Tensorflow{{else if eq .SrcEngine 2}}MindSpore {{end}} + {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}Tensorflow{{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle{{else if eq .SrcEngine 6}} MxNet{{end}}
From 0272d680891e706c88134b4f796228e511f0493c Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 27 Oct 2022 15:33:46 +0800 Subject: [PATCH 8/8] add paddle&mxnet --- templates/repo/modelmanage/convertshowinfo.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertshowinfo.tmpl b/templates/repo/modelmanage/convertshowinfo.tmpl index 34387b90c..f4db2237c 100644 --- a/templates/repo/modelmanage/convertshowinfo.tmpl +++ b/templates/repo/modelmanage/convertshowinfo.tmpl @@ -355,7 +355,7 @@ td, th {
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}Tensorflow{{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle{{else if eq .SrcEngine 6}} MxNet{{end}} + {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}Tensorflow{{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle{{else if eq .SrcEngine 6}} MXNet{{end}}