From f86fe994bec292e3bb0f8677da573f9dbc66f796 Mon Sep 17 00:00:00 2001 From: wqtshg Date: Fri, 6 Nov 2020 14:18:34 +0800 Subject: [PATCH] protobuf test --- CMakeLists.txt | 1 + cmake/external_libs/protobuf.cmake | 110 +++++++++++++++++---------------- cmake/external_libs/protoc.cmake | 112 ++++++++++++++++++++++++++++++++++ src/common/graph/CMakeLists.txt | 4 +- src/ge/CMakeLists.txt | 6 +- src/ge/client/CMakeLists.txt | 4 +- src/ge/common/CMakeLists.txt | 2 +- src/ge/executor/CMakeLists.txt | 2 +- src/ge/ge_local_engine/CMakeLists.txt | 2 +- 9 files changed, 179 insertions(+), 64 deletions(-) create mode 100644 cmake/external_libs/protoc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 457fa086..d842ba4a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ include(${GE_SOURCE_DIR}/cmake/external_libs/json.cmake) include(${GE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) include(${GE_SOURCE_DIR}/cmake/external_libs/gtest.cmake) include(${GE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) +include(${GE_SOURCE_DIR}/cmake/external_libs/protoc.cmake) include(${GE_SOURCE_DIR}/cmake/external_libs/onnx.cmake) include(${GE_SOURCE_DIR}/cmake/external_libs/securec.cmake) set(CMAKE_SKIP_RPATH TRUE) diff --git a/cmake/external_libs/protobuf.cmake b/cmake/external_libs/protobuf.cmake index 8be594c7..08f54d87 100644 --- a/cmake/external_libs/protobuf.cmake +++ b/cmake/external_libs/protobuf.cmake @@ -1,10 +1,15 @@ -if (NOT TARGET protobuf::protobuf) -set(protobuf_USE_STATIC_LIBS ON) -set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2") -set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") -set(_ge_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) -string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") -string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +if (HAVE_PROTOBUF) + return() +endif() + +include(ExternalProject) +include(GNUInstallDirs) + +if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR + (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend")) + set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE) + message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.") +endif() if (ENABLE_GITEE) set(REQ_URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.8.0.tar.gz") @@ -14,50 +19,47 @@ else() set(MD5 "3d9e32700639618a4d2d342c99d4507a") endif () -graphengine_add_pkg(protobuf - VER 3.8.0 - LIBS protobuf - EXE protoc - URL ${REQ_URL} - MD5 ${MD5} - CMAKE_PATH ../cmake/ - CMAKE_OPTION -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF) -set(CMAKE_CXX_FLAGS ${_ge_tmp_CMAKE_CXX_FLAGS}) -endif() -add_library(graphengine::protobuf ALIAS protobuf::protobuf) -set(PROTOBUF_LIBRARY protobuf::protobuf) -include_directories(${protobuf_INC}) -include_directories(${protobuf_DIRPATH}/src) - -function(ge_protobuf_generate comp c_var h_var) - if(NOT ARGN) - message(SEND_ERROR "Error: ge_protobuf_generate() called without any proto files") - return() - endif() - - set(${c_var}) - set(${h_var}) - - foreach(file ${ARGN}) - get_filename_component(abs_file ${file} ABSOLUTE) - get_filename_component(file_name ${file} NAME_WE) - get_filename_component(file_dir ${abs_file} PATH) - - list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc") - list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc" - "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h" - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto/${comp}/proto" - COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto/${comp}/proto ${abs_file} - DEPENDS protobuf::protoc ${abs_file} - COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) - endforeach() - - set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) - set(${c_var} ${${c_var}} PARENT_SCOPE) - set(${h_var} ${${h_var}} PARENT_SCOPE) - -endfunction() +set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2 -Dgoogle=ascend_private") +set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") +ExternalProject_Add(protobuf_build + URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -Dprotobuf_WITH_ZLIB=OFF + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_LINKER=${CMAKE_LINKER} + -DCMAKE_AR=${CMAKE_AR} + -DCMAKE_RANLIB=${CMAKE_RANLIB} + -DLIB_PREFIX=ascend_ + -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protobuf /cmake + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + EXCLUDE_FROM_ALL TRUE +) +include(GNUInstallDirs) + +set(PROTOBUF_SHARED_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf) + +add_library(ascend_protobuf SHARED IMPORTED) + +file(MAKE_DIRECTORY ${PROTOBUF_SHARED_PKG_DIR}/include) + +set_target_properties(ascend_protobuf PROPERTIES + IMPORTED_LOCATION ${PROTOBUF_SHARED_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/libascend_protobuf.so +) + +target_include_directories(ascend_protobuf INTERFACE ${PROTOBUF_SHARED_PKG_DIR}/include) + +set(INSTALL_BASE_DIR "") +set(INSTALL_LIBRARY_DIR lib) + +install(FILES ${PROTOBUF_SHARED_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/ascend_protobuf.so.3.8.0.0 OPTIONAL + DESTINATION ${INSTALL_LIBRARY_DIR}) +install(FILES ${PROTOBUF_SHARED_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/ascend_protobuf.so OPTIONAL + DESTINATION ${INSTALL_LIBRARY_DIR}) + +add_dependencies(ascend_protobuf protobuf_build) + +#set(HAVE_PROTOBUF TRUE CACHE BOOL "protobuf build add") +set(HAVE_PROTOBUF TRUE) diff --git a/cmake/external_libs/protoc.cmake b/cmake/external_libs/protoc.cmake new file mode 100644 index 00000000..d1d329f2 --- /dev/null +++ b/cmake/external_libs/protoc.cmake @@ -0,0 +1,112 @@ +if (HAVE_PROTOC) + return() +endif() + +include(ExternalProject) +include(GNUInstallDirs) +#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output) + +if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR + (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend")) + set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE) + message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.") +endif() + +if (ENABLE_GITEE) + set(REQ_URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.8.0.tar.gz") + set(MD5 "eba86ae9f07ba5cfbaf8af3bc4e84236") +else() + set(REQ_URL "https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz") + set(MD5 "3d9e32700639618a4d2d342c99d4507a") +endif () + + +set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") +set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") +ExternalProject_Add(protoc_build + URL ${REQ_URL} + #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz + #SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0 + CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc /cmake + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + EXCLUDE_FROM_ALL TRUE +) + +set(PROTOC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protoc) + +set(protoc_EXECUTABLE ${PROTOC_PKG_DIR}/${CMAKE_INSTALL_BINDIR}/protoc) + +function(protobuf_generate comp c_var h_var) + if(NOT ARGN) + message(SEND_ERROR "Error: protobuf_generate() called without any proto files") + return() + endif() + set(${c_var}) + set(${h_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + get_filename_component(parent_subdir ${file_dir} NAME) + + if("${parent_subdir}" STREQUAL "proto") + set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto) + else() + set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir}) + endif() + list(APPEND ${c_var} "${proto_output_path}/${file_name}.pb.cc") + list(APPEND ${h_var} "${proto_output_path}/${file_name}.pb.h") + + add_custom_command( + OUTPUT "${proto_output_path}/${file_name}.pb.cc" "${proto_output_path}/${file_name}.pb.h" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}" + COMMAND ${protoc_EXECUTABLE} -I${file_dir} --cpp_out=${proto_output_path} ${abs_file} + DEPENDS protoc_build ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + +endfunction() + +function(protobuf_generate_py comp py_var) + if(NOT ARGN) + message(SEND_ERROR "Error: protobuf_generate_py() called without any proto files") + return() + endif() + set(${py_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + get_filename_component(parent_subdir ${file_dir} NAME) + + if("${parent_subdir}" STREQUAL "proto") + set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto) + else() + set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir}) + endif() + list(APPEND ${py_var} "${proto_output_path}/${file_name}_pb2.py") + + add_custom_command( + OUTPUT "${proto_output_path}/${file_name}_pb2.py" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}" + COMMAND ${protoc_EXECUTABLE} -I${file_dir} --python_out=${proto_output_path} ${abs_file} + DEPENDS protoc_build ${abs_file} + COMMENT "Running PYTHON protocol buffer compiler on ${file}" VERBATIM ) + endforeach() + + set_source_files_properties(${${py_var}} PROPERTIES GENERATED TRUE) + set(${py_var} ${${py_var}} PARENT_SCOPE) + +endfunction() + +#set(HAVE_PROTOC TRUE CACHE BOOL "protoc build add") +set(HAVE_PROTOC TRUE) diff --git a/src/common/graph/CMakeLists.txt b/src/common/graph/CMakeLists.txt index bb63eb81..ef56b130 100755 --- a/src/common/graph/CMakeLists.txt +++ b/src/common/graph/CMakeLists.txt @@ -31,8 +31,8 @@ file(GLOB_RECURSE ONNX_PROTO_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} "${onnx_INC}/onnx/onnx.proto" ) -ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -ge_protobuf_generate(ge PROTO_ONNX_SRCS PROTO_ONNX_HDRS ${ONNX_PROTO_LIST}) +protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) +protobuf_generate(ge PROTO_ONNX_SRCS PROTO_ONNX_HDRS ${ONNX_PROTO_LIST}) # need to remove dependencies on pb files later file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} diff --git a/src/ge/CMakeLists.txt b/src/ge/CMakeLists.txt index 3f4f1a8b..d85adf9e 100755 --- a/src/ge/CMakeLists.txt +++ b/src/ge/CMakeLists.txt @@ -33,9 +33,9 @@ file(GLOB PROTO_HEADER_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} "../proto/op_mapping_info.proto" "../proto/dump_task.proto" ) -ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -ge_protobuf_generate(ge PROTO_CLIENT_SRCS PROTO_CLIENT_HDRS ${PROTO_CLIENT_LIST}) -ge_protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST}) +protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) +protobuf_generate(ge PROTO_CLIENT_SRCS PROTO_CLIENT_HDRS ${PROTO_CLIENT_LIST}) +protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST}) # include directories include_directories(${CMAKE_CURRENT_LIST_DIR}) include_directories(${GE_SOURCE_DIR}) diff --git a/src/ge/client/CMakeLists.txt b/src/ge/client/CMakeLists.txt index b568e3f6..3ac02528 100755 --- a/src/ge/client/CMakeLists.txt +++ b/src/ge/client/CMakeLists.txt @@ -32,8 +32,8 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} "ge_prof.cc" ) -ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -ge_protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST}) +protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) +protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST}) # include directories include_directories(${CMAKE_CURRENT_LIST_DIR}) diff --git a/src/ge/common/CMakeLists.txt b/src/ge/common/CMakeLists.txt index f6c75f87..13a7874f 100755 --- a/src/ge/common/CMakeLists.txt +++ b/src/ge/common/CMakeLists.txt @@ -67,7 +67,7 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} "util.cc" ) -ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) +protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) # include directories include_directories(${CMAKE_CURRENT_LIST_DIR}) diff --git a/src/ge/executor/CMakeLists.txt b/src/ge/executor/CMakeLists.txt index b68507bd..56f9389c 100755 --- a/src/ge/executor/CMakeLists.txt +++ b/src/ge/executor/CMakeLists.txt @@ -87,7 +87,7 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} "../single_op/task/tbe_task_builder.cc" ) -ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) +protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) # include directories include_directories(${CMAKE_CURRENT_LIST_DIR}) diff --git a/src/ge/ge_local_engine/CMakeLists.txt b/src/ge/ge_local_engine/CMakeLists.txt index bcbc3e4c..a725694b 100755 --- a/src/ge/ge_local_engine/CMakeLists.txt +++ b/src/ge/ge_local_engine/CMakeLists.txt @@ -25,7 +25,7 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR} "ops_kernel_store/op/*.cc" ) -ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) +protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) # include directories include_directories(${CMAKE_CURRENT_LIST_DIR})