CMake target dependency to compile protobuf files - cmake

I want to build a c++ static lib out of some protobuf definitions with cmake/make.
I made a custom COMMAND to compile the protobuf to c++, and I set it as a PRE_BUILD dependency to my static lib.
project(mylib)
set(PROTO_PATH "${CMAKE_CURRENT_SOURCE_DIR}/proto_definitions")
file(GLOB PROTO_FILES "${PROTO_PATH}/*.proto")
foreach(PROTO_FILE in ${PROTO_FILES})
string(REGEX REPLACE "[.]proto$" ".pb.cc" OUTPUT_SOURCE ${PROTO_FILE})
list(APPEND OUTPUT_SOURCES ${OUTPUT_SOURCE})
endforeach()
add_custom_command(TARGET ${PROJECT_NAME}
PRE_BUILD
COMMAND protoc --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/compiled_proto ${PROTO_FILES}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "some comment")
add_library(${PROJECT_NAME} STATIC ${OUTPUT_SOURCES})
target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
I get the following error when running cmake:
CMake Error: cannot determine link language for target "mylib"
Regardless of this error, the makefiles are generated, but when I make mylib, it does not trigger any proto compilation

The approach more native to CMake would be to add custom commands with the OUTPUT signature to generate the .cc files, and then use them as sources for the library normally. That way, they CMake will know what they are and how to produce them:
project(mylib)
set(PROTO_PATH "${CMAKE_CURRENT_SOURCE_DIR}/proto_definitions")
file(GLOB PROTO_FILES "${PROTO_PATH}/*.proto")
foreach(PROTO_FILE in ${PROTO_FILES})
string(REGEX REPLACE "[.]proto$" ".pb.cc" OUTPUT_SOURCE ${PROTO_FILE})
list(APPEND OUTPUT_SOURCES ${OUTPUT_SOURCE})
endforeach()
add_custom_command(OUTPUT ${OUTPUT_SOURCES}
COMMAND protoc --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/compiled_proto ${PROTO_FILES}
DEPENDS ${PROTO_FILES}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "some comment")
add_library(${PROJECT_NAME} STATIC ${OUTPUT_SOURCES})
target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
Done this way, there will be one command which reads all the .proto files and produces all the .cc files — which means that if any of the .proto file changes, all the .cc files will be re-generated. I am not familiar with Protobuffers so I cannot know whether that's sane or not. If they are independent, it would be better to introduce one add_custom_command for each output file.
Also, given the arguments you're passing to protocc, you might have to modify the paths in OUTPUT_SOURCES to correctly point to the generated files.
Also note that CMake comes with a FindProtobuf module which defines a protobuf_generate_cpp() command, so you might want to use that instead of hand-coding the Protobuf support.

Comment for the answer above: There is no in in cmake foreach, which spend me some time to solve it.
After studying his answer(very thanks!), I summarized a method that works for me, which has the following characteristics:
Proto dir and Output dir can be different
Generate grpc_output
Create a custom_command for each proto
My project strcuture is:
hwpb/ (a library for super project)
cpp/
hwpb/ (store *.pb.cc, *.pb.h)
CMakeLists.txt
proto/ (store protos)
go/ (and other languages, not mentioned here)
And this is my CMakeListx.txt:
set(PROTO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../proto")
set(OUTPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/hwpb")
file(GLOB PROTO_FILES "${PROTO_DIR}/*.proto")
foreach(PROTO_FILE ${PROTO_FILES})
get_filename_component (FILENAME ${PROTO_FILE} NAME_WLE)
set(PROTO_SRC "${OUTPUT_DIR}/${FILENAME}.pb.cc")
set(PROTO_HDR "${OUTPUT_DIR}/${FILENAME}.pb.h")
set(GRPC_SRC "${OUTPUT_DIR}/${FILENAME}.grpc.pb.cc")
set(GRPC_HDR "${OUTPUT_DIR}/${FILENAME}.grpc.pb.h")
add_custom_command(
OUTPUT "${PROTO_SRC}" "${PROTO_HDR}" "${GRPC_SRC}" "${GRPC_HDR}"
COMMAND ${_PROTOBUF_PROTOC}
ARGS --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN_EXECUTABLE}
--cpp_out="${OUTPUT_DIR}" --grpc_out="${OUTPUT_DIR}"
-I"${PROTO_DIR}" "${PROTO_FILE}"
DEPENDS ${PROTO_FILE}
)
list(APPEND OUTPUT_SOURCES ${PROTO_SRC} ${GRPC_HDR})
endforeach()
add_library(hwpb ${OUTPUT_SOURCES})
target_link_libraries(hwpb ${_GRPC_GRPCPP_UNSECURE} ${_PROTOBUF_LIBPROTOBUF})
target_include_directories(hwpb PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})

Related

CMake protobuf external to application code

Basically, I want to have my structure like such:
MainRepo
+---app1
+---app2
+---common
+---some_lib1
+---some_lib2
+---protobuf
+---comms.proto
+---cs
+---Comms.pb.cs
+---cpp
+---comms.pb.cc
+---comms.pb.h
I want to be able to check out the repo and have a script that runs protoc for all the different languages for the apps that are in the repo. This is a mono repo containing an app for two different arm machines and an x64. I essentially are running protoc and it generates all the source files for c, js, cs, cpp, etc and puts them under protobuf in their own folders.
I want to have app1, for example, go find the c++ header and source and use them to build the app. At the moment, the example I have been hacking, uses cmake to generate the .cc and .h which makes it inconvenient for me as intellisense complains since those files dont exist when I'm writing.
Anyway, I've been hacking away at cmake all day. I always end up with cmake having a forward declaration error and cant compile my .cc and .h
PROJECT(test)
CMAKE_MINIMUM_REQUIRED (VERSION 2.6)
SET(CMAKE_CXX_FLAGS "-g -Wall -Werror -std=c++11")
ADD_EXECUTABLE(main main.cpp)
TARGET_LINK_LIBRARIES(main proto ${PROTOBUF_LIBRARY})
find_package(Protobuf REQUIRED)
set(PROTOBUF_IMPORT_DIRS "../proto")
set (msgs ${PROTOBUF_IMPORT_DIRS}/communications.proto)
PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS ${msgs})
add_library(proto SHARED ${PROTO_SRCS})
target_link_libraries(proto ${PROTOBUF_LIBRARY})
Wondering if there are any suggestions. I dont want to have my protobuf stuff outside of my common folder and I also dont need protoc to generate those files either as I do that another way (although I could change that way). I just want to ensure that the language specific files are still available to view and not just generated during cmake and I cant view them.
My 2 cents,
Here this is what I did for google/or-tools
ref: https://github.com/google/or-tools
Protobuf Integration
I'm using Fetchcontent() (CMake >= 3.18 to have SOURCE_SUBDIR option IIRC), but I also need to patch it (e.g. to have CMP0077)
you can find the protobuf patch here: https://github.com/google/or-tools/blob/stable/patches/protobuf-v3.12.2.patch
cmake/dependencies/CMakeLists.txt
include(FetchContent)
set(BUILD_SHARED_LIBS OFF)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(BUILD_TESTING OFF)
message(CHECK_START "Fetching Protobuf")
list(APPEND CMAKE_MESSAGE_INDENT " ")
set(protobuf_BUILD_TESTS OFF)
set(protobuf_BUILD_EXPORT OFF)
set(protobuf_MSVC_STATIC_RUNTIME OFF)
# FetchContent_Declare(SOURCE_SUBDIR) was introduced in 3.18
FetchContent_Declare(
protobuf
GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git"
GIT_TAG "v3.12.2"
PATCH_COMMAND git apply "${CMAKE_CURRENT_LIST_DIR}/../../patches/protobuf-v3.12.2.patch"
SOURCE_SUBDIR cmake)
FetchContent_MakeAvailable(protobuf)
list(POP_BACK CMAKE_MESSAGE_INDENT)
message(CHECK_PASS "fetched")
ref: https://github.com/google/or-tools/blob/a0a56698ba8fd07b7f84aee4fc45d891a8cd9828/cmake/dependencies/CMakeLists.txt#L142-L168
note: for cmake < 3.18, I use ExternalProject + execute_process()
see: https://github.com/google/or-tools/blob/a0a56698ba8fd07b7f84aee4fc45d891a8cd9828/cmake/utils.cmake#L66-L137
Generate Protobuf files
Since we have integrated Protobuf, now we have access to protobuf::protoc.
To build proto file, simply adapt
# Get Protobuf include dirs
get_target_property(protobuf_dirs protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES)
foreach(dir IN LISTS protobuf_dirs)
if ("${dir}" MATCHES "BUILD_INTERFACE")
message(STATUS "Adding proto path: ${dir}")
list(APPEND PROTO_DIRS "--proto_path=${dir}")
endif()
endforeach()
# Generate Protobuf cpp sources
set(PROTO_HDRS)
set(PROTO_SRCS)
file(GLOB_RECURSE proto_files RELATIVE ${PROJECT_SOURCE_DIR}
"common/protobuf/*.proto"
)
foreach(PROTO_FILE IN LISTS proto_files)
#message(STATUS "protoc proto(cc): ${PROTO_FILE}")
get_filename_component(PROTO_DIR ${PROTO_FILE} DIRECTORY)
get_filename_component(PROTO_NAME ${PROTO_FILE} NAME_WE)
set(PROTO_HDR ${PROJECT_BINARY_DIR}/${PROTO_DIR}/${PROTO_NAME}.pb.h)
set(PROTO_SRC ${PROJECT_BINARY_DIR}/${PROTO_DIR}/${PROTO_NAME}.pb.cc)
#message(STATUS "protoc hdr: ${PROTO_HDR}")
#message(STATUS "protoc src: ${PROTO_SRC}")
add_custom_command(
OUTPUT ${PROTO_SRC} ${PROTO_HDR}
COMMAND protobuf::protoc
"--proto_path=${PROJECT_SOURCE_DIR}"
${PROTO_DIRS}
"--cpp_out=${PROJECT_BINARY_DIR}"
${PROTO_FILE}
DEPENDS ${PROTO_FILE} protobuf::protoc
COMMENT "Generate C++ protocol buffer for ${PROTO_FILE}"
VERBATIM)
list(APPEND PROTO_HDRS ${PROTO_HDR})
list(APPEND PROTO_SRCS ${PROTO_SRC})
endforeach()
ref: https://github.com/google/or-tools/blob/a0a56698ba8fd07b7f84aee4fc45d891a8cd9828/cmake/cpp.cmake#L234-L279
After you can use PROTO_HDR and PROTO_SRC e.g. add them to your target sources etc...
note: for .Net you can take a look at
https://github.com/google/or-tools/blob/a0a56698ba8fd07b7f84aee4fc45d891a8cd9828/cmake/dotnet.cmake#L30-L60
All in all, just adapt to your need, e.g. generate in source tree than binary dir etc. and you should be able to do whatever you want...

CMAKE custom compiler not overwriting

In a CMAKE project, I need to define a custom command for a file type (.osl) which is compiled by a tool (oslc) which compiles to another file type (.oso)
I managed to do it by a function that I can run on a list of source files:
find_program(OSLC_EXECUTABLE oslc)
function(compile_osl out_var)
set(result)
foreach(osl_f ${ARGN})
file(RELATIVE_PATH osl_f_base ${CMAKE_CURRENT_SOURCE_DIR} ${osl_f})
string(REGEX REPLACE "\\.osl$" ".oso" oso_f ${osl_f_base})
set(oso_f "${CMAKE_CURRENT_BINARY_DIR}/${oso_f}")
get_filename_component(oso_f_dir ${oso_f} DIRECTORY)
file(MAKE_DIRECTORY ${oso_f_dir})
add_custom_command(OUTPUT ${oso_f}
COMMAND ${OSLC_EXECUTABLE} ${osl_f} -o ${oso_f}
DEPENDS ${osl_f}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Creating compiled OSL file ${oso_f}"
VERBATIM
)
list(APPEND result ${oso_f})
endforeach()
set(${out_var} "${result}" PARENT_SCOPE)
endfunction()
Thanks to the DEPENDS directive, the compiler is only run if the sources are newer than the output files. However, the compiler does not overwrite output files, so it does not work.
Deleting with file(REMOVE ...) just before the custom command does not work, as it deletes all files, not only the ones requiring recompilation. Also, it deletes at cmake execution, not at make time.
I could maybe define another custom command with "rm" but this not cross platform (I would need to add specific lines for Windows, which I do not like).
Any idea?
Thanks!

Get CMake to not be silent about sources it doesn't understand?

Suppose you have a very simple CMakeLists.txt
add_executable(silent T.cpp A.asm)
CMake will happily generate a C++ target for building silent, with T.cpp in it, but will silently drop any and all reference to A.asm, because it doesn't know what to do with the suffix.
Is there any way to get CMake to loudly complain about this source file it doesn't understand (to aid in porting a Makefile to CMake).
Ignoring unknown file extensions is - unfortunately for your case - by design.
If I look at the code of cmGeneratorTarget::ComputeKindedSources() anything unknown ends up to be classified as SourceKindExtra (to be added as such to generated IDE files).
So I tested a little and came up with the following script that evaluates your executable target source files for valid file extensions by overwriting add_executable() itself:
cmake_minimum_required(VERSION 3.3)
project(silent CXX)
file(WRITE T.cpp "int main() { return 0; }")
file(WRITE T.h "")
file(WRITE A.asm "")
function(add_executable _target)
_add_executable(${_target} ${ARGN})
get_property(_langs GLOBAL PROPERTY ENABLED_LANGUAGES)
foreach(_lang IN LISTS _langs)
list(APPEND _ignore "${CMAKE_${_lang}_IGNORE_EXTENSIONS}")
endforeach()
get_target_property(_srcs ${_target} SOURCES)
foreach(_src IN LISTS _srcs)
get_source_file_property(_lang "${_src}" LANGUAGE)
get_filename_component(_ext "${_src}" EXT)
string(SUBSTRING "${_ext}" 1 -1 _ext) # remove leading dot
if (NOT _lang AND NOT _ext IN_LIST _ignore)
message(FATAL_ERROR "Target ${_target}: Unknown source file type '${_src}'")
endif()
endforeach()
endfunction()
add_executable(silent T.cpp T.h A.asm)
Since you wanted a rather loudly complain by CMake I declared it an FATAL_ERROR in this example implementation.
CMake doesn't just drop unknown files in add_executable().
If alongside with
add_executable(silent T.cpp A.asm)
you have
add_custom_command(OUTPUT A.asm COMMAND <...>
DEPENDS <dependees>)
then whenever <dependees> changed CMake will rerun command for create A.asm before compiling the executable.
Note, that automatical headers scanning doesn't provide such functionality: if your executable includes foo.h then executable will be rebuilt only when foo.h itself is changed. Any custom command creating this header will be ignored.
However, you may change behavior of add_executable by redefining it. See #Florian's answer for example of such redefinition.

cmake does not copy file in custom command

I want to copy qml files from source directory to build directory. Following script works fine only first time. When I change any of the *.qml files and run make, they are not copied to build folder, they are not updated. What am I doing wrong?
file(GLOB_RECURSE SOURCES *.cpp)
file(GLOB_RECURSE QMLS *.qml)
add_library(MyLib SHARED ${SOURCES} ${QMLS})
foreach(QmlFile ${QMLS})
add_custom_command(TARGET MyLib POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
${QmlFile} $<TARGET_FILE_DIR:MyLib>)
endforeach()
While Angew's answer is good, it is possible to eliminate usage of additional target. For doing this, add_library call should use copied .qml files (instead of original ones, like in the script in the question post):
# This part is same as in Angew's answer
set(copiedQmls "")
foreach(QmlFile ${QMLS})
get_filename_component(nam ${QmlFile} NAME)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${nam}
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${QmlFile} ${CMAKE_CURRENT_BINARY_DIR}
DEPENDS ${QmlFile}
COMMENT "Copying ${QmlFile}"
VERBATIM
)
list(APPEND copiedQmls ${CMAKE_CURRENT_BINARY_DIR}/${nam})
endforeach()
# But instead of creating new target, we reuse library one.
add_library(MyLib SHARED ${SOURCES} ${copiedQmls})
When library target is built, it triggers non-sources files in add_library call to be updated (if there is corresponded add_custom_command call), but updating non-source files doesn't force library file to be rebuilt. This is why your original code doesn't work as expected.
Note, because .qml files are not recognized by CMake as sources, you doesn't need to set GENERATED property for them, as stated here.
Your are using the TARGET signature of add_custom_command, which means the commands are executed as part of building the TARGET. In your case, POST_BUILD, which means the commands will be run after the build of MyLib finishes. If MyLib is up to date and does not need to be re-built, the commands will not run.
You might want to use the output-generating signature (OUTPUT) instead. Something like this:
set(copiedQmls "")
foreach(QmlFile ${QMLS})
get_filename_component(nam ${QmlFile} NAME)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${nam}
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${QmlFile} ${CMAKE_CURRENT_BINARY_DIR}
DEPENDS ${QmlFile}
COMMENT "Copying ${QmlFile}"
VERBATIM
)
list(APPEND copiedQmls ${CMAKE_CURRENT_BINARY_DIR}/${nam})
endforeach()
add_custom_target(
CopyQMLs ALL
DEPENDS ${copiedQmls}
)
Note that unfortunately, the OUTPUT argument of add_custom_command does not support generator expressions, so $<TARGET_FILE_DIR> cannot be used there. I used ${CMAKE_CURRENT_BINARY_DIR} in the example above, you might need to customise this to suit your needs. ${CMAKE_CFG_INTDIR} can be used to specify the per-configuration directory for multiconfiguration generators.
Note the presence of the additional custom target in my code above. That's necessary, because CMake will only execute an output-producing custom command if something depends on that output. The custom command does that, by listing the outputs in its DEPENDS section.

custom target as a target library in cmake

I have a custom target that is in fact an externally generated library that I want to integrate in my build.
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/liblib2.a
COMMAND make -f ${CMAKE_CURRENT_SOURCE_DIR}/makefile liblib2.a)
add_custom_target(lib2
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/liblib2.a)
How can I tell cmake that this target is in fact a library, where it can be found and where are the headers ?
To be clear : I don't want the upper CMakeList using this library having to manually specify include folders and the library location folder It must be done automatically (from the target properties).
On a standard cmake library I would just have to add the INTERFACE_INCLUDE_DIRECTORIES property in the library CMakeLists to make cmake link my app with the relevant -I and -L gcc parameters :
set_target_properties(lib1
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES
${CMAKE_CURRENT_SOURCE_DIR})
But in the case of a custom target I don't know how to to it.
Any clue ?
Thanks for your help.
Thanks to zaufi it works!
For others who may be interested in embedded externally build target inside cmake here is what I did :
cmake_minimum_required(VERSION 2.8)
SET(LIB_FILE ${CMAKE_CURRENT_SOURCE_DIR}/bin/liblib2.a)
SET(LIB_HEADER_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/include)
# how to build the result of the library
add_custom_command(OUTPUT ${LIB_FILE}
COMMAND make
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
# create a target out of the library compilation result
add_custom_target(lib2_target DEPENDS ${LIB_FILE})
# create an library target out of the library compilation result
add_library(lib2 STATIC IMPORTED GLOBAL)
add_dependencies(lib2 lib2_target)
# specify where the library is and where to find the headers
set_target_properties(lib2
PROPERTIES
IMPORTED_LOCATION ${LIB_FILE}
INTERFACE_INCLUDE_DIRECTORIES ${LIB_HEADER_FOLDER})
Now in a CMakeLists.txt I can do somthing like
add_subdirectory(${ROOT_DIR}/lib1 bin/lib1)
add_subdirectory(${ROOT_DIR}/lib2 bin/lib2)
add_executable(app app.c )
target_link_libraries(app lib1 lib2)
No need to specify where the .a and the .h are.
You can use add_library() and tell that it actually imported. Then, using set_target_properties() you can set required INTERFACE_XXX properties for it. After that, you can use it as an ordinal target like every other built by your project.
Thank you for posting the solution. I have wrapped your snippet in a function:
function(add_external_library)
set(options)
set(oneValueArgs TARGET WORKING_DIRECTORY OUTPUT COMMENT)
set(multiValueArgs COMMAND INCLUDE_DIRS)
cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" ${multiValueArgs}" ${ARGN})
# Specify how to build the result of the library
add_custom_command(OUTPUT "${ARGS_OUTPUT}"
COMMAND ${ARGS_COMMAND}
WORKING_DIRECTORY "${ARGS_WORKING_DIRECTORY}"
COMMENT "${ARGS_COMMENT}")
# Create a target out of the library compilation result
add_custom_target(${ARGS_TARGET}_target DEPENDS ${ARGS_OUTPUT})
# Create an library target out of the library compilation result
add_library(${ARGS_TARGET} STATIC IMPORTED GLOBAL)
add_dependencies(${ARGS_TARGET} ${ARGS_TARGET}_target)
# Specify where the library is and where to find the headers
set_target_properties(${ARGS_TARGET}
PROPERTIES
IMPORTED_LOCATION "${ARGS_OUTPUT}"
INTERFACE_INCLUDE_DIRECTORIES "${ARGS_INCLUDE_DIRS}")
endfunction()
# Example
add_external_library(TARGET YourLib
COMMAND /bin/bash compile_your_lib.sh
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT "output/yourlib.a"
INCLUDE_DIRS "include/a" "include/b"
COMMENT "Building YourLib")
add_executable(YourExe)
target_link_libraries(YourExe YourLib)