How can I write a CMake test that will show build errors? - testing

I currently have this ingenious code that adds my single-source-file tests to my CMake project:
function(runtime_test test_dir test_name)
add_executable(${test_name} EXCLUDE_FROM_ALL ${TEST_ROOT}/${test_dir}/${test_name}.c++ ${TEST_ROOT}/test.h++)
set_target_properties(${test_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY test/${test_dir})
#add_test(remove/${test_dir}/${test_name} ${CMAKE_COMMAND} -E remove test/${test_dir}/${test_name}${CMAKE_EXECUTABLE_SUFFIX})
#add_test(build/${test_dir}/${test_name} ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target ${test_name})
#add_test(test/${test_dir}/${test_name} test/${test_dir}/${test_name})
add_test(${test_dir}/${test_name} ${CMAKE_COMMAND} -E remove test/${test_dir}/${test_name}${CMAKE_EXECUTABLE_SUFFIX}
&& ${CMAKE_COMMAND} --verbose --build ${CMAKE_BINARY_DIR} --target ${test_name}
&& test/${test_dir}/${test_name})
set_property(TEST ${test_dir}/${test_name} APPEND PROPERTY DEPENDS build/${test_dir}/${test_name})
endfunction()
The commented lines add 3 "tests" which remove, build and run an executable. I run a single test as follows:
ctest --verbose -R some_test
With the commented 3 lines, I can see the build errors output to my screen. Problem is: this makes the number of tests non-representative. If I use the 3-in-1 solution, the build error output is hidden. Is there any way to solve this?

Related

Force custom command to always run without first deleting the output

I have a setup where I use a custom command to check the current hash of a git repository so that other commands can clone it if it has updated
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND ${GIT_EXECUTABLE} ls-remote ${MODULE_URL} master > ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND ${CMAKE_COMMAND} -E rm ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp
)
Of course this will only run once as CMake sees no reason to rerun it. I can force it to run by adding a second (dummy) output - CMake then recognises that this output doesn't exist and then reruns the rule. However the Makefile that this generates actually deletes module_VERSION.txt before running the command rendering the whole pursuit pointless (Ninja does not have this problem).
I am able to get this to work but in an extremely hacky way: creating another target that always runs and then generating a dependency on this.
# Use echo_append as a no-op
add_custom_command(
OUTPUT module_FORCERUN
COMMAND ${CMAKE_COMMAND} -E echo_append
)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND ${GIT_EXECUTABLE} ls-remote ${MODULE_URL} master > ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND ${CMAKE_COMMAND} -E rm ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp
DEPENDS module_FORCERUN
)
This seems just really hacky and like it could be relying on some corner cases in cmake which aren't guaranteed to be stable. Is there a better way to get this working?
I am using cmake 3.21.3
Use add_custom_target for implement "always run" functionality and via BYPRODUCTS keyword specify the file which it could produce/update:
add_custom_target(update_module_version
BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND ${GIT_EXECUTABLE} ls-remote ${MODULE_URL} master > ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND ${CMAKE_COMMAND} -E rm ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt.tmp
)
That way, if any other target will depend on update_module_version one, the module_VERSION.txt file will be created/updated before evaluation of the target.
Such target-level dependency will be created automatically by CMake, if given file will be listed as dependency for target/command in the same directory, where target update_module_version is created:
add_custom_command(OUTPUT <...>
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
COMMAND <...>
)
From other directories the target-level dependency should be specified explicitly:
# If used in other directories
add_custom_command(OUTPUT <...>
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/module_VERSION.txt
update_module_version
COMMAND <...>
)

Passing environment variable to the COMMAND in CMake execute_process

I have the following CMake snippet that runs COMMAND in WORKING_DIRECTORY. I tried different ways to pass the environment variable (MBEDTLS_INCLUDE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/../mbedtls/mbedtls/include) but without success.
The snippet that works (without env variable):
set(BUILD_CMD cargo build --features parsec-client/no-fs-permission-check)
set(WORKING_DIR "${CMAKE_CURRENT_SOURCE_DIR}/parsec_se_driver")
execute_process( COMMAND ${BUILD_CMD}
RESULT_VARIABLE CMD_ERROR
WORKING_DIRECTORY ${WORKING_DIR} )
if(NOT ${CMD_ERROR} MATCHES "0")
MESSAGE(SEND_ERROR "BUILD_CMD STATUS:" ${CMD_ERROR})
endif()
How can I pass the env variable to the execute_process?
If I write something like this:
execute_process( COMMAND MBEDTLS_INCLUDE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/../mbedtls/mbedtls/include cargo build --features parsec-client/no-fs-permission-check
RESULT_VARIABLE CMD_ERROR
WORKING_DIRECTORY ${WORKING_DIR} )
or taking different parts to variables, or adding quotes, I get:
BUILD_CMD STATUS:No such file or directory
As recommended in the CMake mailing list here, your solution using set(ENV ...) is perfectly valid:
set(ENV{MBEDTLS_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../mbedtls/mbedtls/include)
execute_process(
COMMAND ${BUILD_CMD}
RESULT_VARIABLE CMD_ERROR
WORKING_DIRECTORY ${WORKING_DIR}
)
You could also use CMake's command line utility to run the command in a modified environment using cmake -E env:
execute_process(
COMMAND ${CMAKE_COMMAND} -E env
MBEDTLS_INCLUDE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../mbedtls/mbedtls/include" ${BUILD_CMD}
RESULT_VARIABLE CMD_ERROR
WORKING_DIRECTORY ${WORKING_DIR}
)

How to append command to add_custom_target in CMake

Suppose I have a custom target in CMake for unit tests like the below
add_custom_target(
test
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ATest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/BTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/CTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/DTest)
but I want to add an additional test to the target based on whether an external dependency is found. Currently, I did it with
if(EXTERNAL_FOUND)
add_custom_target(
test
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ATest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/BTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/CTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/DTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ETest)
else()
add_custom_target(
test
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ATest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/BTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/CTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/DTest)
endif()
This is not very elegant and it quickly becomes unmanageable when there are multiple conditions. Is there something like append to custom target so we can write the below instead?
add_custom_target(
test
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ATest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/BTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/CTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/DTest)
if(EXTERNAL_FOUND)
# I can't seem to find something like this
append_custom_target(test COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ETest)
else()
Or is there a better way to do this?
You can use add_custom_command and use it as dependency to your target. With the custom command you can APPEND commands with same OUTPUT:
add_custom_target(
test
DEPENDS test-cmd
)
add_custom_command(
OUTPUT test-cmd
COMMAND ${CMAKE_COMMAND} -E echo "ATest"
COMMAND ${CMAKE_COMMAND} -E echo "BTest"
COMMAND ${CMAKE_COMMAND} -E echo "CTest"
COMMAND ${CMAKE_COMMAND} -E echo "DTest"
)
if(EXTERNAL_FOUND)
add_custom_command(
OUTPUT test-cmd APPEND
COMMAND ${CMAKE_COMMAND} -E echo "ETest"
)
endif()
# test-cmd is not actually generated so set it to symbolic
set_source_files_properties(test-cmd PROPERTIES SYMBOLIC "true")
See SYMBOLIC for the artifical source file property.

Have CMake not complain about non-existant file?

I know there probably isn't any workaround to this, but I need to generate a source file dllmain.c for my model.dll. This is done together with an executable that extracts some essential information for me, so my current CMakeLists looks like this
add_executable(main ${SOURCE} otherListSourcesGoHere)
add_custom_command(
TARGET main
POST_BUILD
COMMAND main.exe <--- Main built from above
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/jtox.py
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/dllgen.py <--- Python script generating a
sourcefile for my DLL
COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_DIR}/build/dllmain.c ${PROJECT_DIR}
)
add_library(
${CMAKE_PROJECT_NAME}
SHARED
${PROJECT_DIR}/dllmain.c
otherListSoucrsGoHereAswell
)
But of course, as the dllmain.c file is non-existant before the exe has been completed and ran, I can't use this as it will return an error. Is is possible to execute this somehow without having to run two CMakeLists?
EDIT 1
With the help of #Angew I found out that you could specify the output of a file in a add_custom_command, so currently I have this instead
add_executable(main ${SOURCE} otherListSourcesGoHere)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/dllmain.c
COMMAND main
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/jtox.py
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/dllgen.py -o ${CMAKE_CURRENT_BINARY_DIR}/dllmain.c
)
add_library(
${CMAKE_PROJECT_NAME}
SHARED
${CMAKE_CURRENT_BINARY_DIR}/dllmain.c
otherListSoucrsGoHereAswell
)
But I have the error
CMakeFiles\bil.dir\build.make:60: recipe for target '../dllmain.c' failed
CMakeFiles\Makefile2:66: recipe for target 'CMakeFiles/bil.dir/all' failed
makefile:82: recipe for target 'all' failed
I don't seem to find a solution. How can I go about debugging this issue?
EDIT 2
I fixed this issue by adding DEPENDS main on add_custom_command
add_custom_command(
OUTPUT ${PROJECT_DIR}/dllmain.c
DEPENDS main
COMMAND echo "Executing main.exe"
COMMAND main.exe
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND echo "JSON to XML with Python"
COMMAND python ${PYTHON_SOURCE_DIR}/jtox.py
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/dllgen.py -o ${PROJECT_DIR}
)
The correct way to do this is to tell CMake how to generate the file instead of doing it "manually" and keeping it secret from CMake. To do this, change your CMakeList like this:
add_executable(main ${SOURCE} otherListSourcesGoHere)
add_custom_command(
OUTPUT ${PROJECT_DIR}/dllmain.c
COMMAND main
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/jtox.py
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/dllgen.py
COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_DIR}/build/dllmain.c ${PROJECT_DIR}
)
add_library(
${CMAKE_PROJECT_NAME}
SHARED
${PROJECT_DIR}/dllmain.c
otherListSoucrsGoHereAswell
)
This way, CMake will know that the file ${PROJECT_DIR}/dllmain.c is generated, and also how to generate it. It will correctly replace main with the executable built from target main, and introdce a proper build depencency.
Side note: you should consider modifying dllgen.py so that it's able to generate a file in a directory of your choice, and have it generate into the binary directory. That way, you will not pollute your source tree with build artefacts, which is a very desirable property: it allows you to revert to pristine state just by removing the binary dir. With that change, the CMakeList could look like this:
add_executable(main ${SOURCE} otherListSourcesGoHere)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/dllmain.c
COMMAND main
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/jtox.py
COMMAND ${CMAKE_COMMAND} -E sleep 1
COMMAND python ${PYTHON_SOURCE_DIR}/dllgen.py -o ${CMAKE_CURRENT_BINARY_DIR}/dllmain.c
)
add_library(
${CMAKE_PROJECT_NAME}
SHARED
${CMAKE_CURRENT_BINARY_DIR}/dllmain.c
otherListSoucrsGoHereAswell
)

CMake add_custom_command ('POST_BUILD') 'DEPENDS' option is ignored

I have a library and a test projects on CMake, and I'm using this directory structure with two (project) CMakeLists.txt:
/
|- CMakeLists.txt
|- include/libName
|- src/...
|
|- test/
|- CMakeLists.txt
|- src/...
The outer project list defines the library, like:
add_library(libName ${SRC} ${INCLUDE})
And adds 'test' as subdirectory:
add_subdirectory(test)
The test project list defines the executable and a test, like:
add_executable(NameTest ${SRC})
target_link_libraries(NameTest libName)
add_test(NAME NameTest COMMAND NameTest)
The problem
I'm trying to build and execute the test program when the library is built. If any test fails, I want the build of the library fail too.
This is what I have (inside the outer lists file):
add_custom_command(
TARGET libName
POST_BUILD
COMMAND CTEST_OUTPUT_ON_FAILURE=1 ctest
DEPENDS NameTest # <- This is driving me crazy!
)
This command ignores completely if the target 'NameTest' is built, if there is a file with that name, or if not. I can't notice any difference if the whole 'DEPENDS' option is removed.
I even modified like:
add_custom_command(
TARGET libName
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E echo "Bip! Bip! Bip!"
DEPENDS this_is_not_an_existent_file_nor_target
)
And the command is triggered anyway. I'm not very sure about if this is the option I need, so:
Why is this not working?
How can I achieve my real purpose?
Thank you.
Edit: ctest will execute every test (add_test), but the NameTest executable (yet listed) must be built before calling it! Now would be built after the library, but before the 'POST_BUILD' custom command. It fails, of course.
I want CMake realize NameTest is necessary for running that custom command.
Edit: I find useful the Angew's answer, so I accepted his answer and refined it a little bit:
add_custom_command(
TARGET libName
POST_BUILD
COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target NameTest --config $<CONFIG>
COMMAND ${CMAKE_CTEST_COMMAND} -C $<CONFIG> --output-on-failure
)
Thank you!
1. Why is this not working?
Because you're mixing options from two distinct signatures of add_custom_command. DEPENDS comes from the form which is used to generate a file. TARGET and POST_BUILD are from the form which adds pre/post build commands to existing targets.
See the documentation of add_custom_command for more details on the two uses.
2. How can I achieve my real purpose?
I believe the following should do what you want to:
add_custom_command(
TARGET libName
POST_BUILD
COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target NameTest --config $<CONFIG>
COMMAND CTEST_OUTPUT_ON_FAILURE=1 ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target test --config $<CONFIG>
)