cmake ctest post test delete file - cmake

I have a cmake project that runs through a number of tests, i.e.
add_test(test_title executable arg1 arg2)
On runing these tests a number of files are produced.
Once the test has run I would like to delete one of these files produced, i.e.
delete(${arg1}.txt)
or
delete(${arg2}.pdf)
If you could provide an example it would be very much appreciated.

As advised in the letter, уou can wrap actual testing command with cmake script. In your case it can be:
add_test(NAME test_title
COMMAND ${CMAKE_COMMAND}
-Darg1=${arg1}
-Darg2=${arg2}
-P ${CMAKE_CURRENT_SOURCE_DIR}/runtest.cmake
)
And the wrapper runtest.cmake can be:
execute_process(COMMAND executable ${arg1} ${arg2}
TIMEOUT 1000 # it should be less than in add_test
RESULT_VARIABLE status
)
file(REMOVE ${arg1}.txt)
file(REMOVE ${arg2}.txt)
if(status)
MESSAGE(FATAL_ERROR "Test executing status: ${status}")
endif()

I am not sure whether this option was available at the time of asking the question, but I used a hack along the lines of:
define a test: test_title
define another test: test_title_remove_file
depends on test_title (so test_title will be run first)
requires files you want to delete to exist (explicit is better than implicit)
calls cmake -E remove <file> doc
This way you'll have one fake extra test but at least it will tell you if there are any errors.
My implementation:
add_test(NAME test_title_remove_file
COMMAND ${CMAKE_COMMAND} -E remove
${arg1}
${CMAKE_CURRENT_BINARY_DIR}/${arg2}
)
set_tests_properties(test_title_remove_xmls PROPERTIES
DEPENDS test_title ## note it is your test_title
REQUIRED_FILES
${arg1};
${CMAKE_CURRENT_BINARY_DIR}/${arg2};
)

Related

CMake add_custom_command() POST_BUILD generator expression expansion not working

I want to run a POST_BUILD action after the build (but only in the Debug configuration).
After reading add_custom_command docs and a possible solution I understood that I can "wrap" my COMMAND into $<CONFIG:Debug> generator expression (to be sure it's "empty" in Release mode).
I tried the following:
cmake_minimum_required(VERSION 3.18)
project(post-build CXX)
file(WRITE main.cxx "int main() {}")
add_executable(foo main.cxx)
add_custom_command(
TARGET foo POST_BUILD
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND} -E echo "hi there from debug build">
)
But this gives me the CMake configure-time warnings and a hard failure during a build-time (using Ninja generator):
(...) && "$<1:C:\Program Files\CMake\bin\cmake.exe" -E echo "hi there from debug build" >""
[build] The system cannot find the path specified.
[build] ninja: build stopped: subcommand failed.
[build] Build finished with exit code 1
I tried many possible quotes combinations (including escaped quotes):
COMMAND $<$<CONFIG:Debug>:"${CMAKE_COMMAND} -E echo \"hi there from debug build\"">
and
COMMAND "$<$<CONFIG:Debug>:${CMAKE_COMMAND} -E echo \"hi there from debug build\">"
etc.
But even though it removed the configure-time warning, it still yields a hard error during the build-time.
Question: What would be the correct way to achieve what I want? Is it possible like this or there is a CMake limitation here?
(Note: if possible I'd like to keep the whole command be executed in one place. I am also aware of other workaround possible)
Following the answer of Ben Boeckel here:
Spaces generally aren’t well-formed inside of genexes. You’ll need to replace the spaces with ; to make it parse properly (which is why you’re seeing half-expanded remnants in the build command).
And some discussion in the CMake mailing list (here), what finally worked for me was:
add_custom_command(
TARGET foo POST_BUILD
COMMAND "$<$<CONFIG:Debug>:${CMAKE_COMMAND};-E;echo;\"hi there from debug build\">"
COMMAND_EXPAND_LISTS
)
(Notice the quotes aroung the whole genex, separation with semicolons, backquoting the string, and COMMAND_EXPAND_LISTS to get rid of semicolons in the output -- all-in-all definitely not the most pleasing thing to read)
Edit:
This also works:
set(HELLO_FROM_DEBUG ${CMAKE_COMMAND} -E echo "hi there")
add_custom_command(
TARGET foo POST_BUILD
COMMAND "$<$<CONFIG:Debug>:${HELLO_FROM_DEBUG}>"
COMMAND_EXPAND_LISTS
)

Continue after dependent target fails in custom CMake target

I have a regression test suite consisting of multiple custom targets created with add_custom_target().
Moreover there is a "convenience" target regressions to run all regressions. It simply contains all single regression targets as dependency.
This is reflected in the following MCVE:
cmake_minimum_required(VERSION 3.17)
project (Regressions)
add_custom_target(reg_1 COMMAND ${CMAKE_COMMAND} -E echo 'First regression')
add_custom_target(reg_2 COMMAND ${CMAKE_COMMAND} -E echo 'Second regression')
# ...
add_custom_target(regressions DEPENDS reg_1 reg_2)
Now I can run cmake --build . --target regressions from the build directory and reg_1 and reg_2 are run as part of regressions.
My problem is that if one of the regressions fail, the remaining are not executed. But of course I want to always run all regressions and only have a summary of the failed ones. How can I achieve this behavior, i.e. always execute all subtargets, no matter whether some of them fail?
I assume that the natural way to do this is to use add_test() (after all regressions runs are tests), but I failed because the custom targets are no executables and AFAIK you cannot use custom CMake targets with add_test().
Please feel free to recommend an alternative to my current approach. If I could handle everything using ctest that would be preferred anyway.
Thanks to #KamilCuk's answer, I realized my problem of not being able to add custom targets as tests is not really a problem.
I can invoke CMake with add_test(), and the cmake command can run custom targets.
Adding:
enable_testing()
add_test(NAME regression1 COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target reg_1)
add_test(NAME regression2 COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target reg_2)
to the code above solves the problem. Additionally, instead of depending on the single custom targets, regression can simply invoke ctest -R "regression*" to invoke all (and only) regression targets, in case other tests exist in the CMake project, like this:
add_custom_target(regressions COMMAND ${CMAKE_CTEST_COMMAND} -R "regression*")

cmake compare output of two executables [duplicate]

I have a console application called "foo", which takes a reference text file as input (in.txt) and generates text at standard output (I want to keep this behaviour).
In make (not cmake), I use a test target, which calls foo and redirects the output to a file (out.txt) as follows. Then, I use diff to compare the file out.txt with the expected refernece (ref.txt)
test:
./foo -a test/in.txt > test/out.txt
diff test/out.txt test/ref.txt
This works fine using make. Now my question is; how can I use cmake to create a similar Makefile?
From within a subdrectory called build, I tried
project(foo)
...
add_test(NAME test1 COMMAND ./foo ../test/in.txt > ../test/out.txt)
enable_testing()
Using cmake version 3.5, I get a Makefile without errors, but when I call make test, the test itself fails. It seems the cmake command add_test supports command line arguments, but not the redirection. I tried quotes and escaping witout success. Since I could not pass this part, I didn't try to use diff. I just imagine that I could pack foo and diff in one line using & as you can do with bash. That would be the second step.
Turning my comment into an answer
As #Tsyvarev has stated, CTest commands are not run in a shell's context. But you could just add the shell needed yourself and use e.g. sh as the command to be called with add_test().
I've run some tests with your example code and the following did work successfully:
add_test(NAME test1 COMMAND sh -c "$<TARGET_FILE:foo> ../test/in.txt > ../test/out.txt")
This solution is not platform independent (it depends on sh to be available in the search paths).
So if you want to be more flexible you could do something like:
include(FindUnixCommands)
file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/in.txt" _in)
file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/out.txt" _out)
if (BASH)
add_test(
NAME test1
COMMAND ${BASH} -c "$<TARGET_FILE:foo> ${_in} > ${_out}"
)
else()
if (WIN32)
add_test(
NAME test1
COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:foo> $ENV{ComSpec} /c "$<TARGET_FILE_NAME:foo> ${_in} > ${_out}"
)
else()
message(FATAL_ERROR "Unknown shell command for ${CMAKE_HOST_SYSTEM_NAME}")
endif()
endif()
Additionally there is the possibility to execute a more platform independent diff with ${CMAKE_COMMAND} -E compare_files <file1> <file2>. So you could simplify your complete makefile based example in CMake with:
add_custom_command(
TARGET foo
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E echo "Running $<TARGET_FILE_NAME:foo> ..."
COMMAND foo in.txt > out.txt
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test
)
add_test(
NAME test1
COMMAND ${CMAKE_COMMAND} -E compare_files in.txt out.txt
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test
)
References
Integrate bash test scripts in cmake
CMake: piping commands to executable
cmake: make tests successfully passing part of the build process
They say you cannot:
There is no redirection of output using add_test arguments.
Unlike to commands in add_custom_command, which are executed as a part of makefile receipts (that is, in the context of some shell), tests are executed directly by CTest, without any shell involved. So, shell mechanisms don't work for tests.
You may create wrapper script, which calls program, given as parameter, and performs redirection, futher diff and so on. Then use this script (with appropriate arguments) as a COMMAND for add_test.

How to use redirection in cmake add_test

I have a console application called "foo", which takes a reference text file as input (in.txt) and generates text at standard output (I want to keep this behaviour).
In make (not cmake), I use a test target, which calls foo and redirects the output to a file (out.txt) as follows. Then, I use diff to compare the file out.txt with the expected refernece (ref.txt)
test:
./foo -a test/in.txt > test/out.txt
diff test/out.txt test/ref.txt
This works fine using make. Now my question is; how can I use cmake to create a similar Makefile?
From within a subdrectory called build, I tried
project(foo)
...
add_test(NAME test1 COMMAND ./foo ../test/in.txt > ../test/out.txt)
enable_testing()
Using cmake version 3.5, I get a Makefile without errors, but when I call make test, the test itself fails. It seems the cmake command add_test supports command line arguments, but not the redirection. I tried quotes and escaping witout success. Since I could not pass this part, I didn't try to use diff. I just imagine that I could pack foo and diff in one line using & as you can do with bash. That would be the second step.
Turning my comment into an answer
As #Tsyvarev has stated, CTest commands are not run in a shell's context. But you could just add the shell needed yourself and use e.g. sh as the command to be called with add_test().
I've run some tests with your example code and the following did work successfully:
add_test(NAME test1 COMMAND sh -c "$<TARGET_FILE:foo> ../test/in.txt > ../test/out.txt")
This solution is not platform independent (it depends on sh to be available in the search paths).
So if you want to be more flexible you could do something like:
include(FindUnixCommands)
file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/in.txt" _in)
file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/out.txt" _out)
if (BASH)
add_test(
NAME test1
COMMAND ${BASH} -c "$<TARGET_FILE:foo> ${_in} > ${_out}"
)
else()
if (WIN32)
add_test(
NAME test1
COMMAND ${CMAKE_COMMAND} -E chdir $<TARGET_FILE_DIR:foo> $ENV{ComSpec} /c "$<TARGET_FILE_NAME:foo> ${_in} > ${_out}"
)
else()
message(FATAL_ERROR "Unknown shell command for ${CMAKE_HOST_SYSTEM_NAME}")
endif()
endif()
Additionally there is the possibility to execute a more platform independent diff with ${CMAKE_COMMAND} -E compare_files <file1> <file2>. So you could simplify your complete makefile based example in CMake with:
add_custom_command(
TARGET foo
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E echo "Running $<TARGET_FILE_NAME:foo> ..."
COMMAND foo in.txt > out.txt
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test
)
add_test(
NAME test1
COMMAND ${CMAKE_COMMAND} -E compare_files in.txt out.txt
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test
)
References
Integrate bash test scripts in cmake
CMake: piping commands to executable
cmake: make tests successfully passing part of the build process
They say you cannot:
There is no redirection of output using add_test arguments.
Unlike to commands in add_custom_command, which are executed as a part of makefile receipts (that is, in the context of some shell), tests are executed directly by CTest, without any shell involved. So, shell mechanisms don't work for tests.
You may create wrapper script, which calls program, given as parameter, and performs redirection, futher diff and so on. Then use this script (with appropriate arguments) as a COMMAND for add_test.

Using CMake, how do I get verbose output from CTest?

I'm using CMake to build my project. I have added a unit test binary which is using the Boost unit testing framework. This one binary contains all of the unit tests. I've added that binary to be run by CTest:
ADD_EXECUTABLE( tftest test-main.cpp )
ENABLE_TESTING()
ADD_TEST( UnitTests tftest)
But the build output in Visual Studio only shows the result of running CTest:
Start 1: UnitTests
1/1 Test #1: UnitTests ................***Failed 0.05 sec
0% tests passed, 1 tests failed out of 1
This is not very helpful, because I can't see which test failed. If I run ctest manually from the command line with --verbose I get the output from a Boost unit test which tells what actually failed:
1: Test command: tftest.exe
1: Test timeout computed to be: 9.99988e+006
1: Running 4 test cases...
1: test-main.cpp(20): error in "sanity_check3": check 1 == 2 failed
1:
1: *** 1 failure detected in test suite "Master Test Suite"
1/1 Test #1: UnitTests ................***Failed 0.00 sec
So, what do I need to change in the CMakeLists.txt to have CTest run with --verbose at all times? Is there a better way to use Boost unit tests with CMake/CTest?
You can use the ctest --output-on-failure option, or set the environment variable CTEST_OUTPUT_ON_FAILURE, which will show you any output from the test program whenever the test fails. One way to do this when using Makefiles and the command line would be as follows:
env CTEST_OUTPUT_ON_FAILURE=1 make check
This Stack Overflow question and answer shows how to set environment variables in Visual Studio.
You could call ctest directly, after cmaking and making your project.
ctest --verbose
There is a very simple solution (which for some reason is difficult to find via Google Search):
ctest --output-on-failure
If you use CMake with Visual Studio's open folder function you can add the
"ctestCommandArgs": "--output-on-failure"
setting to your build configuration.
You can check the Testing/Temporary subfolder. It is automatically created after running make test. This folder contains two files: LastTest.log and LastTestsFailed.log. LastTest.log contains desired output for run tests. LastTestFailed.log contains names of failed tests. So you can check them manually after executing make test.
The second way is to get ctest to show you the content of log files after running tests:
place in build dir (from which you run make test) file CTestCustom.ctest (you can do it with configure file command, for example) with following contents
CTEST_CUSTOM_POST_TEST("cat Testing/Temporary/LastTest.log")
Instead of cat you may use whatever Windows cmd command that does similar things.
run make test again and get profit!
additional info about customizing ctest you can find here. Just step to "Customizing cmake" section.
Good luck!
I had to add "check" target by myself. "make tests" does nothing by some reason. So what I did (as was suggest somewhere on stackoverflow) - I added this target manually. To get verbose output I just wrote it like:
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --verbose)
make check CTEST_OUTPUT_ON_FAILURE=TRUE
This makes test output more verbose:
make test ARGS="-V"
My approach is a combination of the answers from ony, from zbyszek, and from tarc. I use the ${CMAKE_COMMAND} variable (which is set to the absolute path to the invoked cmake executable) with the -E env CTEST_OUTPUT_ON_FAILURE=1 argument to invoke the actual ctest command using ${CMAKE_CTEST_COMMAND} -C $<CONFIG>. To help clarify what is going on, I start with three cmake -E echo commands to show the current working directory and the ctest command to be invoked. Here is how I call add_custom_target.
add_custom_target(check
${CMAKE_COMMAND} -E echo CWD=${CMAKE_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E echo CMD=${CMAKE_CTEST_COMMAND} -C $<CONFIG>
COMMAND ${CMAKE_COMMAND} -E echo ----------------------------------
COMMAND ${CMAKE_COMMAND} -E env CTEST_OUTPUT_ON_FAILURE=1
${CMAKE_CTEST_COMMAND} -C $<CONFIG>
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
DEPENDS ALL_BUILD
)
This plays nice with the MSVC IDE where any test errors are shown as clickable compilation errors. See cmake -E env for documentation of the cmake -E portable command line tool mode. I also add a dependency on ALL_BUILD so that all projects will be built before invoking the check target. (On Linux builds, one may need to replace ALL_BUILD with ALL; I have not tested this on Linux yet.)
For people using Visual Studio, here another variation (hack) on the theme:
cmake -E env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target RUN_TESTS
ctest -VV or ctest --extra-verbose
From documentation:
Enable more verbose output from tests.
Test output is normally suppressed and only summary information is
displayed. This option will show even more test output.
There's now a CMake variable that allows you to modify the behaviour of make test. CMAKE_CTEST_ARGUMENTS lets you set a list of arguments to pass to ctest when run via make test.
So adding this to your CMake file:
set(CMAKE_CTEST_ARGUMENTS "--verbose")
Means CTest will always run verbose. Or for just the output of the failed tests, use:
set(CMAKE_CTEST_ARGUMENTS "--output-on-failure")
Edit:
As suggested by RobLoach, since it's a list of arguments, you'll want to append to the list instead.
list(APPEND CMAKE_CTEST_ARGUMENTS "--output-on-failure")
to show the result with XML file you have to execute the test with the following command
~$ ctest -T Test
and we found the result in the Testing/1234123432/test.xml
and other files are generated too in Testing Folder