what is the proper way to set CUDA_VISIBLE_DEVICES in cmake file to use different GPU core - cmake

Question

I have a project that uses cmake for compilation and unittest afterwards. The standard procedure for it is
cmake .. -DUSE_CUDA=ON ; make ; make test ARGS="-j 10"
The problem is that during the make test phase, I have 4 GPUs on my server and only one GPU is used. I can see it through nvidia-smi command. I'm wondering if there is a method to set in Cmakefiles to change the GPU being used and eventually utilize all GPUs.
Here is the unit test code that will execute all the test cases. I tried to force it on GPU 1 by setting environment variable CUDA_VISIBLE_DEVICES to 1. However the code still runs on GPU core 0.
function(py_test TARGET_NAME)
set(options "")
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS ARGS ENVS)
set(ENV{CUDA_VISIBLE_DEVICES} 1)
cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
get_filename_component(work_dir ${py_test_SRCS} DIRECTORY)
add_test(NAME ${TARGET_NAME}
COMMAND ${COVERAGE_EXECUTABLE} run --parallel-mode --source=test_module "${py_test_SRCS}" ${py_test_ARGS}
WORKING_DIRECTORY ${work_dir})
endfunction()
function(add_files)
set(options "")
set(oneValueArgs "")
set(multiValueArgs SRCS)
cmake_parse_arguments(add_files "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
foreach(test_file ${add_files_SRCS})
get_filename_component(test ${test_file} NAME_WE)
get_filename_component(test_abs ${test_file} ABSOLUTE)
message(STATUS "test_file:${test_file}")
py_test(${test} SRCS ${test_abs})
endforeach()
endfunction()
# Put cases cost more time before the less ones
set(COVERAGE_EXECUTABLE /usr/local/bin/coverage-3.6)
# unit test
file(GLOB UNIT_TEST unit_test_folder/test_*.py)
add_files(SRCS ${UNIT_TEST})
I've tried to set CUDA_VISIBLE_DEVICES in the command terminal before running cmake test and it worked.
Also, do I have to preassign the tests to different dedicated GPU or can they be dynamically assigned.

Answer 1

I was able to preassign tests to different GPUs with code like this:
function(py_test TARGET_NAME)
set(options "")
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS ARGS ENVS)
MATH(EXPR CUDA_ID "(${CUDA_ID}+1)%4")
set(CUDA_ID ${CUDA_ID} PARENT_SCOPE)
cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
get_filename_component(work_dir ${py_test_SRCS} DIRECTORY)
add_test(NAME ${TARGET_NAME}
COMMAND ${CMAKE_COMMAND} -E env CUDA_VISIBLE_DEVICES=${CUDA_ID} ${COVERAGE_EXECUTABLE} run --parallel-mode --source=horizon_nn "${py_test_SRCS}" ${py_test_ARGS}
WORKING_DIRECTORY ${work_dir})
message(STATUS CUDA_ID ${CUDA_ID})
endfunction()
set(CUDA_ID 0)
But still, dynamic assign is still not achievable since all these tests are preassigned at cmake phase