Dealing with lots and lots of escaped characters in add_custom_command - cmake

I have a file that contains a bunch of data. I want to turn it into a C++ string literal, because I need to compile this data into the binary - I cannot read it from disk.
One way of doing this is to just generate a C++ source file that declares a string literal with a known name. The CMake code to do this is straightforward, if somewhat awful:
function(make_literal_from_file dest_file source_file literal_name)
add_custom_command(
OUTPUT ${dest_file}
COMMAND printf \'char const* ${literal_name} = R\"\#\(\' > ${dest_file}
COMMAND cat ${source_file} >> ${dest_file}
COMMAND printf \'\)\#\"\;\' >> ${dest_file}
DEPENDS ${source_file})
endfunction()
This works and does what I want (printf is necessary to avoid a new line after the raw string introducer). However, the amount of escaping going on here makes it very difficult to see what's going on. Is there a way to write this function such that it's actually readable?
Note that I cannot use a file(READ ...)/configure_file(...) combo here because source_file could be something that is generated by CMake at build time and so may not be present at configuration time.

I would recommend writing a script to do this. You could write it in CMake, but I personally prefer a better language such as Python:
# Untested, just to show roughly how to do it
import sys
dest_file, source_file, literal_name = sys.argv[1:]
with open(dest_file) as dest, open(source_file) as source:
literal_contents = source.read()
dest.write(f'char const* {literal_name} = R"({literal_contents})";\n')
Corresponding CMake code:
# String interpolation came in Python 3.6, thus the requirement on 3.6.
# If using CMake < 3.12, use find_package(PythonInterp) instead.
find_package(Python3 3.6 COMPONENTS Interpreter)
# Make sure this resolves correctly. ${CMAKE_CURRENT_LIST_DIR} is helpful;
# it's the directory containing the current file (this cmake file)
set(make_literal_from_file_script "path/to/make_literal_from_file.py")
function(make_literal_from_file dest_file source_file literal_name)
add_custom_command(
OUTPUT "${dest_file}"
COMMAND
"${Python3_EXECUTABLE}" "${make_literal_from_file_script}"
"${dest_file}"
"${source_file}"
"${literal_name}"
DEPENDS "${source_file}")
endfunction()
If you don't want the dependency on Python, you could use C++ (only the CMake code shown):
add_executable(make_literal_from_file_exe
path/to/cpp/file.cpp
)
function(make_literal_from_file dest_file source_file literal_name)
add_custom_command(
OUTPUT "${dest_file}"
COMMAND
make_literal_from_file_exe
"${dest_file}"
"${source_file}"
"${literal_name}"
DEPENDS "${source_file}")
endfunction()

Related

Standalone CMake script to cut off file contents by delimiters

I have a project where one repeatable task to do involves manipulating files' contents.
Until now I used a Python script for it, but recently I discovered I can use standalone CMake scripts ("standalone" here means they can be invoked outside of configure/build/test/etc. workflow). As my project already uses CMake for project management I concluded I can save others' problem of installing a Python interpreter (welcome Windows users!) and use CMake project-wide.
Part of my script needs to read a file and cut off everything that appears before "[START-HERE]" and after "[END-HERE]" lines. I am stuck with that part and don't know how to implement it. How can it be done?
You could combine file(READ) with if(MATCHES) to accompilish this. The former is used to read the file, the latter allows you to check for the occurance of a regular expression and to extract a capturing group:
foo.cmake
#[===[
Params:
INPUT_FILE : the path to the file to read
#]===]
file(READ "${INPUT_FILE}" FILE_CONTENTS)
if (FILE_CONTENTS MATCHES "(^|[\r\n])\\[START-HERE\\][\r\n]+(.*)[\r\n]+\\[END-HERE\\]")
# todo: use extracted match stored in CMAKE_MATCH_2 for your own logic
message("Content: '${CMAKE_MATCH_2}'")
else()
message(FATAL_ERROR "[START-HERE]...[END-HERE] doesn't occur in the input file '${INPUT_FILE}'")
endif()
foo.txt
Definetly not
[START-HERE]
working
[END-HERE]
Try again!
Output:
> cmake -D INPUT_FILE=foo.txt -P foo.cmake
Content: 'working'
For the part where you are stuck, here's one approach using the string, file, and math commands:
file(READ data.txt file_str)
string(FIND "${file_str}" "[START-HERE]" start_offset)
# message("${start_offset}")
math(EXPR start_offset "${start_offset}+12")
# message("${start_offset}")
string(FIND "${file_str}" "[END-HERE]" end_offset)
math(EXPR substr_len "${end_offset}-${start_offset}")
# message("${substr_len}")
string(SUBSTRING "${file_str}" "${start_offset}" "${substr_len}" trimmed_str)
# message("${trimmed_str}")
You could also probably do it by using the file(STRINGS) command, which reads lines of a file into an array, and then use the list(FIND) command. The approach shown above has the advantage of working if your delimiters are not on their own lines.
As #fabian shows in their answer post, you can also do this using a regular expression with if(MATCHES) like this:
file(READ "${INPUT_FILE}" FILE_CONTENTS)
if (FILE_CONTENTS MATCHES "(^|[\r\n])\\[START-HERE\\][\r\n]+(.*)[\r\n]+\\[END-HERE\\]")
# todo: use extracted match stored in CMAKE_MATCH_2 for your own logic
message("Content: '${CMAKE_MATCH_2}'")
else()
message(FATAL_ERROR "[START-HERE]...[END-HERE] doesn't occur in the input file '${INPUT_FILE}'")
endif()

cmake 3.15 adding JOB_POOL to add_custom_command SOMETIMES

For users that are using cmake 3.15 or later and are also using Ninja as a generator, I want to set the new JOB_POOL argument to some large add_custom_command() blocks. For other users, I want to keep my add_custom_command() the same (no JOB_POOL).
In earlier steps, I check the version and the generator and set ${JOB_POOLS} and I also set a variable such that users who should use a pool will see (something like):
For historical reasons, I leave this here, although #Tsyvarev
points out that this is the source of my problem!
The double-quotes are NOT wanted here!
set(USE_POOL "JOB_POOL pool_A")
Users that are not using a pool will not have that variable set.
Now how to leverage that variable in my custom command...?
1.) Generator expressions don't work, just including the text with the previous line...
add_custom_command(
...
$<USE_POOL>
)
2.) I can't seem to simply place the variable in the command, again just including the variable contents on the previous line. For example, when ${JOB_POOL} is set to the string "JOB_POOL pool_A", this code...
For historical reasons, I leave this here, although #Tsyvarev
points out that this is the source of my problem!
Don't use a STRING! No double-quotes!
add_custom_command(
OUTPUT foo
DEPENDS bar
# Comment line here...
${USE_POOL}
COMMAND
${CMAKE_COMMAND} -E ...
)
gives this error...
ninja: error: '../path/to/src/dir/JOB_POOL pool_A', needed by 'path/to/src/dir/foo', missing and no known rule to make it
It simply considers the ${JOB_POOL} string to be another dependency!
3.) I can't use the "APPEND" feature of add_custom_command(). It's just ignored...
if (${USE_POOL})
add_custom_command(
...
APPEND
JOB_POOL pool_A
)
endif()
The only thing that seems to work is to put an "if" around my
entire command, which offends my sensibility since I don't like to duplicate so much code...
if(${USE_POOL})
add_custom_command(
...many lines...
JOB_POOL pool_A
)
else()
add_custom_command(
...many lines...
)
endif()
Do you have a better idea...?
Here's a standalone example for #tsyvarev:
cmake_minimum_required(VERSION 3.15)
project foo
set_property(GLOBAL PROPERTY JOB_POOLS pool_A=2)
# For historical reasons, I leave this here, although #Tsyvarev
# points out that this is the source of my problem!
# Don't use a STRING! No double-quotes!
set(USE_POOL "JOB_POOL pool_A")
add_custom_command(
OUTPUT foo.out
DEPENDS foo.in
${USE_POOL}
COMMAND
${CMAKE_COMMAND} -E copy foo.in foo.out
COMMENT "Converting foo.in -> foo.out"
VERBATIM
)
add_custom_target(foo-out
DEPENDS foo.out
)
% cmake -GNinja .
% ninja foo-out
ninja: error: 'JOB_POOL pool_A', needed by 'foo.out', missing and no known rule to make it
It considers the string to be a dependency... If I move the USE_POOL to after the comment, it considers it part of the comment... If I move it to after the command, it considers it part of the command...
Your JOB_POOL option serves for the user's choice. You may create another variable which contains sequence of related parameters for add_custom_command:
if(JOB_POOL)
set(JOB_POOL_PARAMS JOB_POOL pool_A) # Will add 'JOB_POOL pool_A' sequence of params
else()
set(JOB_POOL_PARAMS) # Will add nothing
endif()
Then use new variable directly in add_custom_command call:
add_custom_command(
...
${JOB_POOL_PARAMS} # Will expand to additional parameters when needed
)

Dealing with the separator in CMake

I'm trying to compile some Java code with CMake (I'm aware that Java is not really the use-case for CMake) and I want to provide the class paths for the files. The compilation should work on both Unix and Windows systems. The problem I have is with separating the different class paths. Using:
set(CLASS_PATH ${PATH1} ${PATH2})
message(STATUS "${CLASS_PATH}")
prints
<PATH1>;<PATH2>
But this happens on both Unix and Windows. So I have to manually add separators. The way I'm doing it is
if(${CMAKE_HOST_WIN32})
set(SEP "\;")
elseif(${CMAKE_HOST_UNIX})
set(SEP ":")
endif(${CMAKE_HOST_WIN32})
Is this really the best way to deal with separators? I feel like I'm missing something.
Update - MCVE
To describe my thought: FILE_LIST would be contain all the java files that I want to compile. I defined a custom function which I can call on this FILE_LIST and compile the files. Maybe I'm doing something wrong with the function parameters?
cmake_minimum_required(VERSION 3.11)
set(CLASS_PATH E:/tmp/cmake/separator C:/tmp/)
set(FILE_LIST 1.txt 2.txt 3.txt)
add_custom_target(war ALL)
function(compile_java clp)
foreach(java_file ${ARGN})
add_custom_command(
TARGET war
PRE_BUILD
COMMAND echo "${clp}" ${java_file}
)
endforeach(java_file)
endfunction()
compile_java("${CLASS_PATH}" ${FILE_LIST}) # I have to pass CLASS_PATH in quotes
So, based on comments, you want the path list as a single command-line argument, with a platform-specific separator. You can achieve this using string operations:
function(compile_java clp)
if(NOT CMAKE_HOST_WIN32)
string(REPLACE ";" ":" clp "${clp}")
endif()
foreach(java_file ${ARGN})
add_custom_command(
TARGET war
PRE_BUILD
COMMAND echo "${clp}" ${java_file}
)
endforeach(java_file)
endfunction()

Get full C++ compiler command line

In CMake, the flags for the C++ compiler can be influenced in various ways: setting CMAKE_CXX_FLAGS manually, using add_definitions(), forcing a certain C++ standard, and so forth.
In order to compile a target in the same project with different rules (a precompiled header, in my case), I need to reproduce the exact command that is used to compile files added by a command like add_executable() in this directory.
Reading CMAKE_CXX_FLAGS only returns the value set to it explicitly, CMAKE_CXX_FLAGS_DEBUG and siblings only list default Debug/Release options. There is a special functions to retrieve the flags from add_definitions() and add_compiler_options(), but none seem to be able to return the final command line.
How can I get all flags passed to the compiler into a CMake variable?
To answer my own question: It seems like the only way of getting all compiler flags is to reconstruct them from the various sources. The code I'm working with now is the following (for GCC):
macro (GET_COMPILER_FLAGS TARGET VAR)
if (CMAKE_COMPILER_IS_GNUCXX)
set(COMPILER_FLAGS "")
# Get flags form add_definitions, re-escape quotes
get_target_property(TARGET_DEFS ${TARGET} COMPILE_DEFINITIONS)
get_directory_property(DIRECTORY_DEFS COMPILE_DEFINITIONS)
foreach (DEF ${TARGET_DEFS} ${DIRECTORY_DEFS})
if (DEF)
string(REPLACE "\"" "\\\"" DEF "${DEF}")
list(APPEND COMPILER_FLAGS "-D${DEF}")
endif ()
endforeach ()
# Get flags form include_directories()
get_target_property(TARGET_INCLUDEDIRS ${TARGET} INCLUDE_DIRECTORIES)
foreach (DIR ${TARGET_INCLUDEDIRS})
if (DIR)
list(APPEND COMPILER_FLAGS "-I${DIR}")
endif ()
endforeach ()
# Get build-type specific flags
string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_SUFFIX)
separate_arguments(GLOBAL_FLAGS UNIX_COMMAND
"${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE_SUFFIX}}")
list(APPEND COMPILER_FLAGS ${GLOBAL_FLAGS})
# Add -std= flag if appropriate
get_target_property(STANDARD ${TARGET} CXX_STANDARD)
if ((NOT "${STANDARD}" STREQUAL NOTFOUND) AND (NOT "${STANDARD}" STREQUAL ""))
list(APPEND COMPILER_FLAGS "-std=gnu++${STANDARD}")
endif ()
endif ()
set(${VAR} "${COMPILER_FLAGS}")
endmacro ()
This could be extended to also include options induced by add_compiler_options() and more.
Easiest way is to use make VERBOSE=1 when compiling.
cd my-build-dir
cmake path-to-my-sources
make VERBOSE=1
This will do a single-threaded build, and make will print every shell command it runs just before it runs it. So you'll see output like:
[ 0%] Building CXX object Whatever.cpp.o
<huge scary build command it used to build Whatever.cpp>
There actually is a fairly clean way to do this at compile time using CXX_COMPILER_LAUNCHER:
If you have a script print_args.py
#!/usr/bin/env python
import sys
import argparse
print(" ".join(sys.argv[1:]))
# we need to produce an output file so that the link step does not fail
p = argparse.ArgumentParser()
p.add_argument("-o")
args, _ = p.parse_known_args()
with open(args.o, "w") as f:
f.write("")
You can set the target's properties as follows:
add_library(${TARGET_NAME} ${SOURCES})
set_target_properties(${TARGET_NAME} PROPERTIES
CXX_COMPILER_LAUNCHER
${CMAKE_CURRENT_SOURCE_DIR}/print_args.py
)
# this tells the linker to not actually link. Which would fail because output file is empty
set_target_properties(${TARGET_NAME} PROPERTIES
LINK_FLAGS
-E
)
This will print the exact compilation command at compile time.
Short answer
It's not possible to assign final value of compiler command line to variable in CMake script, working in all use cases.
Long answer
Unfortunately, even solution accepted as answer still not gets all compiler flags. As gets noted in comments, there are Transitive Usage Requirements. It's a modern and proper way to write CMake files, getting more and more popular. Also you may have some compile options defined using generator expressions (they look like variable references but will not expand when needed).
Consider having following example:
add_executable(myexe ...);
target_compile_definitions(myexe PRIVATE "PLATFORM_$<PLATFORM_ID>");
add_library(mylib ...);
target_compile_definitions(mylib INTERFACE USING_MY_LIB);
target_link_libraries(myexe PUBLIC mylib);
If you try to call proposed GET_COMPILER_FLAGS macro with myexe target, you will get resulting output -DPLATFORM_$<PLATFORM_ID> instead of expected -DPLATFORM_Linux -DUSING_MY_LIB.
This is because there are two stages between invoking CMake and getting build system generated:
Processing. At this stage CMake reads and executes commands from cmake script(s), particularly, variable values getting evaluated and assigned. At this moment CMake just collecting all required info and being prepared to generate build system (makefiles).
Generating. CMake uses values of special variables and properties, being left at end of processed scripts to finally decide and form generated output. This is where it constructs final command line for compiler according to its internal algorithm, not avaliable for scripting.
Target properties which might be retrieved at processing stage with get_target_property(...) or get_property(... TARGET ...) aren't complete (even when invoked at the end of script). At generating stage CMake walks through each target dependency tree (recursively) and appends properties values according to transitive usage requirements (PUBLIC and INTERFACE tagged values gets propagated).
Although, there are workarounds, depending on what final result you aiming to achieve. This is possible by applying generator expressions, which allows use final values of properties of any target (defined at processing stage)... but later!
Two general possibilites are avaliable:
Generate any output file based on template, which content contains variable references and/or generator expressions, and defined as either string variable value, or input file. It's not flexible due to very limited support of conditional logic (i.e. you cannot use complex concatenations available only with nested foreach() loops), but has advantages, that no further actions required and content described in platform-independent way. Use file(GENERATE ...) command variant. Note, that it behaves differently from file (WRITE ...) variant.
Add custom target (and/or custom command) which implements further usage of expanded value. It's platform dependent and requires user to additionally invoke make (either with some special target, or include to all target), but has advantage, that it's flexible enough because you may implement shell script (but without executable bit).
Example demonstrating solution with combining these options:
set(target_name "myexe")
file(GENERATE OUTPUT script.sh CONTENT "#!/bin/sh\n echo \"${target_name} compile definitions: $<TARGET_PROPERTY:${target_name},COMPILE_DEFINITIONS>\"")
add_custom_target(mycustomtarget
COMMAND echo "\"Platform: $<PLATFORM_ID>\""
COMMAND /bin/sh -s < script.sh
)
After calling CMake build directory will contain file script.sh and invoking make mycustomtarget will print to console:
Platform: Linux
myexe compile definitions: PLATFORM_Linux USING_MY_LIB
Use
set(CMAKE_EXPORT_COMPILE_COMMANDS true)
and get compile_commands.json

CMake - copying/merging files with dependencies a change-checking

I have a big project with one executable, some plugins and web interface with some generated JSONs.
Therefore, after I compile executables and .so plugins, I'm doing following:
Merge all .js files into one big
Compile "generators" (set of macros and printfs to describe some structures in C++ code)
Run generators and generate JSON files (with some sed and jshon) processing
In install phase, and copy all of this and some other files to their destination directories (which should be created if doesn't exists).
But I don't know, how to use CMake to make correct dependencies and date-time checking. Actually, first step is made with:
FILE(GLOB WEB_INPUT_JS *.js)
FILE(WRITE scripts.js.tmp "")
FOREACH(SCRIPTFILE ${WEB_INPUT_JS})
FILE(READ ${SCRIPTFILE} CONTENTS)
FILE(APPEND scripts.js.tmp "${CONTENTS}")
ENDFOREACH()
CONFIGURE_FILE(scripts.js.tmp ${WEB_BUILD_PATH}/scripts.js COPYONLY)
But this doesn't create dependency in makefiles. I want to re-run this piece of "code", when some of ${WEB_INPUT_JS} files has been changed or ${WEB_BUILD_PATH}/scripts.js has been deleted.
Third step is made with series of
add_custom_command(TARGET gen_somedata POST_BUILD COMMAND gen_somedata | sed ${JSON_SED} | jshon > ${JSON_BUILD_PATH}/somedata.json)
install (FILES ${JSON_BUILD_PATH}/somedata.json ......nextfiles.... DESTINATION ${JSON_OUTPUT_PATH})
How is this done? Thanks much for your answers!
So I've finally found out, how to do some of this things.
Merging files is pretty tricky.
First, cmake "script" doing merging is needed (I will explain some lines later). I will name it "concat.cmake":
FUNCTION(CONCAT_FILES OUTPUT FILELIST)
FILE(WRITE ${OUTPUT} "")
FOREACH(SCRIPTFILE ${FILELIST})
FILE(READ ${SCRIPTFILE} CONTENTS)
FILE(APPEND ${OUTPUT} "${CONTENTS}")
ENDFOREACH()
ENDFUNCTION(CONCAT_FILES)
STRING(REPLACE "," ";" FILELIST ${FILELIST})
CONCAT_FILES(${OUTPUT} "${FILELIST}")
Then, when merging script is used as follows (write it into CMakeLists.txt):
1) First, make an file list (using globbing or by writing file list by hand).
FILE(GLOB INPUT_FILES_LIST *.js) # get list of JS files
2) The only way, how to pass a cmake list to other cmake script is creating file list separated by comma, then passing comma-separated list to external script. I've done this following way:
SET(FILELIST "")
FOREACH(ITEM ${INPUT_FILES_LIST})
SET(FILELIST "${FILELIST},${ITEM}") # append list item by ','
ENDFOREACH()
STRING(SUBSTRING ${JSFILES} 1 -1 JSFILES) # remove first ','
3) Now it's not problem to call merging script..
add_custom_command(OUTPUT some_output_file.ext
COMMAND ${CMAKE_COMMAND} -DFILELIST=${FILELIST} -DOUTPUT=some_output_frile.ext -P ${CMAKE_CURRENT_SOURCE_DIR}/concat.cmake
DEPENDS ${INPUT_FILES_LIST} VERBATIM )
The precedent code will correctly track changes in input files and output file will be generated when missing or input changes. Installation is just easy as
INSTALL (FILES "output.ext" DESTINATION /usr/share/...)