Make/makefile progress indication! - scripting

Question

Look at this makefile, it has some sort of primitive progress indication (could have been a progress bar).
Please give me suggestions/comments on it!
# BUILD is initially undefined
ifndef BUILD
# max equals 256 x's
sixteen := x x x x x x x x x x x x x x x x
MAX := $(foreach x,$(sixteen),$(sixteen))
# T estimates how many targets we are building by replacing BUILD with a special string
T := $(shell $(MAKE) -nrRf $(firstword $(MAKEFILE_LIST)) $(MAKECMDGOALS) \
BUILD="COUNTTHIS" | grep -c "COUNTTHIS")
# N is the number of pending targets in base 1, well in fact, base x :-)
N := $(wordlist 1,$T,$(MAX))
# auto-decrementing counter that returns the number of pending targets in base 10
counter = $(words $N)$(eval N := $(wordlist 2,$(words $N),$N))
# BUILD is now defined to show the progress, this also avoids redefining T in loop
BUILD = #echo $(counter) of $(T)
endif
# dummy phony targets
.PHONY: all clean
all: target
#echo done
clean:
#rm -f target *.c
# dummy build rules
target: a.c b.c c.c d.c e.c f.c g.c
#touch $#
$(BUILD)
%.c:
#touch $#
$(BUILD)
All suggestions welcome!

Answer 1

This one is less intrusive and more awesome.
ifneq ($(words $(MAKECMDGOALS)),1)
.DEFAULT_GOAL = all
%:
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST))
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = echo "`expr " [\`expr $C '*' 100 / $T\`" : '.*\(....\)$$'`%]"
endif
.PHONY: all clean
all: target
#$(ECHO) All done
clean:
#rm -f target *.c
# #$(ECHO) Clean done
target: a.c b.c c.c d.c e.c
#$(ECHO) Linking $#
#sleep 0.1
#touch $#
%.c:
#$(ECHO) Compiling $#
#sleep 0.1
#touch $#
endif

Answer 2

There wasn't really a question so this is less of a standalone answer and more of an extension to Giovanni Funchai's solution. This question is the first google result for "GNU Make Progress" so I ended up here looking for how to do this.
As pointed out by Rob Wells, the solution doesn't work for <10%, but the technique can be extended with the print formatting done by a helper script in whatever language you feel is portable enough for your build. For example, using a python helper script:
echo_progress.py:
"""
Print makefile progress
"""
import argparse
import math
import sys
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stepno", type=int, required=True)
parser.add_argument("--nsteps", type=int, required=True)
parser.add_argument("remainder", nargs=argparse.REMAINDER)
args = parser.parse_args()
nchars = int(math.log(args.nsteps, 10)) + 1
fmt_str = "[{:Xd}/{:Xd}]({:6.2f}%)".replace("X", str(nchars))
progress = 100 * args.stepno / args.nsteps
sys.stdout.write(fmt_str.format(args.stepno, args.nsteps, progress))
for item in args.remainder:
sys.stdout.write(" ")
sys.stdout.write(item)
sys.stdout.write("\n")
if __name__ == "__main__":
main()
And the modified Makefile:
_mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
I := $(patsubst %/,%,$(dir $(_mkfile_path)))
ifneq ($(words $(MAKECMDGOALS)),1)
.DEFAULT_GOAL = all
%:
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST))
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = python $(I)/echo_progress.py --stepno=$C --nsteps=$T
endif
.PHONY: all clean
all: target
#$(ECHO) All done
clean:
#rm -f target *.c
# #$(ECHO) Clean done
target: a.c b.c c.c d.c e.c f.c g.c h.c i.c j.c k.c l.c m.c n.c o.c p.c q.c \
r.c s.c t.c u.c v.c w.c x.c y.c z.c
#$(ECHO) Linking $#
#sleep 0.01
#touch $#
%.c:
#$(ECHO) Compiling $#
#sleep 0.01
#touch $#
endif
yields:
$ make
[ 1/28]( 3.57%) Compiling a.c
[ 2/28]( 7.14%) Compiling b.c
[ 3/28]( 10.71%) Compiling c.c
[ 4/28]( 14.29%) Compiling d.c
[ 5/28]( 17.86%) Compiling e.c
[ 6/28]( 21.43%) Compiling f.c
[ 7/28]( 25.00%) Compiling g.c
[ 8/28]( 28.57%) Compiling h.c
[ 9/28]( 32.14%) Compiling i.c
[10/28]( 35.71%) Compiling j.c
[11/28]( 39.29%) Compiling k.c
[12/28]( 42.86%) Compiling l.c
[13/28]( 46.43%) Compiling m.c
[14/28]( 50.00%) Compiling n.c
[15/28]( 53.57%) Compiling o.c
[16/28]( 57.14%) Compiling p.c
[17/28]( 60.71%) Compiling q.c
[18/28]( 64.29%) Compiling r.c
[19/28]( 67.86%) Compiling s.c
[20/28]( 71.43%) Compiling t.c
[21/28]( 75.00%) Compiling u.c
[22/28]( 78.57%) Compiling v.c
[23/28]( 82.14%) Compiling w.c
[24/28]( 85.71%) Compiling x.c
[25/28]( 89.29%) Compiling y.c
[26/28]( 92.86%) Compiling z.c
[27/28]( 96.43%) Linking target
[28/28](100.00%) All done
One could even print a fancy progress bar with unicode characters.
Modified echo_progress.py:
"""
Print makefile progress
"""
import argparse
import math
import sys
def get_progress_bar(numchars, fraction=None, percent=None):
"""
Return a high resolution unicode progress bar
"""
if percent is not None:
fraction = percent / 100.0
if fraction >= 1.0:
return "█" * numchars
blocks = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"]
length_in_chars = fraction * numchars
n_full = int(length_in_chars)
i_partial = int(8 * (length_in_chars - n_full))
n_empty = max(numchars - n_full - 1, 0)
return ("█" * n_full) + blocks[i_partial] + (" " * n_empty)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stepno", type=int, required=True)
parser.add_argument("--nsteps", type=int, required=True)
parser.add_argument("remainder", nargs=argparse.REMAINDER)
args = parser.parse_args()
nchars = int(math.log(args.nsteps, 10)) + 1
fmt_str = "\r[{:Xd}/{:Xd}]({:6.2f}%) ".replace("X", str(nchars))
progress = 100 * args.stepno / args.nsteps
sys.stdout.write(fmt_str.format(args.stepno, args.nsteps, progress))
sys.stdout.write(get_progress_bar(20, percent=progress))
remainder_str = " ".join(args.remainder)
sys.stdout.write(" {:20s}".format(remainder_str[:20]))
if args.stepno == args.nsteps:
sys.stdout.write("\n")
if __name__ == "__main__":
main()
Which would result in something like this:
$ make clean && make
[12/28]( 42.86%) ███████▊ Compiling k.c
during progress and:
$ make clean && make
[28/28](100.00%) ████████████████████ All done
upon completion.

Answer 3

This is a slight modification to #GiovanniFunchal's excellent answer.
So I wanted to understand this better and make it work for < 10% so I dug into the documentation and learned more about expr.
# PLACE AT THE TOP OF YOUR MAKEFILE
#---------------------------------
# Progress bar defs
#--------------------------------
# words = count the number of words
ifneq ($(words $(MAKECMDGOALS)),1) # if no argument was given to make...
.DEFAULT_GOAL = all # set the default goal to all
# http://www.gnu.org/software/make/manual/make.html
# $# = target name
# %: = last resort recipe
# --no-print-directory = don't print enter/leave messages for each output grouping
# MAKEFILE_LIST = has a list of all the parsed Makefiles that can be found *.mk, Makefile, etc
# -n = dry run, just print the recipes
# -r = no builtin rules, disables implicit rules
# -R = no builtin variables, disables implicit variables
# -f = specify the name of the Makefile
%: # define a last resort default rule
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST)) # recursive make call,
else
ifndef ECHO
# execute a dry run of make, defining echo beforehand, and count all the instances of "COUNTTHIS"
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
# eval = evaluate the text and read the results as makefile commands
N := x
# Recursively expand C for each instance of ECHO to count more x's
C = $(words $N)$(eval N := x $N)
# Multipy the count of x's by 100, and divide by the count of "COUNTTHIS"
# Followed by a percent sign
# And wrap it all in square brackets
ECHO = echo -ne "\r [`expr $C '*' 100 / $T`%]"
endif
#------------------
# end progress bar
#------------------
# REST OF YOUR MAKEFILE HERE
#----- Progressbar endif at end Makefile
endif
I got rid of the : '.*\(....\)$$' part. It would return the last 4 characters of the inner expr command, but would fail if it was less than 4. And now it works for sub 10%!
And here is the comment free version:
ifneq ($(words $(MAKECMDGOALS)),1) # if no argument was given to make...
.DEFAULT_GOAL = all # set the default goal to all
%: # define a last resort default rule
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST)) # recursive make call,
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = echo -ne "\r [`expr $C '*' 100 / $T`%]"
endif
# ...
endif
Hope that helps.

Answer 4

Many thanks to #Giovanni Funchal and #phyatt for the question and answers!
I just simplified it even more for my own better understanding.
ifndef ECHO
HIT_TOTAL != ${MAKE} ${MAKECMDGOALS} --dry-run ECHO="HIT_MARK" | grep -c "HIT_MARK"
HIT_COUNT = $(eval HIT_N != expr ${HIT_N} + 1)${HIT_N}
ECHO = echo "[`expr ${HIT_COUNT} '*' 100 / ${HIT_TOTAL}`%]"
endif
!= assigns from shell command
= evaluates variable each time it's used
eval executes its argument without any output
expr allows to make arithmetic calculations
( Not sure though which approach is faster: to call shell with expr or to count 'x'-es with make. )
Usage is the same:
target:
#$(ECHO) $#

Answer 5

Nice trick! (-:
But not really scalable for growing projects that are distributed across many directories with lots of makefiles.
I'd be more inclined to have logging sprinkled through the [Mm]akefiles* in your project and use that to keep track of progress.
Just a thought. BTW Thanks for sharing this.
Edit: Just had a thought. This could be useful in a modified form to display a throbber to show progress while a long task proceeds, e.g unpacking a large distribution tarball instead of just specifying the -v option to the tar command. Still a bit of sugar coating but a bit of fun aswell. (-:
cheers,
Rob