So about a week ago I worked on setting up a mail server for my company's mail service. I ended up using zpanel to make it easier to create new mail accounts. Zpanel uses postfix and dovecot that is configured via the installation. I've been trying to set up ssl on this server using a self signed certificate but I continue to receive a your server doesn't support ssl.
/etc/postfix/main.cf
# postfix config file
# uncomment for debugging if needed
soft_bounce=yes
# postfix main
mail_owner = postfix
setgid_group = postdrop
delay_warning_time = 4
# postfix paths
html_directory = no
command_directory = /usr/sbin
daemon_directory = /usr/lib/postfix
queue_directory = /var/spool/postfix
sendmail_path = /usr/sbin/sendmail.postfix
newaliases_path = /usr/bin/newaliases.postfix
mailq_path = /usr/bin/mailq.postfix
manpage_directory = /usr/share/man
sample_directory = /usr/share/doc/postfix-2.2.2/samples
readme_directory = /usr/share/doc/postfix-2.2.2/README_FILES
# network settings
inet_interfaces = all
mydomain = mail.thetactilegroup.com
myhostname = mail.thetactilegroup.com
mynetworks = all
mydestination = localhost.$mydomain, localhost
relay_domains = proxy:mysql:/etc/zpanel/configs/postfix/mysql-relay_domains_maps.cf
# mail delivery
recipient_delimiter = +
# mappings
alias_maps = hash:/etc/aliases
alias_database = hash:/etc/aliases
#transport_maps = hash:/etc/postfix/transport
#local_recipient_maps =
# virtual setup
virtual_alias_maps = proxy:mysql:/etc/zpanel/configs/postfix/mysql- virtual_alias_maps.cf,
regexp:/etc/zpanel/configs/postfix/virtual_regexp
virtual_mailbox_base = /var/zpanel/vmail
virtual_mailbox_domains = proxy:mysql:/etc/zpanel/configs/postfix/mysql- virtual_domains_maps.cf
virtual_mailbox_maps = proxy:mysql:/etc/zpanel/configs/postfix/mysql- virtual_mailbox_maps.cf
virtual_mailbox_limit_maps = proxy:mysql:/etc/zpanel/configs/postfix/mysql-virtual_mailbox_limit_maps.cf
virtual_minimum_uid = 150
virtual_uid_maps = static:150
virtual_gid_maps = static:8
virtual_transport = dovecot
dovecot_destination_recipient_limit = 1
# debugging
debug_peer_level = 2
debugger_command =
PATH=/bin:/usr/bin:/usr/local/bin:/usr/X11R6/bin
xxgdb $daemon_directory/$process_name $process_id & sleep 5
# authentication
smtpd_sasl_auth_enable = yes
smtpd_sasl_security_options = noanonymous
smtpd_sasl_local_domain = $myhostname
broken_sasl_auth_clients = yes
smtpd_sasl_type = dovecot
smtpd_sasl_path = private/auth
# tls config
smtp_use_tls = no
smtpd_use_tls = yes
#smtp_tls_note_starttls_offer = yes
#smtpd_tls_loglevel = 1
#smtpd_tls_received_header = yes
#smtpd_tls_session_cache_timeout = 3600s
#tls_random_source = dev:/dev/urandom
#smtp_tls_session_cache_database = btree:$data_directory/smtp_tls_session_cache
# Change mail.example.com.* to your host name
smtpd_tls_key_file = /etc/postfix/ssl/smtp.in.key
smtpd_tls_cert_file = /etc/postfix/ssl/smtp.in.pem
# smtpd_tls_CAfile = /etc/pki/tls/root.crt
# rules restrictions
smtpd_client_restrictions =
smtpd_helo_restrictions =
smtpd_sender_restrictions =
smtpd_recipient_restrictions = permit_sasl_authenticated,
permit_mynetworks,
reject_unauth_destination,
reject_non_fqdn_sender,
reject_non_fqdn_recipient,
reject_unknown_recipient_domain
# uncomment for realtime black list checks
# ,reject_rbl_client zen.spamhaus.org
# ,reject_rbl_client bl.spamcop.net
# ,reject_rbl_client dnsbl.sorbs.net
smtpd_helo_required = yes
unknown_local_recipient_reject_code = 550
disable_vrfy_command = yes
smtpd_data_restrictions = reject_unauth_pipelining
message_size_limit = 52428800
smtp_tls_security_level = may
/etc/postfix/master.cf
#
# Postfix master process configuration file. For details on the format
# of the file, see the Postfix master(5) manual page.
#
# ***** Unused items removed *****
# ==========================================================================
# service type private unpriv chroot wakeup maxproc command + args
# (yes) (yes) (yes) (never) (100)
# ==========================================================================
submission inet n - - - - smtpd
-o smtpd_tls_security_level=encrypt
-o smtpd_sasl_auth_enable=yes
-o smtpd_sasl_type=dovecot
-o smtpd_sasl_path=private/auth
-o smtpd_sasl_security_options=noanonymous
-o smtpd_sasl_local_domain=$myhostname
-o smtpd_client_restrictions=permit_sasl_authenticated,reject
-o smtpd_sender_login_maps=hash:/etc/postfix/virtual
-o smtpd_sender_restrictions=reject_sender_login_mismatch
-o smtpd_recipient_restrictions=reject_non_fqdn_recipient,reject_unknown_recipient_domain,permit_sasl_authenticated,reject
#smtp inet n - n - - smtpd
# -o content_filter=smtp-amavis:127.0.0.1:10024
# -o receive_override_options=no_address_mappings
pickup fifo n - n 60 1 pickup
-o content_filter=
-o receive_override_options=no_header_body_checks
cleanup unix n - n - 0 cleanup
qmgr fifo n - n 300 1 qmgr
#qmgr fifo n - n 300 1 oqmgr
tlsmgr unix - - n 1000? 1 tlsmgr
rewrite unix - - n - - trivial-rewrite
bounce unix - - n - 0 bounce
defer unix - - n - 0 bounce
trace unix - - n - 0 bounce
verify unix - - n - 1 verify
flush unix n - n 1000? 0 flush
proxymap unix - - n - - proxymap
smtp unix - - n - - smtp
# When relaying mail as backup MX, disable fallback_relay to avoid MX loops
relay unix - - n - - smtp
-o fallback_relay=
# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5
showq unix n - n - - showq
error unix - - n - - error
discard unix - - n - - discard
local unix - n n - - local
virtual unix - n n - - virtual
lmtp unix - - n - - lmtp
anvil unix - - n - 1 anvil
scache unix - - n - 1 scache
#
# ====================================================================
# Interfaces to non-Postfix software. Be sure to examine the manual
# pages of the non-Postfix software to find out what options it wants.
# ====================================================================
maildrop unix - n n - - pipe
flags=DRhu user=vmail argv=/usr/local/bin/maildrop -d ${recipient}
uucp unix - n n - - pipe
flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient)
ifmail unix - n n - - pipe
flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient)
bsmtp unix - n n - - pipe
flags=Fq. user=foo argv=/usr/local/sbin/bsmtp -f $sender $nexthop $recipient
#
# spam/virus section
#
smtp-amavis unix - - y - 2 smtp
-o smtp_data_done_timeout=1200
-o disable_dns_lookups=yes
-o smtp_send_xforward_command=yes
127.0.0.1:10025 inet n - y - - smtpd
-o content_filter=
-o smtpd_helo_restrictions=
-o smtpd_sender_restrictions=
-o smtpd_recipient_restrictions=permit_mynetworks,reject
-o mynetworks=127.0.0.0/8
-o smtpd_error_sleep_time=0
-o smtpd_soft_error_limit=1001
-o smtpd_hard_error_limit=1000
-o receive_override_options=no_header_body_checks
-o smtpd_bind_address=127.0.0.1
-o smtpd_helo_required=no
-o smtpd_client_restrictions=
-o smtpd_restriction_classes=
-o disable_vrfy_command=no
-o strict_rfc821_envelopes=yes
#
# Dovecot LDA
dovecot unix - n n - - pipe
flags=DRhu user=vmail:mail argv=/usr/lib/dovecot/deliver -d ${recipient}
#
# Vacation mail
vacation unix - n n - - pipe
flags=Rq user=vacation argv=/var/spool/vacation/vacation.pl -f ${sender} -- ${recipient}
How would I correct this issue?
mydomain = mail.thetactilegroup.com
myhostname = mail.thetactilegroup.com
convert to this
mydomain = thetactilegroup.com
myhostname = mail.thetactilegroup.com
Basically you need to enable smtp_use_tls:
smtp_use_tls = yes
Additionally, I recommend the following switches:
smtp_tls_loglevel = 1
smtpd_tls_loglevel = 1
smtp_tls_note_starttls_offer = yes
smtp_tls_security_level = may
smtpd_tls_security_level = may
smtpd_tls_ciphers = high
smtpd_tls_exclude_ciphers = aNULL, aDSS, aECDSA
smtpd_tls_protocols = !SSLv2, !SSLv3
smtpd_tls_received_header = yes
You can read more about here http://www.postfix.org/postconf.5.html
Try this in master.cf:
smtps inet n - n - - smtpd
-o smtpd_tls_wrappermode=yes
-o smtpd_sasl_auth_enable=yes
-o smtpd_client_restrictions=permit_sasl_authenticated,reject
-o milter_macro_daemon_name=ORIGINATING
Save and restart Postfix:
service postfix restart
Related
In a bash script that I'm writing for Linux/Solaris I need to decode more than a hundred thousand base64-encoded text strings, and, because I don't wanna massively fork a non-portable base64 binary from awk, I wrote a function that does the decoding.
Here's the code of my base64_decode function:
function base64_decode(str, out,i,n,v) {
out = ""
if ( ! ("A" in _BASE64_DECODE_c2i) )
for (i = 1; i <= 64; i++)
_BASE64_DECODE_c2i[substr("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",i,1)] = i-1
i = 0
n = length(str)
while (i <= n) {
v = _BASE64_DECODE_c2i[substr(str,++i,1)] * 262144 + \
_BASE64_DECODE_c2i[substr(str,++i,1)] * 4096 + \
_BASE64_DECODE_c2i[substr(str,++i,1)] * 64 + \
_BASE64_DECODE_c2i[substr(str,++i,1)]
out = out sprintf("%c%c%c", int(v/65536), int(v/256), v)
}
return out
}
Which works fine:
printf '%s\n' SmFuZQ== amRvZQ== |
LANG=C command -p awk '
{ print base64_decode($0) }
function base64_decode(...) {...}
'
Jane
jdoe
SIMPLIFIED REAL-LIFE EXAMPLE THAT DOESN'T WORK AS EXPECTED
I want to get the givenName of the users that are members of GroupCode = 025496 from the output of ldapsearch -LLL -o ldif-wrap=no ... '(|(uid=*)(GroupCode=*))' uid givenName sn GroupCode memberUid:
dn: uid=jsmith,ou=users,dc=example,dc=com
givenName: John
sn: SMITH
uid: jsmith
dn: uid=jdoe,ou=users,dc=example,dc=com
uid: jdoe
givenName:: SmFuZQ==
sn:: RE9F
dn: cn=group1,ou=groups,dc=example,dc=com
GroupCode: 025496
memberUid:: amRvZQ==
memberUid: jsmith
Here would be an awk for doing so:
LANG=C command -p awk -F '\n' -v RS='' -v GroupCode=025496 '
{
delete attrs
for (i = 2; i <= NF; i++) {
match($i,/::? /)
key = substr($i,1,RSTART-1)
val = substr($i,RSTART+RLENGTH)
if (RLENGTH == 3)
val = base64_decode(val)
attrs[key] = ((key in attrs) ? attrs[key] SUBSEP val : val)
}
if ( /\nuid:/ )
givenName[ attrs["uid"] ] = attrs["givenName"]
else
memberUid[ attrs["GroupCode"] ] = attrs["memberUid"]
}
END {
n = split(memberUid[GroupCode],uid,SUBSEP)
for ( i = 1; i <= n; i++ )
print givenName[ uid[i] ]
}
function base64_decode(...) { ... }
'
On BSD and Solaris the result is:
Jane
John
While on Linux it is:
John
I don't know where the issue might be; is there something wrong with the base64_decode function and/or the code that uses it?
Your function generates NUL bytes when its argument (encoded string) ends with padding characters (=s). Below is a corrected version of your while loop:
while (i < n) {
v = _BASE64_DECODE_c2i[substr(str,1+i,1)] * 262144 + \
_BASE64_DECODE_c2i[substr(str,2+i,1)] * 4096 + \
_BASE64_DECODE_c2i[substr(str,3+i,1)] * 64 + \
_BASE64_DECODE_c2i[substr(str,4+i,1)]
i += 4
if (v%256 != 0)
out = out sprintf("%c%c%c", int(v/65536), int(v/256), v)
else if (int(v/256)%256 != 0)
out = out sprintf("%c%c", int(v/65536), int(v/256))
else
out = out sprintf("%c", int(v/65536))
}
Note that if the decoded bytes contains an embedded NUL then this approach may not work properly.
Problem is within base64_decode function that outputs some junk characters on gnu-awk.
You can use this awk code that uses system provided base64 utility as an alternative:
{
delete attrs
for (i = 2; i <= NF; i++) {
match($i,/::? /)
key = substr($i,1,RSTART-1)
val = substr($i,RSTART+RLENGTH)
if (RLENGTH == 3) {
cmd = "echo " val " | base64 -di"
cmd | getline val # should also check exit code here
}
attrs[key] = ((key in attrs) ? attrs[key] SUBSEP val : val)
}
if ( /\nuid:/ )
givenName[ attrs["uid"] ] = attrs["givenName"]
else
memberUid[ attrs["GroupCode"] ] = attrs["memberUid"]
}
END {
n = split(memberUid[GroupCode],uid,SUBSEP)
for ( i = 1; i <= n; i++ )
print givenName[ uid[i] ]
}
I have tested this on gnu and BSD awk versions and I am getting expected output in all the cases.
If you cannot use external base64 utility then I suggest you take a look here for awk version of base64 decode.
This answer is for reference
Here's a working base64_decode function (thanks #MNejatAydin for pointing out the issue(s) in the original one):
function base64_decode(str, out,bits,n,i,c1,c2,c3,c4) {
out = ""
# One-time initialization during the first execution
if ( ! ("A" in _BASE64) )
for (i = 1; i <= 64; i++)
# The "_BASE64" array associates a character to its base64 index
_BASE64[substr("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",i,1)] = i-1
# Decoding the input string
n = length(str)
i = 0
while ( i < n ) {
c1 = substr(str, ++i, 1)
c2 = substr(str, ++i, 1)
c3 = substr(str, ++i, 1)
c4 = substr(str, ++i, 1)
bits = _BASE64[c1] * 262144 + _BASE64[c2] * 4096 + _BASE64[c3] * 64 + _BASE64[c4]
if ( c4 != "=" )
out = out sprintf("%c%c%c", bits/65536, bits/256, bits)
else if ( c3 != "=" )
out = out sprintf("%c%c", bits/65536, bits/256)
else
out = out sprintf("%c", bits/65536)
}
return out
}
WARNING: the function requires LANG=C
It also doesn't check that the input is a valid base64 string; for that you can add a simple condition like:
match( str, "^([a-zA-Z/-9+]{4})*([a-zA-Z/-9+]{2}[a-zA-Z/-9+=]{2})?$" )
Interestingly, the code is 2x faster than base64decode.awk, but it's only 3x faster than forking the base64 binary from inside awk.
notes:
In a base64 encoded string, 4 bytes represent 3 bytes of data; the input have to be processed by groups of 4 characters.
Multiplying and dividing an integer by a power of two is equivalent to do bitwise left and right shifts operations.
262144 is 2^18, so N * 262144 is equivalent to N << 18
4096 is 2^12, so N * 4096 is equivalent to N << 12
64 id 2^6, so N * 4096 is equivalent to N << 6
65536 is 2^16, so N / 65536 (integer division) is equivalent to N >> 16
256 is 2^8, so N / 256 (integer division) is equivalent to N >> 8
What happens in printf "%c", N:
N is first converted to an integer (if need be) and then, WITH LANG=C, the 8 least significant bits are taken in for the %c formatting.
How the possible padding of one or two trailing = characters at the end of the encoded string is handled:
If the 4th char isn't = (i.e. there's no padding) then the result should be 3 bytes of data.
If the 4th char is = and the 3rd char isn't = then there's 2 bytes of of data to decode.
If the fourth char is = and the third char is = then there's only one byte of data.
I'm working on a RPi Pico W based project, and I need to use a TLC5947 led driver. The connection is SPI, which I'm told is pretty simple, but I tried to implement it myself and couldn't. Adafruit has a circuitpython module, but it dosen't seem to translate (directly at least) into micropython.
Do I need to keep researching making it myself or is there a module already made?
My attempt: (I assume write or pwmbuffer is the problem fwiw. Comments are copied directly from Adafruit's C++ version of the library for arduino.)
from machine import Pin
import machine
class TLC5947:
def __init__(self, clock: int = 2, data: int = 3, latch: int = 5):
self.numdrivers = 1
self.data = Pin(data, Pin.OUT)
self.clock = Pin(clock, Pin.OUT)
self.latch = Pin(latch, Pin.OUT)
self.latch.low()
self._spi = machine.SPI(0)
# self.OE = OE
self.pwmbuffer = [0] * (24 * 2 * self.numdrivers) # memset(pwmbuffer, 0, 2 * 24 * n);
# self.spi = machine.SPI(0)
def write(self):
self.latch.low() # digitalWrite(_lat, LOW);
# // 24 channels per TLC5974
for c in range(24 * self.numdrivers - 1, -1, -1): # for (int16_t c = 24 * numdrivers - 1; c >= 0; c--) {
# // 12 bits per channel, send MSB first
for b in range(11, -1, -1): # for (int8_t b = 11; b >= 0; b--) {
self.clock.low() # digitalWrite(_clk, LOW);
if self.pwmbuffer[c] & (1 << b): # if (pwmbuffer[c] & (1 << b))
self.data.high() # digitalWrite(_dat, HIGH);
else: # else
self.data.low() # digitalWrite(_dat, LOW);
#
self.clock.high() # digitalWrite(_clk, HIGH);
# }
# }
self.clock.low() # digitalWrite(_clk, LOW);
self.latch.high() # digitalWrite(_lat, HIGH);
self.latch.low() # digitalWrite(_lat, LOW);
def setLed(self, lednum, r,g,b):
self.setPWM(lednum * 3, r)
self.setPWM(lednum * 3 + 1, g)
self.setPWM(lednum * 3 + 2, b)
def setPWM(self, chan: int, pwm: int):
if (pwm > 4095):
pwm = 4095
try:
self.pwmbuffer[chan] = pwm
except:
pass
Edit:
Got it. That repo refers to a folder structure like this:
project/
├── modules/
│ └──tlc5947-rgb-micropython/
│ ├──...
│ └──micropython.mk
└── micropython/
├──ports/
... ├──stm32/
...
But I don't have anything like that. Mine is:
project/
|_ .vscode/
| |_ ...
|_ lib/
|_ code.py
|_ i2c_display.py
|_ tlc5947_ME.py
|_ .picowgo # Used by Pico-W-Go vscode extention to allow Pico programming in vscode
Via Awesome MicroPython, I found https://gitlab.com/peterzuger/tlc5947-rgb-micropython - it looks fairly up-to-date.
The command :
ioreg -p IOUSB -l -b | grep -E "#|PortNum|USB Serial Number"
gives :
o USB 2.0 Hub#14200000 <class AppleUSBDevice, id 0x1000311c8, registered, matched, active, busy 0 (1 ms), retain 14>
{
"sessionID" = 28933243797551
"iManufacturer" = 0
"bNumConfigurations" = 1
"idProduct" = 1025
"bcdDevice" = 256
"Bus Power Available" = 500
"USB Address" = 9
"bMaxPacketSize0" = 64
"iProduct" = 1
"iSerialNumber" = 0
"bDeviceClass" = 9
"Built-In" = No
"locationID" = 337641472
"bDeviceSubClass" = 0
"bcdUSB" = 512
"USB Product Name" = "USB 2.0 Hub"
"PortNum" = 2
"non-removable" = "no"
"IOCFPlugInTypes" = {"9dc7b780-9ec0-11d4-a54f-000a27052861"="IOUSBFamily.kext/Contents/PlugIns/IOUSBLib.bundle"}
"bDeviceProtocol" = 1
"IOUserClientClass" = "IOUSBDeviceUserClientV2"
"IOPowerManagement" = {"DevicePowerState"=0,"CurrentPowerState"=3,"CapabilityFlags"=65536,"MaxPowerState"=4,"DriverPowerState"=3}
"Device Speed" = 2
"idVendor" = 6720
"IOGeneralInterest" = "IOCommand is not serializable"
"IOClassNameOverride" = "IOUSBDevice"
}
This is the exact device. I am however having trouble figuring out the port name in the "tty" system.
I have a file named bt.B.1.log that looks like this:
.
.
.
Time in seconds = 260.37
.
.
.
Compiled procs = 1
.
.
.
Time in seconds = 260.04
.
.
.
Compiled procs = 1
.
.
.
and so on for 40 records of Time in seconds and Compiled procs (dots represent useless lines).
How do I add a single column with the value of Compiled procs (which is 1) to the result of the following two commands:
This prints the average of Time in seconds values (thanks to dawg for this one)
awk -F= '/Time in seconds/ {s+=$2; c++} END {print s/c}' bt.B.1.log > t1avg.dat
Desired output:
260.20 1
This prints all of the ocurrences of Time in seconds, but there is a small problem with it; it is printing an extra blank line at the beginning of the list.
awk 'BEGIN { FS = "Time in seconds =" } ; { printf $2 } {printf " "}' bt.B.1.log > t1.dat
Desired output:
260.37 1
260.04
.
.
.
In both cases I need the value of Compiled procs to appear only once, preferrably in the first line, and no use of intermediate files.
What I managed to do so far prints all values of Time in seconds with the Compiled procs column appearing in every line and with a strange identation:
awk '/seconds/ {printf $5} {printf " "} /procs/ {print $4}' bt.B.1.log > t1.dat
Please help!
UPDATE
Contents of file bt.B.1.log:
-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-
Start in 16:40:51--25/12/2014
NAS Parallel Benchmarks 3.3 -- BT Benchmark
No input file inputbt.data. Using compiled defaults
Size: 102x 102x 102
Iterations: 200 dt: 0.0003000
Number of active processes: 1
Time step 1
Time step 20
Time step 40
Time step 60
Time step 80
Time step 100
Time step 120
Time step 140
Time step 160
Time step 180
Time step 200
Verification being performed for class B
accuracy setting for epsilon = 0.1000000000000E-07
Comparison of RMS-norms of residual
1 0.1423359722929E+04 0.1423359722929E+04 0.7507984505732E-14
2 0.9933052259015E+02 0.9933052259015E+02 0.3147459568137E-14
3 0.3564602564454E+03 0.3564602564454E+03 0.4783990739472E-14
4 0.3248544795908E+03 0.3248544795908E+03 0.2309751522921E-13
5 0.3270754125466E+04 0.3270754125466E+04 0.8481098651866E-14
Comparison of RMS-norms of solution error
1 0.5296984714094E+02 0.5296984714094E+02 0.2682819657265E-15
2 0.4463289611567E+01 0.4463289611567E+01 0.1989963674771E-15
3 0.1312257334221E+02 0.1312257334221E+02 0.4060995034457E-15
4 0.1200692532356E+02 0.1200692532356E+02 0.2958887128106E-15
5 0.1245957615104E+03 0.1245957615104E+03 0.2281113665977E-15
Verification Successful
BT Benchmark Completed.
Class = B
Size = 102x 102x 102
Iterations = 200
Time in seconds = 260.37
Total processes = 1
Compiled procs = 1
Mop/s total = 2696.83
Mop/s/process = 2696.83
Operation type = floating point
Verification = SUCCESSFUL
Version = 3.3
Compile date = 25 Dec 2014
Compile options:
MPIF77 = mpif77
FLINK = $(MPIF77)
FMPI_LIB = -L/usr/lib/openmpi/lib -lmpi -lopen-rte -lo...
FMPI_INC = -I/usr/lib/openmpi/include -I/usr/lib/openm...
FFLAGS = -O
FLINKFLAGS = -O
RAND = (none)
Please send the results of this run to:
NPB Development Team
Internet: npb#nas.nasa.gov
If email is not available, send this to:
MS T27A-1
NASA Ames Research Center
Moffett Field, CA 94035-1000
Fax: 650-604-3957
Finish in 16:45:14--25/12/2014
-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-
-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-
Start in 16:58:50--25/12/2014
NAS Parallel Benchmarks 3.3 -- BT Benchmark
No input file inputbt.data. Using compiled defaults
Size: 102x 102x 102
Iterations: 200 dt: 0.0003000
Number of active processes: 1
Time step 1
Time step 20
Time step 40
Time step 60
Time step 80
Time step 100
Time step 120
Time step 140
Time step 160
Time step 180
Time step 200
Verification being performed for class B
accuracy setting for epsilon = 0.1000000000000E-07
Comparison of RMS-norms of residual
1 0.1423359722929E+04 0.1423359722929E+04 0.7507984505732E-14
2 0.9933052259015E+02 0.9933052259015E+02 0.3147459568137E-14
3 0.3564602564454E+03 0.3564602564454E+03 0.4783990739472E-14
4 0.3248544795908E+03 0.3248544795908E+03 0.2309751522921E-13
5 0.3270754125466E+04 0.3270754125466E+04 0.8481098651866E-14
Comparison of RMS-norms of solution error
1 0.5296984714094E+02 0.5296984714094E+02 0.2682819657265E-15
2 0.4463289611567E+01 0.4463289611567E+01 0.1989963674771E-15
3 0.1312257334221E+02 0.1312257334221E+02 0.4060995034457E-15
4 0.1200692532356E+02 0.1200692532356E+02 0.2958887128106E-15
5 0.1245957615104E+03 0.1245957615104E+03 0.2281113665977E-15
Verification Successful
BT Benchmark Completed.
Class = B
Size = 102x 102x 102
Iterations = 200
Time in seconds = 260.04
Total processes = 1
Compiled procs = 1
Mop/s total = 2700.25
Mop/s/process = 2700.25
Operation type = floating point
Verification = SUCCESSFUL
Version = 3.3
Compile date = 25 Dec 2014
Compile options:
MPIF77 = mpif77
FLINK = $(MPIF77)
FMPI_LIB = -L/usr/lib/openmpi/lib -lmpi -lopen-rte -lo...
FMPI_INC = -I/usr/lib/openmpi/include -I/usr/lib/openm...
FFLAGS = -O
FLINKFLAGS = -O
RAND = (none)
Please send the results of this run to:
NPB Development Team
Internet: npb#nas.nasa.gov
If email is not available, send this to:
MS T27A-1
NASA Ames Research Center
Moffett Field, CA 94035-1000
Fax: 650-604-3957
Finish in 17:03:12--25/12/2014
-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-
There are 40 entries in the log, but I've provided only 2 for abbreviation purposes.
To fix the first issue, replace:
awk -F= '/Time in seconds/ {s+=$2; c++} END {print s/c}' bt.B.1.log > t1avg.dat
with:
awk 'BEGIN { FS = "[ \t]*=[ \t]*" } /Time in seconds/ { s += $2; c++ } /Compiled procs/ { if (! CP) CP = $2 } END { print s/c, CP }' bt.B.1.log >t1avg.dat
A potential minor issue is that 260.205 1 might be output but the question does not address this as a weakness of the given script. Rounding with something like printf "%.2f %s\n", s/c, CP gives 260.21 1 though. To truncate the extra digit, use something like printf "%.2f %s\n", int (s/c * 100) / 100, CP.
To fix the second issue, replace:
awk 'BEGIN { FS = "Time in seconds =" } ; { printf $2 } {printf " "}' bt.B.1.log > t1.dat
with:
awk 'BEGIN { FS = "[ \t]*[=][ \t]" } /Time in seconds/ { printf "%s", $2 } /Compiled procs/ { if (CP) { printf "\n" } else { CP = $2; printf " %s\n", $2 } }' bt.B.1.log > t1.dat
BTW, the "strange indentation" is a result of failing to output a newline when using printf and failing to filter unwanted input lines from processing.
Look at this makefile, it has some sort of primitive progress indication (could have been a progress bar).
Please give me suggestions/comments on it!
# BUILD is initially undefined
ifndef BUILD
# max equals 256 x's
sixteen := x x x x x x x x x x x x x x x x
MAX := $(foreach x,$(sixteen),$(sixteen))
# T estimates how many targets we are building by replacing BUILD with a special string
T := $(shell $(MAKE) -nrRf $(firstword $(MAKEFILE_LIST)) $(MAKECMDGOALS) \
BUILD="COUNTTHIS" | grep -c "COUNTTHIS")
# N is the number of pending targets in base 1, well in fact, base x :-)
N := $(wordlist 1,$T,$(MAX))
# auto-decrementing counter that returns the number of pending targets in base 10
counter = $(words $N)$(eval N := $(wordlist 2,$(words $N),$N))
# BUILD is now defined to show the progress, this also avoids redefining T in loop
BUILD = #echo $(counter) of $(T)
endif
# dummy phony targets
.PHONY: all clean
all: target
#echo done
clean:
#rm -f target *.c
# dummy build rules
target: a.c b.c c.c d.c e.c f.c g.c
#touch $#
$(BUILD)
%.c:
#touch $#
$(BUILD)
All suggestions welcome!
This one is less intrusive and more awesome.
ifneq ($(words $(MAKECMDGOALS)),1)
.DEFAULT_GOAL = all
%:
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST))
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = echo "`expr " [\`expr $C '*' 100 / $T\`" : '.*\(....\)$$'`%]"
endif
.PHONY: all clean
all: target
#$(ECHO) All done
clean:
#rm -f target *.c
# #$(ECHO) Clean done
target: a.c b.c c.c d.c e.c
#$(ECHO) Linking $#
#sleep 0.1
#touch $#
%.c:
#$(ECHO) Compiling $#
#sleep 0.1
#touch $#
endif
There wasn't really a question so this is less of a standalone answer and more of an extension to Giovanni Funchai's solution. This question is the first google result for "GNU Make Progress" so I ended up here looking for how to do this.
As pointed out by Rob Wells, the solution doesn't work for <10%, but the technique can be extended with the print formatting done by a helper script in whatever language you feel is portable enough for your build. For example, using a python helper script:
echo_progress.py:
"""
Print makefile progress
"""
import argparse
import math
import sys
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stepno", type=int, required=True)
parser.add_argument("--nsteps", type=int, required=True)
parser.add_argument("remainder", nargs=argparse.REMAINDER)
args = parser.parse_args()
nchars = int(math.log(args.nsteps, 10)) + 1
fmt_str = "[{:Xd}/{:Xd}]({:6.2f}%)".replace("X", str(nchars))
progress = 100 * args.stepno / args.nsteps
sys.stdout.write(fmt_str.format(args.stepno, args.nsteps, progress))
for item in args.remainder:
sys.stdout.write(" ")
sys.stdout.write(item)
sys.stdout.write("\n")
if __name__ == "__main__":
main()
And the modified Makefile:
_mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
I := $(patsubst %/,%,$(dir $(_mkfile_path)))
ifneq ($(words $(MAKECMDGOALS)),1)
.DEFAULT_GOAL = all
%:
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST))
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = python $(I)/echo_progress.py --stepno=$C --nsteps=$T
endif
.PHONY: all clean
all: target
#$(ECHO) All done
clean:
#rm -f target *.c
# #$(ECHO) Clean done
target: a.c b.c c.c d.c e.c f.c g.c h.c i.c j.c k.c l.c m.c n.c o.c p.c q.c \
r.c s.c t.c u.c v.c w.c x.c y.c z.c
#$(ECHO) Linking $#
#sleep 0.01
#touch $#
%.c:
#$(ECHO) Compiling $#
#sleep 0.01
#touch $#
endif
yields:
$ make
[ 1/28]( 3.57%) Compiling a.c
[ 2/28]( 7.14%) Compiling b.c
[ 3/28]( 10.71%) Compiling c.c
[ 4/28]( 14.29%) Compiling d.c
[ 5/28]( 17.86%) Compiling e.c
[ 6/28]( 21.43%) Compiling f.c
[ 7/28]( 25.00%) Compiling g.c
[ 8/28]( 28.57%) Compiling h.c
[ 9/28]( 32.14%) Compiling i.c
[10/28]( 35.71%) Compiling j.c
[11/28]( 39.29%) Compiling k.c
[12/28]( 42.86%) Compiling l.c
[13/28]( 46.43%) Compiling m.c
[14/28]( 50.00%) Compiling n.c
[15/28]( 53.57%) Compiling o.c
[16/28]( 57.14%) Compiling p.c
[17/28]( 60.71%) Compiling q.c
[18/28]( 64.29%) Compiling r.c
[19/28]( 67.86%) Compiling s.c
[20/28]( 71.43%) Compiling t.c
[21/28]( 75.00%) Compiling u.c
[22/28]( 78.57%) Compiling v.c
[23/28]( 82.14%) Compiling w.c
[24/28]( 85.71%) Compiling x.c
[25/28]( 89.29%) Compiling y.c
[26/28]( 92.86%) Compiling z.c
[27/28]( 96.43%) Linking target
[28/28](100.00%) All done
One could even print a fancy progress bar with unicode characters.
Modified echo_progress.py:
"""
Print makefile progress
"""
import argparse
import math
import sys
def get_progress_bar(numchars, fraction=None, percent=None):
"""
Return a high resolution unicode progress bar
"""
if percent is not None:
fraction = percent / 100.0
if fraction >= 1.0:
return "█" * numchars
blocks = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"]
length_in_chars = fraction * numchars
n_full = int(length_in_chars)
i_partial = int(8 * (length_in_chars - n_full))
n_empty = max(numchars - n_full - 1, 0)
return ("█" * n_full) + blocks[i_partial] + (" " * n_empty)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stepno", type=int, required=True)
parser.add_argument("--nsteps", type=int, required=True)
parser.add_argument("remainder", nargs=argparse.REMAINDER)
args = parser.parse_args()
nchars = int(math.log(args.nsteps, 10)) + 1
fmt_str = "\r[{:Xd}/{:Xd}]({:6.2f}%) ".replace("X", str(nchars))
progress = 100 * args.stepno / args.nsteps
sys.stdout.write(fmt_str.format(args.stepno, args.nsteps, progress))
sys.stdout.write(get_progress_bar(20, percent=progress))
remainder_str = " ".join(args.remainder)
sys.stdout.write(" {:20s}".format(remainder_str[:20]))
if args.stepno == args.nsteps:
sys.stdout.write("\n")
if __name__ == "__main__":
main()
Which would result in something like this:
$ make clean && make
[12/28]( 42.86%) ███████▊ Compiling k.c
during progress and:
$ make clean && make
[28/28](100.00%) ████████████████████ All done
upon completion.
This is a slight modification to #GiovanniFunchal's excellent answer.
So I wanted to understand this better and make it work for < 10% so I dug into the documentation and learned more about expr.
# PLACE AT THE TOP OF YOUR MAKEFILE
#---------------------------------
# Progress bar defs
#--------------------------------
# words = count the number of words
ifneq ($(words $(MAKECMDGOALS)),1) # if no argument was given to make...
.DEFAULT_GOAL = all # set the default goal to all
# http://www.gnu.org/software/make/manual/make.html
# $# = target name
# %: = last resort recipe
# --no-print-directory = don't print enter/leave messages for each output grouping
# MAKEFILE_LIST = has a list of all the parsed Makefiles that can be found *.mk, Makefile, etc
# -n = dry run, just print the recipes
# -r = no builtin rules, disables implicit rules
# -R = no builtin variables, disables implicit variables
# -f = specify the name of the Makefile
%: # define a last resort default rule
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST)) # recursive make call,
else
ifndef ECHO
# execute a dry run of make, defining echo beforehand, and count all the instances of "COUNTTHIS"
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
# eval = evaluate the text and read the results as makefile commands
N := x
# Recursively expand C for each instance of ECHO to count more x's
C = $(words $N)$(eval N := x $N)
# Multipy the count of x's by 100, and divide by the count of "COUNTTHIS"
# Followed by a percent sign
# And wrap it all in square brackets
ECHO = echo -ne "\r [`expr $C '*' 100 / $T`%]"
endif
#------------------
# end progress bar
#------------------
# REST OF YOUR MAKEFILE HERE
#----- Progressbar endif at end Makefile
endif
I got rid of the : '.*\(....\)$$' part. It would return the last 4 characters of the inner expr command, but would fail if it was less than 4. And now it works for sub 10%!
And here is the comment free version:
ifneq ($(words $(MAKECMDGOALS)),1) # if no argument was given to make...
.DEFAULT_GOAL = all # set the default goal to all
%: # define a last resort default rule
#$(MAKE) $# --no-print-directory -rRf $(firstword $(MAKEFILE_LIST)) # recursive make call,
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
-nrRf $(firstword $(MAKEFILE_LIST)) \
ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = echo -ne "\r [`expr $C '*' 100 / $T`%]"
endif
# ...
endif
Hope that helps.
Many thanks to #Giovanni Funchal and #phyatt for the question and answers!
I just simplified it even more for my own better understanding.
ifndef ECHO
HIT_TOTAL != ${MAKE} ${MAKECMDGOALS} --dry-run ECHO="HIT_MARK" | grep -c "HIT_MARK"
HIT_COUNT = $(eval HIT_N != expr ${HIT_N} + 1)${HIT_N}
ECHO = echo "[`expr ${HIT_COUNT} '*' 100 / ${HIT_TOTAL}`%]"
endif
!= assigns from shell command
= evaluates variable each time it's used
eval executes its argument without any output
expr allows to make arithmetic calculations
( Not sure though which approach is faster: to call shell with expr or to count 'x'-es with make. )
Usage is the same:
target:
#$(ECHO) $#
Nice trick! (-:
But not really scalable for growing projects that are distributed across many directories with lots of makefiles.
I'd be more inclined to have logging sprinkled through the [Mm]akefiles* in your project and use that to keep track of progress.
Just a thought. BTW Thanks for sharing this.
Edit: Just had a thought. This could be useful in a modified form to display a throbber to show progress while a long task proceeds, e.g unpacking a large distribution tarball instead of just specifying the -v option to the tar command. Still a bit of sugar coating but a bit of fun aswell. (-:
cheers,
Rob