AWK: sums problematic in awk table transform script - awk

Question

I have the following file:
cat st_in.txt
2015-01-01 2 A FI
2015-02-03 4 B VI
2015-03-01 6 A FI
2015-01-08 -4 C VE
2016-01-05 -3 B VE
2016-02-03 -1 D FE
2016-04-01 -2 B FE
2016-06-13 -5 D VE
2017-01-01 2 A VI
2017-02-03 3 A VI
2017-02-04 8 C FI
2017-01-05 -1 B FE
And want to transform it like this:
2015 2016 2017
A 8.00 0.00 0.00
C 0.00 0.00 8.00
sumFI 8.00 0.00 8.00
A 0.00 0.00 5.00
B 4.00 0.00 0.00
sumVI 4.00 0.00 5.00
sumI 12.00 0.00 13.00
B 0.00 -2.00 -1.00
D 0.00 -1.00 0.00
sumFE 0.00 -3.00 -1.00
B 0.00 -3.00 0.00
C -4.00 0.00 0.00
D 0.00 -5.00 0.00
sumVE -4.00 -8.00 0.00
sumE -4.00 -11.00 -1.00
net 8.00 -11.00 12.00
To do that I have written the following script:
#!/usr/bin/env bash
awk '
BEGIN {
OFS = "\t"
}
{
yr = substr($1, 1, 4)
sub(/-.*/, "", $1)
minYr = (NR == 1 || $1 < minYr ? $1 : minYr)
maxYr = (NR == 1 || $1 > maxYr ? $1 : maxYr)
H[$4][$3]
W[yr][$4][$3] += $2
yT[$1] += $2
val[$1][$4] += $2
}
END {
for (yr = minYr; yr <= maxYr; yr++) {
printf "%s%s", OFS, yr
}
print ""
print ""
for (cT in H) {
for (c in H[cT]) {
printf c, OFS
for (yr = minYr; yr <= maxYr; yr++) {
printf "%s%0.2f", OFS, W[yr][cT][c]
}
print ""
}
printf "sum" cT, OFS
for (yr = minYr; yr <= maxYr; yr++) {
printf "%s%0.2f", OFS, val[yr][cT]
}
print ""
print ""
if (cT == "VI") {
printf "sumI", OFS
for (yr = minYr; yr <= maxYr; yr++) {
printf "%s%0.2f", OFS, W[yr][cT][c]
}
} else if (cT == "VE") {
printf "sumE", OFS
for (yr = minYr; yr <= maxYr; yr++) {
printf "%s%0.2f", OFS, W[yr][cT][c]
}
}
print ORS
}
printf "net"
for (yr = minYr; yr <= maxYr; yr++) {
printf "%s%0.2f", OFS, yT[yr]
}
print ""
}
' "${#:--}"
Which prints the following:
./trans1 st_in.txt
2015 2016 2017
A 8.00 0.00 0.00
C 0.00 0.00 8.00
sumFI 8.00 0.00 8.00
A 0.00 0.00 5.00
B 4.00 0.00 0.00
sumVI 4.00 0.00 5.00
sumI 4.00 0.00 0.00
B 0.00 -2.00 -1.00
D 0.00 -1.00 0.00
sumFE 0.00 -3.00 -1.00
B 0.00 -3.00 0.00
C -4.00 0.00 0.00
D 0.00 -5.00 0.00
sumVE -4.00 -8.00 0.00
sumE 0.00 -5.00 0.00
net 8.00 -11.00 12.00
I am not worried about the formatting here. The console printout is similar to the target. But what I am struggling with is to get the correct sums for sumI (sumFI + sumVI) and sumE(sumFE + sumVE). Can someone please help with that?
EDIT by Ed Morton to provide meaningful data types and variable names, assuming the OPs current data is column types of Date Amount Item Type as it was in their first question:
$ cat trans1
#!/usr/bin/env bash
awk '
BEGIN {
OFS = "\t"
}
{
date = $1
amount = $2
item = $3
type = $4
year = substr(date, 1, 4)
minYear = (NR == 1 || year < minYear ? year : minYear)
maxYear = (NR == 1 || year > maxYear ? year : maxYear)
types_items[type][item]
yearsTypesItems2amounts[year][type][item] += $2
years2amounts[year] += $2
yearsTypes2amounts[year][type] += $2
}
END {
for (year = minYear; year <= maxYear; year++) {
printf "%s%s", OFS, year
}
print ""
print ""
for (type in types_items) {
for (item in types_items[type]) {
printf item, OFS
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, yearsTypesItems2amounts[year][type][item]
}
print ""
}
printf "sum" type, OFS
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, yearsTypes2amounts[year][type]
}
print ""
print ""
if (type == "VI") {
printf "sumI", OFS
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, yearsTypesItems2amounts[year][type][item] # <--- NOTE: item unset here
}
} else if (type == "VE") {
printf "sumE", OFS
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, yearsTypesItems2amounts[year][type][item] # <--- NOTE: item unset here
}
}
print ORS
}
printf "net"
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, years2amounts[year]
}
print ""
}
' "${#:--}"
See the "NOTE"s above where by just doing this renaming exercise it became obvious where some bugs are (looks like you were using the wrong array and should be using yearsTypes2amounts[year][type] instead of yearsTypesItems2amounts[year][type][item]).

Answer 1

Continuing to use GNU awk for arrays of arrays and assuming the OPs current data is column types of Date Amount Item Type as it was in your first question:
$ cat trans1
#!/usr/bin/env bash
awk '
BEGIN {
OFS = "\t"
}
{
date = $1
amount = $2
item = $3
type = $4
year = substr(date, 1, 4)
minYear = (NR == 1 || year < minYear ? year : minYear)
maxYear = (NR == 1 || year > maxYear ? year : maxYear)
cat = substr(type,2)
ctiys2amounts[cat][type][item][year] += amount
}
END {
for (year = minYear; year <= maxYear; year++) {
printf "%s%s", OFS, year
}
print ORS
for (cat in ctiys2amounts) {
delete catSum
for (type in ctiys2amounts[cat]) {
delete typeSum
for (item in ctiys2amounts[cat][type]) {
printf "%s", item
for (year = minYear; year <= maxYear; year++) {
amount = ctiys2amounts[cat][type][item][year]
printf "%s%0.2f", OFS, amount
typeSum[year] += amount
}
print ""
}
printf "sum%s", type
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, typeSum[year]
catSum[year] += typeSum[year]
}
print ORS
}
printf "sum%s", cat
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, catSum[year]
yearSum[year] += catSum[year]
}
print ORS
}
printf "net"
for (year = minYear; year <= maxYear; year++) {
printf "%s%0.2f", OFS, yearSum[year]
}
print ""
}
' "${#:--}"
$ ./trans1 st_in.txt
2015 2016 2017
B 0.00 -2.00 -1.00
D 0.00 -1.00 0.00
sumFE 0.00 -3.00 -1.00
B 0.00 -3.00 0.00
C -4.00 0.00 0.00
D 0.00 -5.00 0.00
sumVE -4.00 -8.00 0.00
sumE -4.00 -11.00 -1.00
A 8.00 0.00 0.00
C 0.00 0.00 8.00
sumFI 8.00 0.00 8.00
A 0.00 0.00 5.00
B 4.00 0.00 0.00
sumVI 4.00 0.00 5.00
sumI 12.00 0.00 13.00
net 8.00 -11.00 12.00