Union "tables" with awk - awk

Question

I have multiple "tables" in a file, such as:
col1, col2, col3, col4
1, 2, 3, 4
5, 6, 7, 8
col2, col3, col5
10, 11, 12
13, 14, 15
And I would like to collapse these 2 tables to:
col1, col2, col3, col4, col5
1 , 2 , 3 , 4 ,
5 , 6 , 7 , 8 ,
, 10 , 11 , , 12
, 13 , 14 , , 15
(Note: extra whitespace left just to make things easier to understand)
This would seem to require at least 2 passes, one to collect the full list of columns, and another one to create the output table. Is it possible to do this with awk? If not, what other tool would you recommend?

Answer 1

give this a try:
Code:
$ cat s.awk
NR==FNR{
if (match($1, /^col/))
maxIndex=(substr($NF,4,1)>maxIndex)?substr($NF,4,1):maxColumn
next
}
FNR==1{
for (i=1;i<=maxIndex;i++)
header=(i==maxIndex)?header "col"i:header "col" i ", "
print header
}
/^col[1-9]/{
for (i in places)
delete places[i]
for (i=1;i<=NF;i++){
n=substr($i,4,1)
places[n]=i
}
}
/^[0-9]/{
s=""
for (i=1;i<=maxIndex;i++)
s=(i in places)? s $places[i] " " : s ", "
print s
}
Call with:
awk -f s.awk file file | column -t
Output:
col1, col2, col3, col4, col5
1, 2, 3, 4 ,
5, 6, 7, 8 ,
, 10, 11, , 12
, 13, 14, , 15
HTH Chris

Answer 2

The code assumes that the tables are separated by empty lines:
awk -F', *' 'END {
for (i = 0; ++i <= c;)
printf "%s", (cols[i] (i < c ? OFS : RS))
for (i = 0; ++i <= n;)
for (j = 0; ++j <= c;)
printf "%s", (vals[i, cols[j]] (j < c ? OFS : RS))
}
!NF {
fnr = NR + 1; next
}
NR == 1 || NR == fnr {
for (i = 0; ++i <= NF;) {
_[$i]++ || cols[++c] = $i
idx[i] = $i
}
next
}
{
++n; for (i = 0; ++i <= NF;)
vals[n, idx[i]] = $i
}' OFS=', ' tables
If you have the tables in separate files:
awk -F', *' 'END {
for (i = 0; ++i <= c;)
printf "%s", (cols[i] (i < c ? OFS : RS))
for (i = 0; ++i <= n;)
for (j = 0; ++j <= c;)
printf "%s", (vals[i, cols[j]] (j < c ? OFS : RS))
}
FNR == 1 {
for (i = 0; ++i <= NF;) {
_[$i]++ || cols[++c] = $i
idx[i] = $i
}
next
}
{
++n; for (i = 0; ++i <= NF;)
vals[n, idx[i]] = $i
}' OFS=', ' file1 file2 [.. filen]

Answer 3

Here's a one-pass perl solution. It assumes there is at least one blank line between each table in the file.
perl -00 -ne '
BEGIN {
%column2idx = ();
#idx2column = ();
$lineno = 0;
#lines = ();
}
chomp;
#rows = split /\n/;
#field_map = ();
#F = split /, /, $rows[0];
for ($i=0; $i < #F; $i++) {
if (not exists $column2idx{$F[$i]}) {
$idx = #idx2column;
$column2idx{$F[$i]} = $idx;
$idx2column[$idx] = $F[$i];
}
$field_map[$i] = $column2idx{$F[$i]};
}
for ($i=1; $i < #rows; $i++) {
#{$lines[$lineno]} = ();
#F = split /, /, $rows[$i];
for ($j=0; $j < #F; $j++) {
$lines[$lineno][$field_map[$j]] = $F[$j];
}
$lineno++;
}
END {
$ncols = #idx2column;
print join(", ", #idx2column), "\n";
foreach $row (#lines) {
#row = ();
for ($i=0; $i < $ncols; $i++) {
push #row, $row->[$i];
}
print join(", ", #row), "\n";
}
}
' tables | column -t
output
col1, col2, col3, col4, col5
1, 2, 3, 4,
5, 6, 7, 8,
, 10, 11, , 12
, 13, 14, , 15