How to convert row to column on specific condition using awk? - awk

Question

I have a file while I want to convert from row to column on specific condition.
Input file:
cat f
"0/35","0eij8401c
"0/35","59ij41015
"0/35","21ij3e01c
"0/35","dbije401b
"1/35","dbij8a015
"1/35","67ijb9011
"1/35","b5ije001b
"1/35","bdij3701d
"2/35","abij3b011
"2/35","7fij70018
"2/35","77ijf9010
"2/35","e5ij64015
"3/35","59ij41015
"3/35","f6ijae01e
"3/35","c4ij5801c
"3/35","dbij98012
"4/35","edij6801e
"4/35","pdij6801e
"4/35","kdij6801e
"4/35","8cij57018
NOTE: here I am fetching 1st, 5th, 9th, 13th and 17th row's second column in first column in output below. like wise 2nd, 6th, 10th, 14th and 18th row's second column to print second column in output and same for rest of the rows.
There are two expected output:
Expected output 1: (To see it in a report format)
"0eij8401c "dbij8a015 "abij3b011 "59ij41015 "edij6801e
"59ij41015 "67ijb9011 "7fij70018 "f6ijae01e "pdij6801e
"21ij3e01c "b5ije001b "77ijf9010 "c4ij5801c "kdij6801e
"dbije401b "bdij3701d "e5ij64015 "dbij98012 "8cij57018
Expected output2:
And then convert the expected output1 into a single column to perform some operation:
0eij8401c
dbij8a015
abij3b011
59ij41015
edij6801e
59ij41015
67ijb9011
7fij70018
f6ijae01e
pdij6801e
21ij3e01c
b5ije001b
77ijf9010
c4ij5801c
kdij6801e
dbije401b
bdij3701d
e5ij64015
dbij98012
8cij57018
I tried combination of awk and paste, trying to achieve both with awk command.
This is what I tried -
cat f | awk -v batchNo=1 -v Num=4 '{print $1 > "batch_" batchNo ".txt";if(NR%Num==0) {batchNo++}}'
to generate 5 files like below -
ls batch_*
batch_1.txt batch_2.txt batch_3.txt batch_4.txt batch_5.txt
and then combined with paste like below -
paste batch_1.txt batch_2.txt batch_3.txt batch_4.txt batch_5.txt
"0eij8401c "dbij8a015 "abij3b011 "59ij41015 "edij6801e
"59ij41015 "67ijb9011 "7fij70018 "f6ijae01e "pdij6801e
"21ij3e01c "b5ije001b "77ijf9010 "c4ij5801c "kdij6801e
"dbije401b "bdij3701d "e5ij64015 "dbij98012 "8cij57018
I also tried something like this to get the desired result but didn't get it.
awk '{a[$1]++; b[$2]++;c[$3]++;d[$4]++;e[$5]++} END {for (k in a) print k > "out.txt"; for (j in b) print j > "out.txt";for (k in c) print j > "out.txt";for(l in d) print l> "out.txt"; for (m in e) print m> "out.txt";}' batch_*
Any suggestion please.

Answer 1

In addition to the other two good answers, there is yet another simplified way to approach each of your separate output problems. In the first case, you can simply save the values from the second column in an indexed array and then output in rows by groups of 5, e.g.
awk -F, '
{ a[++n] = $2 }
END {
for (i=1; i<=(n/5); i++)
printf "%s %s %s %s %s\n", a[i], a[i+4], a[i+8], a[i+12], a[i+16]
}
' f
Output
"0eij8401c "dbij8a015 "abij3b011 "59ij41015 "edij6801e
"59ij41015 "67ijb9011 "7fij70018 "f6ijae01e "pdij6801e
"21ij3e01c "b5ije001b "77ijf9010 "c4ij5801c "kdij6801e
"dbije401b "bdij3701d "e5ij64015 "dbij98012 "8cij57018
If you need the column output in the specific order shown, you can use the approach to save to an indexed array and then output with '\n' separators instead along with trimming the first char with substr(), e.g.
awk -F, '
{ a[++n]=$2 }
END {
for (i=1; i<=(n/5); i++)
printf "%s\n%s\n%s\n%s\n%s\n", substr(a[i],2), substr(a[i+4],2),
substr(a[i+8],2), substr(a[i+12],2), substr(a[i+16],2)
}
' f
Output
0eij8401c
dbij8a015
abij3b011
59ij41015
edij6801e
59ij41015
67ijb9011
7fij70018
f6ijae01e
pdij6801e
21ij3e01c
b5ije001b
77ijf9010
c4ij5801c
kdij6801e
dbije401b
bdij3701d
e5ij64015
dbij98012
8cij57018
If you just need a column of output of the 2nd field, regardless of order, you can simply use substring to output all but the first character, e.g.
awk -F, '{ print substr($2,2) }' f
Output
0eij8401c
59ij41015
21ij3e01c
dbije401b
dbij8a015
67ijb9011
b5ije001b
bdij3701d
abij3b011
7fij70018
77ijf9010
e5ij64015
59ij41015
f6ijae01e
c4ij5801c
dbij98012
edij6801e
pdij6801e
kdij6801e
8cij57018

Answer 2

About solutions: 3 of these solutions will print continuous view(details one by one for file's continuity) AND report view as well where you need them horizontally. 1st solution: considers that your Input_file is sorted by digits/digits format. 2nd solution sorts Input_file then does the job. 3rd: solution print both styles and create output file batches also.
1st solution: (Considers that your Input_file is sorted by "digit/digits" format)With your shown samples please try following awk code. This will print the output directly sorting order of 1st field eg: "0/35", "1/35" and so on.
awk -v count=0 -v s1="\"" -F'^"|","' '
prev!=$2{
countFile++
max=(max>count?max:count)
count=1
}
{
arr[countFile,count++]=$3
prev=$2
}
END{
print "Printing continous view from here..."
for(i=1;i<=max;i++){
for(j=1;j<countFile;j++){
print(arr[i,j])
}
}
print "Printing REPORT view from here......"
for(i=1;i<countFile;i++){
for(j=1;j<=max;j++){
printf("%s%s",s1 arr[j,i],j==max?ORS:OFS)
}
}
}
' Input_file
2nd solution: In case your Input_file is NOT sorted with("digit/digits" format) then try this code.
awk -F'^"|","' '{print $2,$0}' Input_file | sort -t/ -nk1 -k2 | cut -d' ' -f2 |
awk -v count=0 -v s1="\"" -F'^"|","' '
prev!=$2{
countFile++
max=(max>count?max:count)
count=1
}
{
arr[countFile,count++]=$3
prev=$2
}
END{
print "Printing continous view from here..."
for(i=1;i<=max;i++){
for(j=1;j<countFile;j++){
print(arr[i,j])
}
}
print "Printing REPORT view from here......"
for(i=1;i<countFile;i++){
for(j=1;j<=max;j++){
printf("%s%s",s1 arr[j,i],j==max?ORS:OFS)
}
}
}
'
OR 3rd solution: In case you want to print data on screen as well as you want to create output files also within same awk program then try following:
awk -v count=0 -v s1="\"" -F'^"|","' '
prev!=$2{
close(outputFile)
countFile++
outputFile="batch_"countFile".txt"
max=(max>count?max:count)
count=1
}
{
arr[countFile,count++]=$3
prev=$2
print (s1 $3) > (outputFile)
}
END{
print "Printing continous view from here..."
for(i=1;i<=max;i++){
for(j=1;j<countFile;j++){
print(arr[i,j])
}
}
print "Printing REPORT view from here......"
for(i=1;i<countFile;i++){
for(j=1;j<=max;j++){
printf("%s%s",s1 arr[j,i],j==max?ORS:OFS)
}
}
}
' Input_file
Output will be as follows which it will print:
"0eij8401c "dbij8a015 "abij3b011 "59ij41015 "edij6801e
"59ij41015 "67ijb9011 "7fij70018 "f6ijae01e "pdij6801e
"21ij3e01c "b5ije001b "77ijf9010 "c4ij5801c "kdij6801e
"dbije401b "bdij3701d "e5ij64015 "dbij98012 "8cij57018

Answer 3

As your shown input is already sorted on first field, you may use this solution:
awk -F, '{gsub(/^"|\/[0-9]+"/, ""); print $2 > "batch_" ($1+1) ".txt"}' f
paste batch_1.txt batch_2.txt batch_3.txt batch_4.txt batch_5.txt
"0eij8401c "dbij8a015 "abij3b011 "59ij41015 "edij6801e
"59ij41015 "67ijb9011 "7fij70018 "f6ijae01e "pdij6801e
"21ij3e01c "b5ije001b "77ijf9010 "c4ij5801c "kdij6801e
"dbije401b "bdij3701d "e5ij64015 "dbij98012 "8cij57018
For output2 as per edited question use:
awk '{
a[FNR] = a[FNR] substr($0,2) "\n"
}
END {
for (i=1; i<=FNR; ++i) printf "%s", a[i]
}' batch_1.txt batch_2.txt batch_3.txt batch_4.txt batch_5.txt
0eij8401c
dbij8a015
abij3b011
59ij41015
edij6801e
59ij41015
67ijb9011
7fij70018
f6ijae01e
pdij6801e
21ij3e01c
b5ije001b
77ijf9010
c4ij5801c
kdij6801e
dbije401b
bdij3701d
e5ij64015
dbij98012
8cij57018

Answer 4

Using any awk:
$ cat tst.awk
BEGIN { FS="\"" }
{ vals[++numVals] = $NF }
END {
numValsPerBatch = int(numVals / numBatches) + ( numVals % numBatches ? 1 : 0 )
for ( batchNr=1; batchNr<=numBatches; batchNr++ ) {
for ( valNr=1; valNr<=numValsPerBatch; valNr++ ) {
valIdx = batchNr + (valNr - 1) * numBatches
printf "%s%s", vals[valIdx], (valNr<numValsPerBatch ? OFS : ORS) > "out1.txt"
print vals[valIdx] > "out2.txt"
}
}
}
$ awk -v numBatches=4 -f tst.awk f
$ head -100 out?.txt
==> out1.txt <==
0eij8401c dbij8a015 abij3b011 59ij41015 edij6801e
59ij41015 67ijb9011 7fij70018 f6ijae01e pdij6801e
21ij3e01c b5ije001b 77ijf9010 c4ij5801c kdij6801e
dbije401b bdij3701d e5ij64015 dbij98012 8cij57018
==> out2.txt <==
0eij8401c
dbij8a015
abij3b011
59ij41015
edij6801e
59ij41015
67ijb9011
7fij70018
f6ijae01e
pdij6801e
21ij3e01c
b5ije001b
77ijf9010
c4ij5801c
kdij6801e
dbije401b
bdij3701d
e5ij64015
dbij98012
8cij57018
or if you want the number of batches to be calculated from the key values (YMMV if there's different numbers of values per key in your input):
$ cat tst.awk
BEGIN { FS="\"" }
!seen[$2]++ { numKeys++ }
{ vals[++numVals] = $NF }
END {
numBatches = int(numVals / numKeys) + (numVals % numKeys ? 1 : 0)
numValsPerBatch = int(numVals / numBatches) + (numVals % numBatches ? 1 : 0)
for ( batchNr=1; batchNr<=numBatches; batchNr++ ) {
for ( valNr=1; valNr<=numValsPerBatch; valNr++ ) {
valIdx = batchNr + (valNr - 1) * numBatches
printf "%s%s", vals[valIdx], (valNr<numValsPerBatch ? OFS : ORS) > "out1.txt"
print vals[valIdx] > "out2.txt"
}
}
}
$ awk -f tst.awk f
$ head -100 out?.txt
==> out1.txt <==
0eij8401c dbij8a015 abij3b011 59ij41015 edij6801e
59ij41015 67ijb9011 7fij70018 f6ijae01e pdij6801e
21ij3e01c b5ije001b 77ijf9010 c4ij5801c kdij6801e
dbije401b bdij3701d e5ij64015 dbij98012 8cij57018
==> out2.txt <==
0eij8401c
dbij8a015
abij3b011
59ij41015
edij6801e
59ij41015
67ijb9011
7fij70018
f6ijae01e
pdij6801e
21ij3e01c
b5ije001b
77ijf9010
c4ij5801c
kdij6801e
dbije401b
bdij3701d
e5ij64015
dbij98012
8cij57018

Answer 5

TXR solution:
#(collect)
# (all)
"#id/#nil
# (and)
# (collect :gap 0)
"#id/#nil","#data
# (bind qdata `"#data`)
# (end)
# (end)
#(end)
#(bind tdata #(transpose qdata))
#(bind fdata #(flatten (transpose data)))
#(output)
# (repeat)
#{tdata " "}
# (end)
# (repeat)
#fdata
# (end)
#(end)
$ txr soln.txr data
"0eij8401c "dbij8a015 "abij3b011 "59ij41015 "edij6801e
"59ij41015 "67ijb9011 "7fij70018 "f6ijae01e "pdij6801e
"21ij3e01c "b5ije001b "77ijf9010 "c4ij5801c "kdij6801e
"dbije401b "bdij3701d "e5ij64015 "dbij98012 "8cij57018
0eij8401c
dbij8a015
abij3b011
59ij41015
edij6801e
59ij41015
67ijb9011
7fij70018
f6ijae01e
pdij6801e
21ij3e01c
b5ije001b
77ijf9010
c4ij5801c
kdij6801e
dbije401b
bdij3701d
e5ij64015
dbij98012
8cij57018