I have a very large variant calling data. I can not pull out the result I want.
here is an example
bac1 bac2 bac3 bac4
1 0 0 1
Now I want to drop the columns that contain 0 using the ubuntu command-line. The result would be like this
bac1 bac4
1 1
I tried this
awk -F "\t" -v "pat=0\t" 'NR == 2 {for (i=1; i <= NF; i++) Take[i] = (pat != $i)}{for (i =1; i <= NF; i++) if (Take [i]) printf $i FS; print ""}'
And the output is this:
NC_045512.2 18876 NC_045512.2_18876_T_C T C . PASS GT 1
Header of this output is:
#CHROM POS ID REF ALT QUAL FILTER FORMAT EPI_ISL_422804
So the final output had to be like this:
#CHROM POS ID REF ALT QUAL FILTER FORMAT EPI_ISL_422804
NC_045512.2 18876 NC_045512.2_18876_T_C T C . PASS GT 1
The file is not always 2 lines but at most it can be 4 lines.
It does not return the header line that's because I used NR == 2. Is there any way I cant get the header column as well??
If your input file always only has 1 data line as in your example then:
$ cat tst.awk
BEGIN { FS=OFS="\t" }
NR == 1 { split($0,hdr); next }
{
for (i = 1; i <= NF; i++) {
if ($i != 0) {
cols[++nf] = i
}
}
for (i = 1; i <= nf; i++) {
printf "%s%s", hdr[cols[i]], (i<nf ? OFS : ORS)
}
for (i = 1; i <= nf; i++) {
printf "%s%s", $(cols[i]), (i<nf ? OFS : ORS)
}
}
.
$ awk -f tst.awk file
bac1 bac4
1 1
otherwise if your input can have more than 1 data line then you need a 2-pass approach:
$ cat tst.awk
BEGIN { FS=OFS="\t" }
NR == FNR {
if (NR > 1) {
for (i = 1; i <= NF; i++) {
if ($i == 0) {
zeroCols[i]
}
}
}
next
}
FNR == 1 {
for (i = 1; i <= NF; i++) {
if (! (i in zeroCols) ) {
cols[++nf] = i
}
}
}
{
for (i = 1; i <= nf; i++) {
printf "%s%s", $(cols[i]), (i<nf ? OFS : ORS)
}
}
.
$ awk -f tst.awk file file
bac1 bac4
1 1
Long version with if:
awk 'NR==1{
split($0,array,FS)
}
NR==2{
s=0
for(i=1;i<=NF;i++){
if($i!=0){
if(s==0){
s=1
printf("%s",array[i])
}
else{
printf("%s%s",OFS,array[i])
}
}
}
print ""
s=0
for(i=1;i<=NF;i++){
if($i!=0){
if(s==0){
s=1
printf("%s",$i)
}
else{
printf("%s%s",OFS,$i)
}
}
}
print ""
}' FS='\t' OFS="\t" file
One line:
awk 'NR==1{split($0,array,FS)} NR==2{s=0; for(i=1;i<=NF;i++) {if($i!=0) {if(s==0) {s=1; printf("%s",array[i])} else {printf("%s%s",OFS,array[i])}}} print ""; s=0; for(i=1;i<=NF;i++){if($i!=0){if(s==0){s=1; printf("%s",$i)} else {printf("%s%s",OFS,$i)}}} print ""}' FS='\t' OFS="\t" file
Output:
bac1 bac4
1 1
I browse 2 files with awk. I browse the first file and store the columns I need in arrays. I use after these arrays to make a comparison with a column (8) of the second file.
my script runs very slowly. I would like to know if there is not a way to optimize it?
FNR==NR
{
a[$1];
ip[NR]=$1;
site[NR]=$2;
next
}
BEGIN{
FS="[\t,=]";
OFS="|";
}
sudo awk -f{
l=length(ip);
if($8 in a)
{
for(k=0;k<=l;k++)
{
if(ip[k]== $8)
{
if(NF <= 70)
{
print "siteID Ipam: "site[k],"siteID zsc: "$14,"date: " $4,"src: "$8,"dst: "$10,"role: "$22,"urlcategory: "$36, "urlsupercategory: "$38,"urlclass: "$40;
}
else
{
print "siteID Ipam: "site[k], "siteID zsc: "$14,"date: " $4, "src: " $8, "dst: " $10, "role: "$22, "urlcategory: " $37, "urlsupercategory: "$39, "urlclass: $41;
}
break;
}
}
}
else
{
print $8 " is not in referentiel ";
}
}
Here is a better formatted same code with the initial typo.
BEGIN {
FS = "[\t,=]";
OFS = "|";
}
FNR == NR {
a[$1];
ip[NR] = $1;
site[NR] = $2;
next;
}
sudo awk -f {
l = length(ip);
if($8 in a) {
for(k = 0; k <= l; k++) {
if(ip[k] == $8) {
if(NF <= 70) {
print "siteID Ipam: "site[k],"siteID zsc: "$14,"date: " $4,"src: "$8,"dst: "$10,"role: "$22,"urlcategory: "$36, "urlsupercategory: "$38,"urlclass: "$40;
}
else {
print "siteID Ipam: "site[k], "siteID zsc: "$14,"date: " $4, "src: " $8, "dst: " $10, "role: "$22, "urlcategory: " $37, "urlsupercategory: "$39, "urlclass: $41;
}
break;
}
}
} else {
print $8 " is not in referentiel ";
}
}
suggest:
fix sudo awk -f typo.
a[$1]; --> a[$1] = 1;
($8 in a) --> (a[$8])
I have these two csv files:
File A:
veículo;carro;sust
automóvel;carro;sust
viatura;carro;sust
breve;rápido;adj
excepcional;excelente;adj
maravilhoso;excelente;adj
amistoso;simpático;adj
amigável;simpático;adj
...
File B:
"A001","carro","sust","excelente","adj","ocorrer","adv","bom","adj"
...
In the file A, $1(word) is synonym for $2(word) and $3(word) the part of speech.
In the lines of the file B we can skip $1,the remaining columns are words and their part of speech.
What I need to to do is to look line by line each pair (word-pos) in the file A and generate a line for each synonym. It is difficult to explain.
Desired Output:
"A001","carro","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","viatura","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","veículo","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","automóvel","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","carro","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","viatura","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","veículo","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","automóvel","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","carro","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","viatura","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","veículo","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","automóvel","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
Done:
BEGIN {
FS="[,;]";
OFS=";";
}
FNR==NR{
sinonim[$1","$2","$3]++;
next;
}
{
s1=split($0,AX,"\n");
for (i=1;i<=s1;i++)
{
s2=split(AX[i],BX,",");
for (j=2;j<=NF;j+=2)
{
lineX=BX[j]","BX[j+1];
gsub(/\"/,"",lineX);
for (item in sinonim)
{
s3=split(item,CX,",");
lineS=CX[2]","CX[3];
if (lineX == lineS)
{
BX[j]=CX[1];
lineD=""
for (t=1;t<=s2;t++)
{
lineD=lineD BX[t]",";
}
lineF=lineF lineD"\n";
}
}
}
}
print lineF
}
$ cat tst.awk
BEGIN { FS=";" }
NR==FNR { synonyms[$2,$3][$2]; synonyms[$2,$3][$1]; next }
FNR==1 { FS=OFS="\",\""; $0=$0 }
{
gsub(/^"|"$/,"")
for (i=2;i<NF;i+=2) {
if ( ($i,$(i+1)) in synonyms) {
for (synonym in synonyms[$i,$(i+1)]) {
$i = synonym
for (j=2;j<NF;j+=2) {
if ( ($j,$(j+1)) in synonyms) {
for (synonym in synonyms[$j,$(j+1)]) {
orig = $0
$j = synonym
if (!seen[$0]++) {
print "\"" $0 "\""
}
$0 = orig
}
}
}
}
}
}
}
.
$ awk -f tst.awk fileA fileB
"A001","carro","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","veículo","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","automóvel","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","viatura","sust","excelente","adj","ocorrer","adv","bom","adj"
"A001","carro","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","carro","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","veículo","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","veículo","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","automóvel","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","automóvel","sust","excepcional","adj","ocorrer","adv","bom","adj"
"A001","viatura","sust","maravilhoso","adj","ocorrer","adv","bom","adj"
"A001","viatura","sust","excepcional","adj","ocorrer","adv","bom","adj"
The above uses GNU awk for multi-dimensional arrays, with other awks it's a simple tweak to use synonyms[$2,$3] = synonyms[$2,$3] " " $2 etc. or similar and then split() later instead of synonyms[$2,$3][$2] and in.
BEGIN { FS="[,;]"; OFS="," }
NR == FNR { key = "\"" $2 "\""; synonym[key] = synonym[key] "," $1; next }
{
print;
if ($2 in synonym) {
count = split(substr(synonym[$2], 2), choices)
for (i = 1; i <= count; i++) {
$2 = "\"" choices[i] "\""
print
}
}
}
How do I use the awk range pattern '/begin regex/,/end regex/' within a self-contained awk script?
To clarify, given program csv.awk:
#!/usr/bin/awk -f
BEGIN {
FS = "\""
}
/TREE/,/^$/
{
line="";
for (i=1; i<=NF; i++) {
if (i != 2) line=line $i;
}
split(line, v, ",");
if (v[5] ~ "FOAM") {
print NR, v[5];
}
}
and file chunk:
TREE
10362900,A,INSTL - SEAL,Revise
,10362901,A,ASSY / DETAIL - PANEL,Revise
,,-203,ASSY - PANEL,Qty -,Add
,,,-309,PANEL,Qty 1,Add
,,,,"FABRICATE FROM TEKLAM NE1G1-02-250 PER TPS-CN-500, TYPE A"
,,,-311,PANEL,Qty 1,Add
,,,,"FABRICATE FROM TEKLAM NE1G1-02-750 PER TPS-CN-500, TYPE A"
,,,-313,FOAM SEAL,1.00 X 20.21 X .50 THK,Qty 1,Add
,,,,"BMS1-68, GRADE B, FORM II, COLOR BAC706 (BLACK)"
,,,-315,FOAM SEAL,1.50 X 8.00 X .25 THK,Qty 1,Add
,,,,"BMS1-68, GRADE B, FORM II, COLOR BAC706 (BLACK)"
,PN HERE,Dual Lock,Add
,
10442900,IR,INSTL - SEAL,Update (not released)
,10362901,A,ASSY / DETAIL - PANEL,Revise
,PN HERE,Dual Lock,Add
I want to have this output:
27 FOAM SEAL
29 FOAM SEAL
What is the syntax for adding the command line form '/begin regex/,/end regex/' to the script to operate on those lines only? All my attempts lead to syntax errors and googling only gives me the cli form.
why not use 2 steps:
% awk '/start/,/end/' < input.csv | awk csv.awk
Simply do:
#!/usr/bin/awk -f
BEGIN {
FS = "\""
}
/from/,/to/ {
line="";
for (i=1; i<=NF; i++) {
if (i != 2) line=line $i;
}
split(line, v, ",");
if (v[5] ~ "FOAM") {
print NR, v[5];
}
}
If the from to regexes are dynamic:
#!/usr/bin/awk -f
BEGIN {
FS = "\""
FROM=ARGV[1]
TO=ARGV[2]
if (ARGC == 4) { # the pattern was the only thing, so force read from standard input
ARGV[1] = "-"
} else {
ARGV[1] = ARGV[3]
}
}
{ if ($0 ~ FROM) { p = 1 ; l = 0} }
{ if ($0 ~ TO) { p = 0 ; l = 1} }
{
if (p == 1 || l == 1) {
line="";
for (i=1; i<=NF; i++) {
if (i != 2) line=line $i;
}
split(line, v, ",");
if (v[5] ~ "FOAM") {
print NR, v[5];
}
l = 0 }
}
Now you have to call it like: ./scriptname.awk "FROM_REGEX" "TO_REGEX" INPUTFILE. The last param is optional, if missing STDIN can be used.
HTH
You need to show us what you have tried. Is there something about /begin regex/ or /end regex/ you're not telling us, other wise your script with the additions should work, i.e.
#!/usr/bin/awk -f
BEGIN {
FS = "\""
}
/begin regex/,/end regex/{
line="";
for (i=1; i<=NF; i++) {
if (i != 2) line=line $i;
}
split(line, v, ",");
if (v[5] ~ "FOAM") {
print NR, v[5];
}
}
OR are you using an old Unix, where there is old awk as /usr/bin/awk and New awk as /usr/bin/nawk. Also see if you have /usr/xpg4/bin/awk or gawk (path could be anything).
Finally, show us the error messages you are getting.
I hope this helps.