select Not null values from mutiple columns in pyspark - dataframe

Question

Below is my dataframe
+---------+-----+--------+---------+-------+
| NAME|Actor| Doctor|Professor|Singer |
+---------+-----+--------+---------+-------+
| Samantha| null|Samantha| null| null|
|Christeen| null| null|Christeen| null|
| Meera| null| null| null| Meera|
| Julia|Julia| null| null| null|
| Priya| null| null| null| Priya|
| Ashley| null| null| Ashley| null|
| Jenny| null| Jenny| null| null|
| Maria|Maria| null| null| null|
| Jane| Jane| null| null| null|
| Ketty| null| null| Ketty| null|
+---------+-----+--------+---------+-------+
I want to select all not null values from ACTOR,DOCTOR,PROFESSOR AND SINGER

Answer 1

This can be achieved via isNotNull and creating a condn of your desired rules and finally filter -
You can modify the condn depending on your requirement further -
Preparing Data
input_str = """
Samantha|None|Samantha| None| None|
Christeen| None| None|Christeen| None|
Meera| None| None| None| Meera|
Julia|Julia| None| None| None|
Priya| None| None| None| Priya|
Ashley| None| None| Ashley| None|
Jenny| None| Jenny| None| None|
Maria|Maria| None| None| None|
Jane| Jane| None| None| None|
Ketty| None| None| Ketty| None|
Aditya| None| None| None| None|
""".split("|")
input_values = list(map(lambda x:x.strip(),input_str))[:-1]
i = 0
n = len(input_values)
input_list = []
while i < n:
input_list += [ tuple(input_values[i:i+5]) ]
i += 5
input_list
[('Samantha', 'None', 'Samantha', 'None', 'None'),
('Christeen', 'None', 'None', 'Christeen', 'None'),
('Meera', 'None', 'None', 'None', 'Meera'),
('Julia', 'Julia', 'None', 'None', 'None'),
('Priya', 'None', 'None', 'None', 'Priya'),
('Ashley', 'None', 'None', 'Ashley', 'None'),
('Jenny', 'None', 'Jenny', 'None', 'None'),
('Maria', 'Maria', 'None', 'None', 'None'),
('Jane', 'Jane', 'None', 'None', 'None'),
('Ketty', 'None', 'None', 'Ketty', 'None'),
('Aditya', 'Aditya', 'Aditya', 'Aditya', 'Aditya')]
Converting None to Null
def blank_as_null(x):
return F.when(F.col(x) == "None", None).otherwise(F.col(x))
sparkDF = sql.createDataFrame(input_list,['Name','Actor','Doctor','Professor','Singer'])
to_convert = set(['Actor','Doctor','Professor','Singer'])
sparkDF = reduce(lambda df, x: df.withColumn(x, blank_as_null(x)), to_convert, sparkDF)
sparkDF.show()
+---------+------+--------+---------+------+
| Name| Actor| Doctor|Professor|Singer|
+---------+------+--------+---------+------+
| Samantha| null|Samantha| null| null|
|Christeen| null| null|Christeen| null|
| Meera| null| null| null| Meera|
| Julia| Julia| null| null| null|
| Priya| null| null| null| Priya|
| Ashley| null| null| Ashley| null|
| Jenny| null| Jenny| null| null|
| Maria| Maria| null| null| null|
| Jane| Jane| null| null| null|
| Ketty| null| null| Ketty| null|
| Aditya|Aditya| Aditya| Aditya|Aditya|
+---------+------+--------+---------+------+
Filtering Not Null Values
condn = (
(F.col('Actor').isNotNull())
& (F.col('Doctor').isNotNull())
& (F.col('Professor').isNotNull())
& (F.col('Singer').isNotNull())
)
sparkDF.filter(condn).show()
+------+------+------+---------+------+
| Name| Actor|Doctor|Professor|Singer|
+------+------+------+---------+------+
|Aditya|Aditya|Aditya| Aditya|Aditya|
+------+------+------+---------+------+