Hive sql query optimisation - sql

Question

How to optimise query?
df = hive.sql("""
WITH cte AS (
SELECT
MAX(timestamp) as event_time,
name, domain, type, event_id
FROM table1
WHERE
device = '{device}'
AND name IS NOT NULL
AND (timestamp >= (CAST(NOW() as INT) - {delta}*24*60*60))
AND name NOT IN (
SELECT
DISTINCT(name)
FROM table1
WHERE (
device = '{device}' AND
name IS NOT NULL AND org ='{org}' AND
((timestamp < CAST(NOW() as INT) - {delta}*24*60*60) AND
(timestamp >= CAST(NOW() as INT) - (120+{delta})*24*60*60)))
)
AND event_id NOT IN ('4624', '4656')
AND LOWER(status) NOT IN ('failure')
GROUP BY
name, domain, type, event_id
)
SELECT
from_unixtime(timestamp, 'dd-MMM-yy HH:mm:ss') as timestamp,
name, domain, type, event_id
FROM cte
WHERE
event_time IN (
SELECT
MAX(event_time)
FROM cte
GROUP BY
name
)
""".format(device=device, device=device))
result = df.toPandas()
...
I'am trying to get all latest events with some parameters in table, which were not in last 120 days before delta.
My spark config:
SparkConf()).config("spark.sql.execution.arrow.pyspark.enabled", "true").("spark.driver.memory", "16G").config("spark.driver.maxResultSize", "8G").config("spark.sql.broadcastTimeout", 36000)
Here is an example of my table.
timestamp
type
aux1
aux2
event_id
dst.fqdn
dst.host
src.cat
src.host
src.hostname
src.ip
subsystem
title
vendor
action
severity
status
name
type
domain
device
org
1676619776
norm
0x80
%%4423
4776
do.uwc.com
uwc-nb-121
OperatingSystem
192.168.40.100
uwc-infra-fs1
192.168.40.100
Security
Windows
Microsoft
access
LOW
success
ifernin
account
uwc
mswin
uwc
1676637549
norm
0x100081
%%1541%%4423
5145
do.uwc.com
uwc-pc-022
OperatingSystem
192.168.40.100
uwc-infra-fs1
192.168.40.100
Security
Windows
Microsoft
access
LOW
success
kiersk
account
uwc
mswin
uwc
1676632328
norm
0x3e7
0xa18
4658
do.uwc.com
uwc-nb-144
OperatingSystem
192.168.40.100
uwc-infra-fs1
192.168.40.100
Security
Windows
Microsoft
close
LOW
success
aseen
account
uwc
mswin
uwc
1676620697
norm
0x100081
%%1541%%4416
5145
do.uwc.com
uwc-nb-124
OperatingSystem
192.168.40.100
uwc-infra-fs1
192.168.40.100
Security
Windows
Microsoft
access
LOW
success
aseen
account
uwc
mswin
uwc
1676641029
norm
0x100081
%%1541%%4416
5145
do.uwc.com
10.77.11.30
OperatingSystem
192.168.40.100
uwc-infra-fs1
192.168.40.100
Security
Windows
Microsoft
access
LOW
failure
aseen
account
uwc
mswin
uwc
1676540338
norm
0x80
%%4423
5145
eff2.uniwc.com
uwc-infra-fs1
OperatingSystem
192.168.40.100
uwc-infra-fs1
192.168.40.100
Security
Windows
Microsoft
access
LOW
success
pgolubeva
account
uwc
mswin
uwc
1676632808
norm
4776
eff2.uniwc.com
uwc-infra-dc2
OperatingSystem
192.168.40.10
uwc-infra-dc2
192.168.40.10
Security
Windows
Microsoft
login
HIGH
ongoing
vdolgov
account
mswin
uwc
1676540338
norm
S-1-0-0
4624
eff2.uniwc.com
192.168.40.12
OperatingSystem
192.168.40.10
uwc-infra-dc2
192.168.40.10
Security
Windows
Microsoft
login
LOW
failure
popovm
account
uwc.com
mswin
uwc
1676673260
norm
0x100081
%%1541%%4416%%4423
5145
eff2.uniwc.com
10.77.11.51
OperatingSystem
192.168.40.10
uwc-infra-fs1
192.168.40.10
Security
Windows
Microsoft
access
LOW
ongoing
evteeva
account
twqp.uwc.com
mswin
uwc
1676540338
norm
4776
eff2.uniwc.com
uwc-nb-101
OperatingSystem
192.168.40.10
uwc-infra-dc2
192.168.40.10
Security
Windows
Microsoft
login
HIGH
success
monitor
account
mswin
uwc

Answer 1

the last select statement can be re-written as window function to avoid possibly costly, see the example query below:
WITH cte AS (
SELECT
MAX(timestamp) as event_time,
name, domain, type, event_id
FROM table1
WHERE
device = '{device}'
AND name IS NOT NULL
AND (timestamp >= (CAST(NOW() as INT) - {delta}*24*60*60))
AND name NOT IN (
SELECT
DISTINCT(name)
FROM table1
WHERE (
device = '{device}' AND
name IS NOT NULL AND org ='{org}' AND
((timestamp < CAST(NOW() as INT) - {delta}*24*60*60) AND
(timestamp >= CAST(NOW() as INT) - (120+{delta})*24*60*60)))
)
AND event_id NOT IN ('4624', '4656')
AND LOWER(status) NOT IN ('failure')
GROUP BY
name, domain, type, event_id
)
SELECT
from_unixtime(timestamp, 'dd-MMM-yy HH:mm:ss') as timestamp,
name,
domain,
type,
event_id
FROM
(SELECT
timestamp,
name, domain, type, event_id
rank() over (partition by name order by event_time desc) as event_time_rank
FROM cte
) t
WHERE
event_time_rank = 1