amazon redshift sql: remove top 1% of data as outliers - sql

Question

i'm trying to remove 1% of data as we consider those outliers that greatly skew data. i've tried using SELECT TOP 99 PERC, but Amazon Redshift does not support percentages with TOP.
I've tried something like:
WITH
elapsed_times AS (
SELECT
COALESCE(anonymous_id, distinct_id) as id,
elapsed_time
FROM studio_production.interaction
WHERE project_id = '55062b464a9bc578006987ff'
),
max_elapsed_time AS (
SELECT elapsed_time
FROM elapsed_times
ORDER BY elapsed_time ASC
OFFSET ROUND(0.99 * (SELECT COUNT(*) FROM elapsed_times))
LIMIT 1
),
user_times AS (
SELECT
id,
LEAST(elapsed_time, max_elapsed_time) as elapsed_time
FROM elapsed_times
GROUP BY 1
)
SELECT AVG(elapsed_time) FROM user_times
But I get: argument of OFFSET must not contain subqueries
thus, my query is now:
WITH
elapsed_times AS (
SELECT
COALESCE(anonymous_id, distinct_id) as id,
elapsed_time,
RANK() OVER (ORDER BY elapsed_time ASC) as rnk
FROM studio_production.interaction
WHERE project_id = '55062b464a9bc578006987ff'
),
user_times AS (
SELECT
id,
LEAST(MAX(elapsed_time), (
SELECT MIN(elapsed_time)
FROM elapsed_times
WHERE rnk > ROUND(0.99 * (SELECT COUNT(*) FROM elapsed_times))
)) as elapsed_time
FROM elapsed_times
GROUP BY 1
)
SELECT AVG(elapsed_time) FROM user_times
which is actually quite slow. what is the correct approach to this problem?

Answer 1

You could use ntile() (see here):
select avg(elapsed_time)
from (select et.*,
ntile(100) over (order by elapsed_time) as thetile
from elapsed_times et
) et
where thetile not in (1, 100);
EDIT:
I admit that I often do this using row_number() and count():
select avg(elapsed_time)
from (select et.*,
row_number() over (order by elapsed_time) as seqnum,
count(*) over () as cnt
from elapsed_times et
) et
where (seqnum <= 0.01 * cnt) or (seqnum >= 0.99 * cnt);