SQL query with row number and data cleaning there is no validation - sql

Question

I have "prices" table with this data:
id
start_date
end_date
paid
787
1/1/2022
1/15/2022
100
787
1/15/2022
1/22/2022
100,000
787
1/22/2022
2/25/2022
115,000
787
2/25/2022
2/28/2022
14,000
787
2/28/2022
3/05/2022
122,000
787
3/05/2022
3/09/2022
120
423
1/25/2022
2/25/2022
1
423
2/25/2022
2/28/2022
7,500,000
423
3/19/2022
3/22/2022
7,200,000
423
3/25/2022
3/29/2022
1,111,111,111
423
4/13/2022
4/26/2022
999,999,999,999
423
4/26/2022
4/28/2022
999,999,999,99
423
4/28/2022
5/09/2022
7,100,000
423
5/09/2022
5/22/2022
666,666,666,666
I want to stay just with one raw per id with start date ASC and end_date DESC
but the most important is if I have a small number or bigger then others like the price is not between 30% or -30% from the following price take me the last/after price and then its will be the "real" price.
my output:
id
start_date
end_date
first_paid
last_paid
787
1/1/2022
3/09/2022
100,000
122,000
423
1/25/2022
5/22/2022
7,500,000
7,100,000
I created this query but it's with all numbers including the "fake" numbers and I want to clean the "fake" numbers:
select a.id, b.start_date, a.end_date, b.paid as first_paid, a.price as last_paid
from(
select id, start_date, end_date, paid
from (
select *,
row_number() over(partition by id order by end_date desc) rn
from "prices"
)
where rn = 1) as a
left join
(select id, start_date, end_date, paid
from(
select id, start_date, end_date, paid
from (
select *,
row_number() over(partition by id order by start_date) rn
from "prices"
)
where rn = 1)) as b
on a.id = b.id
if someone has a good idea or solution and it will be in presto syntax it will be amazing.
thanks all.

Answer 1

I do hope your explanation was received correctly..
see: DBFIDDLE with some explanations on what is done to get this:
SELECT
x.id,
x.start_date,
x.end_date,
min(x.first_paid) as first_paid,
max(x.last_paid) as last_paid
FROM (
SELECT DISTINCT
id,
min(start_date) over (partition by id) as start_date,
max(end_date) over (partition by id) as end_date,
first_value(paid) over (partition by id order by start_date) as first_paid,
last_value(paid) over (partition by id order by end_date) as last_paid
FROM (
SELECT
id,
start_date,
end_date,
paid as old_paid,
lag(paid) over (partition by id order by start_date) as previus_paid,
CASE WHEN abs(((lag(paid) over (partition by id order by start_date))-paid)
/(lag(paid) over (partition by id order by start_date))) > 30
THEN lag(paid) over (partition by id order by start_date)
ELSE paid END as paid
FROM mytable2 m2
) m1) x
GROUP BY x.id,x.start_Date,x.end_date
;
output:
id
start_date
end_date
first_paid
last_paid
423
2022-01-25
2022-05-22
1.00
99999999999.00
787
2022-01-01
2022-03-09
100.00
122000.00
I now see that I must a made a type somewhere, because 100 seems not be correct...
But I will leave that for you to find out where I wen wrong.....
EDIT: You will have to find a smart way to filter the low and high values that you do not want to include.
Something like:
select * from (
SELECT
id,
paid,
avg(paid) over (partition by id) as avg1,
abs((COALESCE(lead(paid) over (partition by id order by paid),min(paid) over (partition by id)) +
COALESCE(lag(paid) over (partition by id order by paid),max(paid) over (partition by id)) +
paid)/3 - paid ) as avg2
FROM mytable2 m2
) x where x.avg2 < avg1
order by x.id,x.paid
output:
id
paid
avg1
avg2
423
7100000.00
220974947222.000000
2333333.000000
423
7200000.00
220974947222.000000
66666.666666
423
7500000.00
220974947222.000000
367770370.333333
423
1111111111.00
220974947222.000000
32595092592.333333
423
99999999999.00
220974947222.000000
155925925926.333333
423
666666666666.00
220974947222.000000
77777777778.000000
787
100.00
58536.666666
40640.000000
787
120.00
58536.666666
4620.000000
787
14000.00
58536.666666
24040.000000
787
100000.00
58536.666666
23666.666667
787
115000.00
58536.666666
2666.666667
787
122000.00
58536.666666
42966.666667
This filters the 1 and the 999,999,999,999 ....