SQL Table Join with Weighting - sql

Question

I'm trying to create a table that will calculate the weights of spend for a customer in the months they shop for a period. For example, the following customer (faux data) has this spend profile:
/* Customer spend */
create or replace temp table ts_all_transactions
(
inferred_customer_id varchar(128)
,nw_date date
,spend number(21,2)
);
insert into ts_all_transactions
values
('52f5','2019-06-01',17.35)
,('52f5','2018-11-01',24.85)
,('52f5','2019-12-01',1.40)
,('52f5','2019-01-01',2.45)
,('52f5','2019-03-01',3.90)
,('52f5','2020-01-01',37.55)
,('52f5','2019-10-01',13.20)
,('52f5','2019-09-01',5.70)
;
A calendar containing the months in which a period falls in, along with a weighting is then created:
-- Calculate weights for each period of the time series
-- Create a staging table
create or replace temp table period_dimension as
select abs(seq4()-12) as period,
dateadd(month, seq4(), dateadd(month, -23, date_trunc('Month', current_date()))) as start_date,
dateadd(month, 12, start_date) as end_date
from table(generator(rowcount => 12)) -- number of months after reference date in previous line
;
select * from period_dimension;
create or replace temp table my_date_dimension
(
my_date date not null
,year smallint not null
,month smallint not null
,month_name char(3) not null
,day_of_mon smallint not null
,day_of_week varchar(9) not null
,week_of_year smallint not null
,day_of_year smallint not null
)
as
with my_date as (
select
seq4(),
dateadd(month, seq4(), dateadd(month, -23, date_trunc('Month', current_date()))) as my_date
from table(generator(rowcount=>23))
)
select my_date
,year(my_date)
,month(my_date)
,monthname(my_date)
,day(my_date)
,dayofweek(my_date)
,weekofyear(my_date)
,dayofyear(my_date)
from my_date
;
create or replace table weight_lookup as
select
a.period
,b.my_date
,rank() over (partition by a.period order by b.my_date) as weight
from period_dimension a
inner join my_date_dimension b
where b.my_date >= a.start_date
and b.my_date < a.end_date
order by 1,2
;
-- Create a staging table
create or replace temp table period_dimension2 as
select abs(seq4()-12) as period,
dateadd(month, seq4(), dateadd(month, -23, date_trunc('Month', current_date()))) as start_date,
last_day(dateadd(month, 11, start_date)) as end_date
from table(generator(rowcount => 12)) -- number of months after reference date in previous line
;
The above is then used to calculate an average spend based on the months the customer shops in the period, however, I'm not getting the output I expect:
-- For each month of each period, group all together by period here so we have 12 periods
-- so each period represents 12 rolling months with period 12 being the oldest period
create or replace temp table ts_spend_time as
select
a.inferred_customer_id
,b.period
,max(a.nw_date) as max_mnth /* Month in period where most spend was made */
,sum(a.spend * b.weight) / 78 as avg_spend /* Sum of weights 12,11,10...1 to give 78 */
from ts_all_transactions a
inner join weight_lookup b on a.nw_date = b.my_date
inner join period_dimension2 c on b.my_date = c.start_date and b.period = c.period
where b.my_date >= c.start_date
and b.my_date <= c.end_date
group by 1,2
order by 1 desc, 2,3
;
The output I get from the above code is this:
create or replace temp table ts_spend_time_wrong_out
(
inferred_customer_id varchar(128)
,period number(11)
,max_mnth date
,avg_spend number(38,8)
);
insert into ts_spend_time_wrong_out
values
('52f5',3,'2019-03-01',0.05000000)
,('52f5',5,'2019-01-01',0.03141026)
,('52f5',7,'2018-11-01',0.31858974)
;
I would like to get an output like this:
create or replace temp table ts_spend_time_should_be
(
inferred_customer_id varchar(128)
,period number(11)
,max_mnth date
,avg_spend number(38,8)
);
insert into ts_spend_time_should_be
values
('52f5',1,'01JAN2020',6.301923077)
,('52f5',2,'01JAN2020',7.266025641)
,('52f5',3,'01JAN2020',8.280128205)
,('52f5',4,'01JAN2020',9.294230769)
,('52f5',5,'01DEC2019',4.081410256)
,('52f5',6,'01OCT2019',4.412179487)
,('52f5',7,'01OCT2019',5.276923077)
,('52f5',8,'01SEP2019',3.941666667)
,('52f5',9,'01JUN2019',3.687179487)
,('52f5',10,'01JUN2019',4.309615385)
,('52f5',11,'01JUN2019',4.932051282)
,('52f5',12,'01MAR2019',2.662820513)
;
In the correct solution example, the average spend is calculated by period as follows: ((17.35*2)+(5.7*5)+(13.20*6)+(1.4*8)+(37.55*9)) / 78
How can I resolve this? TIA

Answer 1

firstly you should use row_number() over(order by seq4()) as there can be gaps in a seq()
so working half way through you question
with ts_all_transactions as (
select id, nw_date::date as nw_date, spend from values
('52f5','2019-06-01',17.35)
,('52f5','2018-11-01',24.85)
,('52f5','2019-12-01',1.40)
,('52f5','2019-01-01',2.45)
,('52f5','2019-03-01',3.90)
,('52f5','2020-01-01',37.55)
,('52f5','2019-10-01',13.20)
,('52f5','2019-09-01',5.70)
v(id,nw_date, spend)
), period_dimension as (
select
row_number() over(order by seq4())-1 as rn0,
abs(rn0-12) as period,
dateadd('month', rn0, dateadd(month, -23, date_trunc('Month', current_date()))) as start_date,
dateadd('month', 12, start_date) as end_date
from table(generator(rowcount => 12)) -- number of months after reference date in previous line
), weight_periods as (
select p.period
,p.start_date
,p.end_date
,row_number() over(partition by p.period order by seq4())-1 as rn1
,dateadd('month',-rn1, p.end_date ) as weight_month
,12 - rn1 + 1 as weight
from period_dimension p,
table(generator(rowcount => 12))
), monthly_spends as (
select id
,date_trunc('month', nw_date) as m_date
,sum(spend) as s_spend
from ts_all_transactions
group by 1,2
)
select m.id
,w.period
,w.end_date
,w.weight_month
,m.s_spend
,w.weight
,m.s_spend * w.weight as w_spend
from monthly_spends m
join weight_periods w on m.m_date = w.weight_month
order by 1,2,3,4;
gives:
ID PERIOD END_DATE WEIGHT_MONTH S_SPEND WEIGHT W_SPEND
52f5 1 2020-05-01 2019-06-01 17.35 2 34.70
52f5 1 2020-05-01 2019-09-01 5.70 5 28.50
52f5 1 2020-05-01 2019-10-01 13.20 6 79.20
52f5 1 2020-05-01 2019-12-01 1.40 8 11.20
52f5 1 2020-05-01 2020-01-01 37.55 9 337.95
52f5 2 2020-04-01 2019-06-01 17.35 3 52.05
...
This shows up to this point we can see the inputs to the values you are expecting for the "weighted average" which can be done via :
select m.id
,w.period
,sum(m.s_spend * w.weight) as t_w_spend
,round(t_w_spend / 78,3) as weighted_avg_spend
from monthly_spends m
join weight_periods w on m.m_date = w.weight_month
group by 1,2
order by 1,2;
which gives:
ID PERIOD T_W_SPEND WEIGHTED_AVG_SPEND
52f5 1 491.55 6.302
52f5 2 566.75 7.266
52f5 3 641.95 8.230
52f5 4 724.95 9.294
52f5 5 804.05 10.308
52f5 6 362.35 4.646
52f5 7 386.75 4.958
52f5 8 479.05 6.142
52f5 9 361.70 4.637
52f5 10 336.15 4.310
52f5 11 384.70 4.932
52f5 12 433.25 5.554
which starts the same, but diverge as I think your date periods are done "wrong"
but the next point is you have this line
,max(a.nw_date) as max_mnth /* Month in period where most spend was made */
which does not do what you comment it as doing..
what it does is find the max date value in the aggregate.
to do that you need to go back to the monthly results sql and put a first_value() into the mix then select the results via:
id, period, max_spend_month, sum(w_spend)/78 as weighted_avg_spend
from (
select m.id
,w.period
,w.end_date
,w.weight_month
,m.s_spend
,w.weight
,m.s_spend * w.weight as w_spend
,first_value(w.weight_month) over (partition by m.id, w.period order by m.s_spend desc) as max_spend_month
from monthly_spends m
join weight_periods w on m.m_date = w.weight_month
)
group by 1,2,3
order by 1,2;
which now matches your expectations:
ID PERIOD MAX_SPEND_MONTH WEIGHTED_AVG_SPEND
52f5 1 2020-01-01 6.30192308
52f5 2 2020-01-01 7.26602564
52f5 3 2020-01-01 8.23012821
52f5 4 2020-01-01 9.29423077
52f5 5 2020-01-01 10.30833333
52f5 6 2019-06-01 4.64551282
52f5 7 2019-06-01 4.95833333
52f5 8 2018-11-01 6.14166667
52f5 9 2018-11-01 4.63717949
52f5 10 2018-11-01 4.30961538
52f5 11 2018-11-01 4.93205128
52f5 12 2018-11-01 5.55448718