Nested select and joins - sql

Question

I use the code below to calculate the required information from other tables. I used joins to display Names instead of IDs and to get required sums from other tables. I used COALESCE to convert null to Zero.
I had to used it again if I need to sum already COALESCED values, the above code is hard to understand and it's getting harder because I need to add more information, and this is just a small part of the main project so it will be really hard to work with it and will have many error and bugs.
Does it have to be so complicated? Or did I do it wrong? If it has to be complicated like this is there any replacement to get same results with easier way and code? Another RDBMS or anything else?
SELECT P.PROJ_ID, P.PROJ_STATUS, P.TYPE_ID, PT.TYPE_NAME, P.CLASS_ID, CLA.CLASS_NAME, P.PROJ_NO, P.PROJ_YEAR, P.PROJ_NAME, P.OLD_PROJ_NAME,
P.AGENCY_ID, A.AGENCY_NAME, P.CONTRACT_NO, P.CONTRACT_DATE, P.MINISTRY_ID, M.MINISTRY_NAME,
P.DIRECTORATE_ID, DIR.DIRECTORATE_NAME,
P.COST, P.ESTIMATED_COST, COALESCE(CO.ADDED_COSTS, 0) AS ADDED_COSTS, (COALESCE(P.COST, 0) + COALESCE(CO.ADDED_COSTS, 0)) AS TOTAL_COST,
P.ALLOCATION,
COALESCE(EPY.PAST_YEARS, 0) AS PAST_YEARS,
COALESCE(EF.PAST_MONTHS, 0) AS PAST_MONTHS,
COALESCE(ECM.CURRENT_MONTH, 0) AS CURRENT_MONTH,
COALESCE(ECY.CURRENT_YEAR, 0) AS CURRENT_YEAR,
COALESCE(E.TOTAL_EXPENSES, 0) AS TOTAL_EXPENSES,
COALESCE(CASH_EPY.CASH_PAST_YEARS, 0) AS CASH_PAST_YEARS,
COALESCE(CASH_EF.CASH_PAST_MONTHS, 0) AS CASH_PAST_MONTHS,
COALESCE(CASH_ECM.CASH_CURRENT_MONTH, 0) AS CASH_CURRENT_MONTH,
COALESCE(CASH_ECY.CASH_CURRENT_YEAR, 0) AS CASH_CURRENT_YEAR,
COALESCE(CASH_E.CASH_TOTAL_EXPENSES, 0) AS CASH_TOTAL_EXPENSES,
COALESCE(TOTAL_E.TOTAL_EXPENSES_CASH, 0) AS TOTAL_EXPENSES_CASH,
((COALESCE(P.COST, 0) + COALESCE(CO.ADDED_COSTS, 0)) - COALESCE(E.TOTAL_EXPENSES, 0)) AS REMAINING,
P.DURATION, COALESCE(DU.ADDED_DURATIONS, 0) AS ADDED_DURATIONS,
(COALESCE(P.DURATION, 0) + COALESCE(DU.ADDED_DURATIONS, 0)) AS TOTAL_DURATION, P.START_DATE, P.FINISH_DATE,
P.GOVERNORATE_ID, G.GOVERNORATE_NAME, P.PROVINCE_ID, PR.PROVINCE_NAME, P.DISTRICT_ID, D.DISTRICT_NAME,
P.TOWN_ID, T.TOWN_NAME,
COALESCE( (E.TOTAL_EXPENSES / (COALESCE(P.COST, 0) + COALESCE(CO.ADDED_COSTS, 0)))/100, 0) AS FINANCIAL_ACHIEVEMENT,
P.MATERIAL_ACHIEVEMENT, P.NOTES
FROM PROJECTS P
INNER JOIN PROJECTS_TYPES PT
ON P.TYPE_ID = PT.TYPE_ID
INNER JOIN CLASSES CLA
ON P.CLASS_ID = CLA.CLASS_ID
INNER JOIN AGENCIES A
ON P.AGENCY_ID = A.AGENCY_ID
LEFT JOIN MINISTRIES M
ON P.MINISTRY_ID = M.MINISTRY_ID
LEFT JOIN DIRECTORATES DIR
ON P.DIRECTORATE_ID = DIR.DIRECTORATE_ID
INNER JOIN GOVERNORATES G
ON P.GOVERNORATE_ID = G.GOVERNORATE_ID
LEFT JOIN PROVINCES PR
ON P.PROVINCE_ID = PR.PROVINCE_ID
LEFT JOIN DISTRICTS D
ON P.DISTRICT_ID = D.DISTRICT_ID
LEFT JOIN TOWNS T
ON P.TOWN_ID = T.TOWN_ID
-- ADDED COSTS
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(ADDED_VALUE, 0) - COALESCE(REMOVED_VALUE, 0)) as ADDED_COSTS
FROM COSTS
GROUP BY PROJ_ID ) AS CO
ON P.PROJ_ID = CO.PROJ_ID
-- EXPENSES FROM PAST YEARS
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as PAST_YEARS
FROM EXPENSES WHERE EXTRACT(YEAR FROM DOC_DATE) < EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
GROUP BY PROJ_ID ) AS EPY
ON P.PROJ_ID= EPY.PROJ_ID
-- EXPENSES FROM PAST MONTHS IN CUREENT YEAR
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as PAST_MONTHS
FROM EXPENSES WHERE EXTRACT(MONTH FROM DOC_DATE) < EXTRACT(MONTH FROM CURRENT_DATE)
AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
GROUP BY PROJ_ID ) AS EF
ON P.PROJ_ID= EF.PROJ_ID
-- EXPENSES FROM CURRENT MONTH AND YEAR
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CURRENT_MONTH
FROM EXPENSES WHERE EXTRACT(MONTH FROM DOC_DATE) = EXTRACT(MONTH FROM CURRENT_DATE) AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
GROUP BY PROJ_ID ) AS ECM
ON P.PROJ_ID= ECM.PROJ_ID
-- SUM OF EXPENSES IN CURRENT YEAR
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CURRENT_YEAR
FROM EXPENSES WHERE EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
GROUP BY PROJ_ID ) AS ECY
ON P.PROJ_ID= ECY.PROJ_ID
-- TOTAL EXPENSES FROM ALL TIME
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as TOTAL_EXPENSES
FROM EXPENSES WHERE CASH_DEDUCTIONS = FALSE
GROUP BY PROJ_ID ) AS E
ON P.PROJ_ID= E.PROJ_ID
-- CASH DEDUCTIONS SUMS
-- CASH DEDUCTIONS FROM PAST YEARS
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CASH_PAST_YEARS
FROM EXPENSES WHERE EXTRACT(YEAR FROM DOC_DATE) < EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
GROUP BY PROJ_ID ) AS CASH_EPY
ON P.PROJ_ID= CASH_EPY.PROJ_ID
-- CASH DEDUCTIONS FROM PAST MONTHS IN CUREENT YEAR
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CASH_PAST_MONTHS
FROM EXPENSES WHERE EXTRACT(MONTH FROM DOC_DATE) < EXTRACT(MONTH FROM CURRENT_DATE)
AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
GROUP BY PROJ_ID ) AS CASH_EF
ON P.PROJ_ID= CASH_EF.PROJ_ID
-- CASH DEDUCTIONS FROM CURRENT MONTH AND YEAR
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CASH_CURRENT_MONTH
FROM EXPENSES WHERE EXTRACT(MONTH FROM DOC_DATE) = EXTRACT(MONTH FROM CURRENT_DATE) AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
GROUP BY PROJ_ID ) AS CASH_ECM
ON P.PROJ_ID= CASH_ECM.PROJ_ID
-- SUM OF CASH DEDUCTIONS IN CURRENT YEAR
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CASH_CURRENT_YEAR
FROM EXPENSES WHERE EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
GROUP BY PROJ_ID ) AS CASH_ECY
ON P.PROJ_ID= CASH_ECY.PROJ_ID
-- TOTAL CASH DEDUCTIONS FROM ALL TIME
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CASH_TOTAL_EXPENSES
FROM EXPENSES WHERE CASH_DEDUCTIONS = TRUE
GROUP BY PROJ_ID ) AS CASH_E
ON P.PROJ_ID= CASH_E.PROJ_ID
-- TOTAL EXPENSES AND CASH DEDUCTIONS FROM ALL TIME
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as TOTAL_EXPENSES_CASH
FROM EXPENSES
GROUP BY PROJ_ID ) AS TOTAL_E
ON P.PROJ_ID= TOTAL_E.PROJ_ID
-- ADDED DURATIONS
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(ADDED_VALUE, 0) - COALESCE(REMOVED_VALUE, 0)) as ADDED_DURATIONS
FROM DURATIONS
GROUP BY PROJ_ID ) AS DU
ON P.PROJ_ID= DU.PROJ_ID
ORDER BY P.PROJ_YEAR, P.TYPE_ID, P.PROJ_NO

Answer 1

I think it would be good to start with getting down to just one subquery on the Expenses table. Looks like your coalesces in the sub-queries are just to replace nulls with 0 on 1 column - you could just run an UPDATE to fix that, but if not, I've included a way to do that just once in the example below.
You've got (for example):
-- TOTAL CASH DEDUCTIONS FROM ALL TIME
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as CASH_TOTAL_EXPENSES
FROM EXPENSES WHERE CASH_DEDUCTIONS = TRUE
GROUP BY PROJ_ID ) AS CASH_E
ON P.PROJ_ID= CASH_E.PROJ_ID
-- TOTAL EXPENSES AND CASH DEDUCTIONS FROM ALL TIME
LEFT JOIN (SELECT PROJ_ID, SUM(COALESCE(TOTAL_VALUE, 0)) as TOTAL_EXPENSES_CASH
FROM EXPENSES
GROUP BY PROJ_ID ) AS TOTAL_E
ON P.PROJ_ID= TOTAL_E.PROJ_ID
You could do something like this instead:
with MY_EXPENSES AS
(
select COALESCE(TOTAL_VALUE, 0) as MY_TOTAL_VALUE, exp.*
from EXPENSES),
EXPENSES_SUMMARY AS
(
SELECT PROJ_ID,
sum(case when CASH_DEDUCTIONS = TRUE
then MY_TOTAL_VALUE else 0
end) as CASH_TOTAL_EXPENSES,
sum(MY_TOTAL_VALUE) as TOTAL_EXPENSES_CASH
FROM MY_EXPENSES
GROUP BY PROJ_ID )
<Big Query>
LEFT OUTER JOIN EXPENSES_SUMMARY es P.PROJ_ID= TOTAL_E.PROJ_ID
Basically take the conditions out of the where clauses in the sub-query and put them in a case statement within a sum - then you can have all 15 or however many it is items from that EXPENSES table in sub-query item. Only have to coalesce the TOTAL_VALUE once instead of in every sub-query.
You could also make a view that does the summary data.
Are there any proj_id's that don't exist at all in EXPENSES, DURATION, or COSTS? If not, then once you make the above change, you shouldn't have to do any COALESCE's in the main query. If so, then you probably still need to replace a few nulls with 0's - could do that client side if you just want to get the coalesce out of the query.
In general, I find the 'with . . . select' more readable for complex queries, but that may just be a preference thing.

Answer 2

This is the last code faster and cleaner, thanks to aduguid and dandarc
WITH
EXPENSES_SUMS
AS
(
SELECT
PROJ_ID
, SUM(CASE
WHEN EXTRACT(YEAR FROM DOC_DATE) < EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
THEN TOTAL_VALUE
END) AS PAST_YEARS
, SUM(CASE
WHEN EXTRACT(MONTH FROM DOC_DATE) < EXTRACT(MONTH FROM CURRENT_DATE) AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
THEN TOTAL_VALUE
END) AS PAST_MONTHS
, SUM(CASE
WHEN EXTRACT(MONTH FROM DOC_DATE) = EXTRACT(MONTH FROM CURRENT_DATE) AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
THEN TOTAL_VALUE
END) AS CURRENT_MONTH
, SUM(CASE
WHEN EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = FALSE
THEN TOTAL_VALUE
END) AS CURRENT_YEAR
, SUM(CASE
WHEN CASH_DEDUCTIONS = FALSE
THEN TOTAL_VALUE
END) AS TOTAL_EXPENSES
, SUM(CASE
WHEN EXTRACT(YEAR FROM DOC_DATE) < EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
THEN TOTAL_VALUE
END) AS CASH_PAST_YEARS
, SUM(CASE
WHEN EXTRACT(MONTH FROM DOC_DATE) < EXTRACT(MONTH FROM CURRENT_DATE) AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
THEN TOTAL_VALUE
END) AS CASH_PAST_MONTHS
, SUM(CASE
WHEN EXTRACT(MONTH FROM DOC_DATE) = EXTRACT(MONTH FROM CURRENT_DATE) AND EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
THEN TOTAL_VALUE
END) AS CASH_CURRENT_MONTH
, SUM(CASE
WHEN EXTRACT(YEAR FROM DOC_DATE) = EXTRACT(YEAR FROM CURRENT_DATE) AND CASH_DEDUCTIONS = TRUE
THEN TOTAL_VALUE
END) AS CASH_CURRENT_YEAR
, SUM(CASE
WHEN CASH_DEDUCTIONS = TRUE
THEN TOTAL_VALUE
END) AS CASH_TOTAL_EXPENSES
, SUM(TOTAL_VALUE) AS TOTAL_EXPENSES_CASH
--you can add more columns with your conditions here in case expressions
FROM
EXPENSES
GROUP BY
PROJ_ID
)
,
COSTS_SUMS
AS
(
SELECT
PROJ_ID
, SUM(COALESCE(ADDED_VALUE, 0) - COALESCE(REMOVED_VALUE, 0)) AS ADDED_COSTS
--you can add more columns with your conditions here in case expressions
FROM
COSTS
GROUP BY
PROJ_ID
)
,
DURATIONS_SUMS
AS
(
SELECT
PROJ_ID
, SUM(COALESCE(ADDED_VALUE, 0) - COALESCE(REMOVED_VALUE, 0)) AS ADDED_DURATIONS
--you can add more columns with your conditions here in case expressions
FROM
DURATIONS
GROUP BY
PROJ_ID
)
SELECT
P.PROJ_ID
, P.PROJ_STATUS
, P.TYPE_ID
, PT.TYPE_NAME
, P.CLASS_ID
, CLA.CLASS_NAME
, P.PROJ_NO
, P.PROJ_YEAR
, P.PROJ_NAME
, P.OLD_PROJ_NAME
, P.AGENCY_ID
, A.AGENCY_NAME
, P.CONTRACT_NO
, P.CONTRACT_DATE
, P.MINISTRY_ID
, M.MINISTRY_NAME
, P.DIRECTORATE_ID
, DIR.DIRECTORATE_NAME
, P.COST
, P.ESTIMATED_COST
, COALESCE(CO.ADDED_COSTS, 0) AS ADDED_COSTS
, (COALESCE(P.COST, 0) + COALESCE(CO.ADDED_COSTS, 0)) AS TOTAL_COST
, P.ALLOCATION
, COALESCE(ED.PAST_YEARS, 0) AS PAST_YEARS
, COALESCE(ED.PAST_MONTHS, 0) AS PAST_MONTHS
, COALESCE(ED.CURRENT_MONTH, 0) AS CURRENT_MONTH
, COALESCE(ED.CURRENT_YEAR, 0) AS CURRENT_YEAR
, COALESCE(ED.TOTAL_EXPENSES, 0) AS TOTAL_EXPENSES
, COALESCE(ED.CASH_PAST_YEARS, 0) AS CASH_PAST_YEARS
, COALESCE(ED.CASH_PAST_MONTHS, 0) AS CASH_PAST_MONTHS
, COALESCE(ED.CASH_CURRENT_MONTH, 0) AS CASH_CURRENT_MONTH
, COALESCE(ED.CASH_CURRENT_YEAR, 0) AS CASH_CURRENT_YEAR
, COALESCE(ED.CASH_TOTAL_EXPENSES, 0) AS CASH_TOTAL_EXPENSES
, COALESCE(ED.TOTAL_EXPENSES_CASH, 0) AS TOTAL_EXPENSES_CASH
, ((COALESCE(P.COST, 0) + COALESCE(CO.ADDED_COSTS, 0)) - COALESCE(ED.TOTAL_EXPENSES, 0)) AS REMAINING
, P.DURATION
, COALESCE(DU.ADDED_DURATIONS, 0) AS ADDED_DURATIONS
, (COALESCE(P.DURATION, 0) + COALESCE(DU.ADDED_DURATIONS, 0)) AS TOTAL_DURATION
, P.START_DATE
, P.FINISH_DATE
, P.GOVERNORATE_ID
, G.GOVERNORATE_NAME
, P.PROVINCE_ID
, PR.PROVINCE_NAME
, P.DISTRICT_ID
, D.DISTRICT_NAME
, P.TOWN_ID
, T.TOWN_NAME
, COALESCE( (ED.TOTAL_EXPENSES / (COALESCE(P.COST, 0) + COALESCE(CO.ADDED_COSTS, 0)))/100, 0) AS FINANCIAL_ACHIEVEMENT
, P.MATERIAL_ACHIEVEMENT
, P.NOTES
FROM
PROJECTS P
INNER JOIN PROJECTS_TYPES PT ON P.TYPE_ID = PT.TYPE_ID
INNER JOIN CLASSES CLA ON P.CLASS_ID = CLA.CLASS_ID
INNER JOIN AGENCIES A ON P.AGENCY_ID = A.AGENCY_ID
LEFT JOIN MINISTRIES M ON P.MINISTRY_ID = M.MINISTRY_ID
LEFT JOIN DIRECTORATES DIR ON P.DIRECTORATE_ID = DIR.DIRECTORATE_ID
INNER JOIN GOVERNORATES G ON P.GOVERNORATE_ID = G.GOVERNORATE_ID
LEFT JOIN PROVINCES PR ON P.PROVINCE_ID = PR.PROVINCE_ID
LEFT JOIN DISTRICTS D ON P.DISTRICT_ID = D.DISTRICT_ID
LEFT JOIN TOWNS T ON P.TOWN_ID = T.TOWN_ID
LEFT JOIN EXPENSES_SUMS ED ON P.PROJ_ID = ED.PROJ_ID
LEFT JOIN COSTS_SUMS CO ON P.PROJ_ID = CO.PROJ_ID
LEFT JOIN DURATIONS_SUMS DU ON P.PROJ_ID = CO.PROJ_ID
ORDER BY
P.PROJ_YEAR
, P.TYPE_ID
, P.PROJ_NO