Transform Multiple rows of JSON format to SQL table - sql

Question

I have JSON file in Blob Storage that looks like this:
{
"Id": "****************************",
"Status": "OK",
"ProviderName": "Xero API Previewer",
"DateTimeUTC": "\/Date(1576561543598)\/",
"Invoices": [
{
"Type": "ACCPAY",
"InvoiceID": "*****************************",
"InvoiceNumber": "457489",
"Reference": "",
"Payments": [],
"CreditNotes": [],
"Prepayments": [],
"Overpayments": [],
"AmountDue": 0.00,
"AmountPaid": 0.00,
"AmountCredited": 0.00,
"CurrencyRate": 1.000000,
"IsDiscounted": false,
"HasAttachments": false,
"HasErrors": false,
"Contact": {
"ContactID": "************************************",
"Name": "********************",
"Addresses": [],
"Phones": [],
"ContactGroups": [],
"ContactPersons": [],
"HasValidationErrors": false
},
"DateString": "2102-11-26T00:00:00",
"Date": "\/Date(4193942400000+0000)\/",
"DueDateString": "2012-11-28T00:00:00",
"DueDate": "\/Date(1354060800000+0000)\/",
"Status": "VOIDED",
"LineAmountTypes": "Inclusive",
"LineItems": [
{
"Description": "Parking ",
"UnitAmount": 465.01,
"TaxType": "INPUT2",
"TaxAmount": 60.65,
"LineAmount": 465.01,
"AccountCode": "274",
"Tracking": [
{
"Name": "Region",
"Option": "New Zealand",
"TrackingCategoryID": "****************************************",
"Options": []
},
{
"Name": "Owner",
"Option": "Head Office",
"TrackingCategoryID": "***************************************",
"Options": []
}
],
"Quantity": 1.0000,
"LineItemID": "**************************************"
}
],
"SubTotal": 404.36,
"TotalTax": 60.65,
"Total": 465.01,
"UpdatedDateUTC": "\/Date(1355876228590+0000)\/",
"CurrencyCode": "NZD"
},
{
"Type": "ACCPAY",
"InvoiceID": "**************************************",
"InvoiceNumber": "176295-01",
"Reference": "",
"Payments": [
{
"PaymentID": "********************************************",
"Date": "\/Date(1576454400000+0000)\/",
"Amount": 137.43,
"Reference": "",
"CurrencyRate": 1.000000,
"HasAccount": false,
"HasValidationErrors": false
}
],
"CreditNotes": [],
"Prepayments": [],
"Overpayments": [],
"AmountDue": 0.00,
"AmountPaid": 137.43,
"AmountCredited": 0.00,
"CurrencyRate": 1.000000,
"IsDiscounted": false,
"HasAttachments": true,
"HasErrors": false,
"Contact": {
"ContactID": "************************************",
"Name": "InkWorks",
"Addresses": [],
"Phones": [],
"ContactGroups": [],
"ContactPersons": [],
"HasValidationErrors": false
},
"DateString": "2019-12-17T00:00:00",
"Date": "\/Date(1576540800000+0000)\/",
"DueDateString": "2020-01-20T00:00:00",
"DueDate": "\/Date(1579478400000+0000)\/",
"Status": "PAID",
"LineAmountTypes": "Inclusive",
"LineItems": [
{
"Description": "general stationery",
"UnitAmount": 137.43,
"TaxType": "INPUT2",
"TaxAmount": 17.93,
"LineAmount": 137.43,
"AccountCode": "273",
"Tracking": [
{
"Name": "Owner",
"Option": "Head Office",
"TrackingCategoryID": "******************************",
"Options": []
}
],
"Quantity": 1.0000,
"LineItemID": "****************************************"
}
],
"SubTotal": 119.50,
"TotalTax": 17.93,
"Total": 137.43,
"UpdatedDateUTC": "\/Date(1576524509820+0000)\/",
"CurrencyCode": "NZD",
"FullyPaidOnDate": "\/Date(1576454400000+0000)\/"
},
I want to store LineItems into a Table with columns as follows:
InvoiceID LineItemID Description LineAmount AccountCode Date
There can be multiple LineItems in a single invoice.
I managed to get the top level only with the following code.
WITH cte AS (
SELECT CAST (BulkColumn AS NVARCHAR(MAX)) AS jsonData
FROM
OPENROWSET(
BULK 'temp/XeroJson.json',
DATA_SOURCE = 'test'
-- FORMATFILE_DATA_SOURCE = 'test'
, SINGLE_CLOB
) AS blob
)
SELECT *
FROM cte
CROSS APPLY
OPENJSON(cte.jsonData) j
How can I make this work?

Answer 1

This is my test .json file, I placed it in my storage account :
{
"Id": "****************************",
"Status": "OK",
"ProviderName": "Xero API Previewer",
"DateTimeUTC": "\/Date(1576561543598)\/",
"Invoices":
[
{
"Type": "ACCPAY",
"InvoiceID": "123456",
"InvoiceNumber": "457489",
"Reference": "",
"Payments": [],
"CreditNotes": [],
"Prepayments": [],
"Overpayments": [],
"AmountDue": 0.00,
"AmountPaid": 0.00,
"AmountCredited": 0.00,
"CurrencyRate": 1.000000,
"IsDiscounted": false,
"HasAttachments": false,
"HasErrors": false,
"Contact": {
"ContactID": "************************************",
"Name": "********************",
"Addresses": [],
"Phones": [],
"ContactGroups": [],
"ContactPersons": [],
"HasValidationErrors": false
},
"DateString": "2102-11-26T00:00:00",
"Date": "\/Date(4193942400000+0000)\/",
"DueDateString": "2012-11-28T00:00:00",
"DueDate": "\/Date(1354060800000+0000)\/",
"Status": "VOIDED",
"LineAmountTypes": "Inclusive",
"LineItems": [{
"Description": "Parking ",
"UnitAmount": 465.01,
"TaxType": "INPUT2",
"TaxAmount": 60.65,
"LineAmount": 465.01,
"AccountCode": "274",
"Tracking": [{
"Name": "Region",
"Option": "New Zealand",
"TrackingCategoryID": "****************************************",
"Options": []
}, {
"Name": "Owner",
"Option": "Head Office",
"TrackingCategoryID": "***************************************",
"Options": []
}
],
"Quantity": 1.0000,
"LineItemID": "**************************************"
},
{
"Description": "Parking2 ",
"UnitAmount": 111.11,
"TaxType": "INPUT2",
"TaxAmount": 60.65,
"LineAmount": 465.01,
"AccountCode": "276",
"Tracking": [{
"Name": "Region",
"Option": "New Zealand",
"TrackingCategoryID": "****************************************",
"Options": []
}, {
"Name": "Owner",
"Option": "Head Office",
"TrackingCategoryID": "***************************************",
"Options": []
}
],
"Quantity": 1.0000,
"LineItemID": "**************************************"
}
],
"SubTotal": 404.36,
"TotalTax": 60.65,
"Total": 465.01,
"UpdatedDateUTC": "\/Date(1355876228590+0000)\/",
"CurrencyCode": "NZD"
},
{
"Type": "ACCPAY",
"InvoiceID": "1234567",
"InvoiceNumber": "176295-01",
"Reference": "",
"Payments": [{
"PaymentID": "********************************************",
"Date": "\/Date(1576454400000+0000)\/",
"Amount": 137.43,
"Reference": "",
"CurrencyRate": 1.000000,
"HasAccount": false,
"HasValidationErrors": false
}
],
"CreditNotes": [],
"Prepayments": [],
"Overpayments": [],
"AmountDue": 0.00,
"AmountPaid": 137.43,
"AmountCredited": 0.00,
"CurrencyRate": 1.000000,
"IsDiscounted": false,
"HasAttachments": true,
"HasErrors": false,
"Contact": {
"ContactID": "************************************",
"Name": "InkWorks",
"Addresses": [],
"Phones": [],
"ContactGroups": [],
"ContactPersons": [],
"HasValidationErrors": false
},
"DateString": "2019-12-17T00:00:00",
"Date": "\/Date(1576540800000+0000)\/",
"DueDateString": "2020-01-20T00:00:00",
"DueDate": "\/Date(1579478400000+0000)\/",
"Status": "PAID",
"LineAmountTypes": "Inclusive",
"LineItems": [{
"Description": "general stationery",
"UnitAmount": 137.43,
"TaxType": "INPUT2",
"TaxAmount": 17.93,
"LineAmount": 137.43,
"AccountCode": "273",
"Tracking": [{
"Name": "Owner",
"Option": "Head Office",
"TrackingCategoryID": "******************************",
"Options": []
}
],
"Quantity": 1.0000,
"LineItemID": "****************************************"
}
],
"SubTotal": 119.50,
"TotalTax": 17.93,
"Total": 137.43,
"UpdatedDateUTC": "\/Date(1576524509820+0000)\/",
"CurrencyCode": "NZD",
"FullyPaidOnDate": "\/Date(1576454400000+0000)\/"
}
]
}
As you can see the first invoice has two lineItems for demo here.
This is my sql script to exract from the json from storage :
CREATE DATABASE SCOPED CREDENTIAL AccessAzureInvoices
WITH
IDENTITY = 'SHARED ACCESS SIGNATURE'
-- REMOVE ? FROM THE BEGINNING OF THE SAS TOKEN
, SECRET = '<YOUR STORAGE SAS TOKEN>'
;
CREATE EXTERNAL DATA SOURCE MyAzureInvoices
WITH
( LOCATION = 'https://<YOUR STORAGE NAME>.blob.core.windows.net/<CONTAINER NAME WHERE YOUR JSON FILE IN>'
, CREDENTIAL = AccessAzureInvoices
, TYPE = BLOB_STORAGE
)
;
DECLARE #jsonVariable NVARCHAR(MAX);
set #jsonVariable = (select * from OPENROWSET(
BULK '<YOUR JSON FILE NAME>.json',
DATA_SOURCE = 'MyAzureInvoices',
SINGLE_CLOB
) AS blob)
select * from (
SELECT *
FROM OPENJSON(#jsonVariable,N'$.Invoices') WITH (
InvoiceID VARCHAR(200) N'$.InvoiceID',
Date VARCHAR(200) N'$.Date',
LineItems NVARCHAR(MAX) N'$.LineItems' as json
))as table1
CROSS APPLY OPENJSON(LineItems) with (
LineAmount VARCHAR(200) '$.LineAmount',
AccountCode VARCHAR(200)'$.AccountCode',
LineItemID VARCHAR(200) '$.LineItemID',
Description VARCHAR(200) '$.Description'
)
Result :
Hope it helps !