Need help deserializing JSON data from an external Web API and storing the data in a SQL database - sql

Question

So, I'm working with JSON for the first time within ASP.net.
Apologies, should have specified, language being worked on is c# within ASP.Net.
I currently have the following:
private static async void UpdateStreetWebApiProperties()
{
var client = new HttpClient();
var request = new HttpRequestMessage
{
Method = HttpMethod.Get,
RequestUri = new Uri("https://inventorymanchestertest.co.uk/api/property-feed/sales/search"),
Headers =
{
{ "ContentType", "application/json" },
{ "Authorization", "Bearer API_Key" },
},
};
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
if (response.IsSuccessStatusCode)
{
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
StreetWebApi_GetProperties streetwebapi_getproperties = new StreetWebApi_GetProperties();
}
else
{
Console.WriteLine("Could not get properties");
}
Console.WriteLine();
}
}
The variable body successfully draws down the JSON data in it's complete form. However, I'm looking to seperate each part into it's respective columns in a SQL DB.
The JSON looks like this:
{"data":[{"type":"property","id":"a91ab45e-5db8-4486-9bdf-f38dcb63c400","attributes":{"branch_uuid":"3e7a4a68-ab41-46c3-9a48-e3d1635cd056","inline_address":"101 London Road, Peterborough","public_address":"London Road, Peterborough, PE2","postcode":"PE2 9DD","bedrooms":5,"bathrooms":2,"receptions":2,"floor_area":null,"plot_area":null,"land_area":null,"property_type":"Detached House","property_age_bracket":null,"construction_year":null,"status":"For Sale","sale_status":"For Sale","lettings_status":null,"owner_label":"Vendor","tenure":null,"tenure_notes":null,"lease_expiry_year":null,"lease_expiry_date":null,"public_url":"https:\/\/inventorymanchester.co.uk\/platform\/properties\/a91ab45e-5db8-4486-9bdf-f38dcb63c400","created_at":"2022-06-17T15:18:53+01:00","updated_at":"2022-07-12T11:23:11+01:00","custom_meta_data":[],"property_urls":[],"viewing_booking_url":"https:\/\/inventorymanchester.co.uk\/platform\/properties\/a91ab45e-5db8-4486-9bdf-f38dcb63c400\/book-viewing"},"relationships":{"address":{"data":{"type":"address","id":"433518e4-d544-42ce-aba4-7d1137465af1"}},"details":{"data":{"type":"details","id":"1bf2b0fc-36c1-40f1-9e04-5b5cf72ffd0c"}},"salesListing":{"data":{"type":"sales_listing","id":"992114a6-3fcf-48b1-af1d-f5f3976a23da"}},"lettingsListing":{"data":null},"primaryImage":{"data":{"type":"media","id":"9ed40865-0873-4159-808b-5941faa520c9"}}}},{"type":"property","id":"4fd57964-71ea-4a77-b773-b4079a0f95dc","attributes":{"branch_uuid":"3e7a4a68-ab41-46c3-9a48-e3d1635cd056","inline_address":"4 Riverside Mead, Peterborough","public_address":"Riverside Mead, Peterborough, PE2","postcode":"PE2 8JN","bedrooms":4,"bathrooms":3,"receptions":2,"floor_area":null,"plot_area":null,"land_area":null,"property_type":"Detached House","property_age_bracket":null,"construction_year":null,"status":"Sold STC","sale_status":"Sold STC","lettings_status":null,"owner_label":"Vendor","tenure":null,"tenure_notes":null,"lease_expiry_year":null,"lease_expiry_date":null,"public_url":"https:\/\/inventorymanchester.co.uk\/platform\/properties\/4fd57964-71ea-4a77-b773-b4079a0f95dc","created_at":"2022-06-17T16:39:19+01:00","updated_at":"2022-07-19T11:39:26+01:00","custom_meta_data":[],"property_urls":[],"viewing_booking_url":"https:\/\/inventorymanchester.co.uk\/platform\/properties\/4fd57964-71ea-4a77-b773-b4079a0f95dc\/book-viewing"},"relationships":{"address":{"data":{"type":"address","id":"03d1a68a-6f4a-42ff-bf65-5b9768d6ce81"}},"details":{"data":{"type":"details","id":"f2b1a173-0611-4014-a980-894257b0bab0"}},"salesListing":{"data":{"type":"sales_listing","id":"be1cec3a-cf2f-40c4-a627-427cf3fbdfa7"}},"lettingsListing":{"data":null},"primaryImage":{"data":{"type":"media","id":"125542ce-27f1-4852-8fb6-b71daaaa70d1"}}}}],"included":[{"type":"address","id":"433518e4-d544-42ce-aba4-7d1137465af1","attributes":{"anon_address":"London Road, Peterborough, PE2","line_1":"101 London Road","line_2":"Peterborough","line_3":null,"town":"Peterborough","postcode":"PE2 9DD","inline":"101 London Road, Peterborough, PE2 9DD","longitude":-0.2465764,"latitude":52.560172}},{"type":"details","id":"1bf2b0fc-36c1-40f1-9e04-5b5cf72ffd0c","attributes":{"display_property_style":null,"work_required":null,"heating_system":null,"council_tax_band":null,"council_tax_cost":null,"local_authority":null,"service_charge":null,"service_charge_period":"month","service_charge_notes":null,"ground_rent":null,"ground_rent_period":"month","ground_rent_review_period_years":null,"ground_rent_uplift":null,"ground_rent_expiry":null,"full_description":"<p>Tortoise Property are pleased to offer this five bed detached house that is situated in the popular location of London Road, Fletton.<br><br>**Please call for either a viewing or virtual tour of this property.**<br><br>The property has a hallway, ground floor bathroom, bedroom, kitchen, dining room, lounge and converted garage on the ground floor. There are four bedrooms and the family bathroom on the first floor.<br><br>Outside the property has a front garden, a back garden and off-road parking for four cars.<br><\/p>","short_description":null,"location_summary":"London Road is a great location that is situated within walking distance of the city centre and local amenities. The Queensgate shopping centre is an 16 minute walk. The train station is a 21 minute walk or 5 minutes by car. The Kings secondary school is a 6 minute drive.\r\n\r\nPeterborough City Centre can be reached by car in 5 minutes and by bus in 10 minutes.\r\n\r\nThe A1 Junction can be reached by car in 11 minutes and the surrounding parkways give access to the A47 both east and west. \r\n\r\nWe love Fletton because of the lifestyle you can enjoy here. Great homes, close to nature and superb facilities make this one of our favourite places to live and work.","has_parking":null,"has_outdoor_space":null,"virtual_tour":null,"shared_ownership":false,"shared_ownership_notes":null,"shared_ownership_rent":null,"shared_ownership_rent_frequency":null,"shared_ownership_percentage_sold":null,"created_at":"2022-06-17T15:18:54+01:00","updated_at":"2022-06-17T15:28:08+01:00"}},{"type":"sales_listing","id":"992114a6-3fcf-48b1-af1d-f5f3976a23da","attributes":{"status":"For Sale","price":300000,"price_qualifier":"In Excess of","display_price":true,"archived":false,"is_low_profile":false,"occupancy_status":1,"new_home":false,"created_at":"2022-06-17T15:29:16+01:00","updated_at":"2022-06-17T15:29:57+01:00"}},{"type":"media","id":"9ed40865-0873-4159-808b-5941faa520c9","attributes":{"name":"136511_31517777_IMG_17_0000","order":0,"is_featured":true,"feature_index":1,"title":null,"is_image":true,"url":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg","urls":{"thumbnail":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg?tr=pr-true,n-property_thumb","small":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg?tr=pr-true,n-property_small_fill_crop","medium":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg?tr=pr-true,n-property_medium_fill_crop","large":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg?tr=pr-true,n-property_large_fill_crop","hero":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg?tr=pr-true,n-property_hero","full":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844548\/136511_31517777_IMG_17_0000.jpeg"}}},{"type":"address","id":"03d1a68a-6f4a-42ff-bf65-5b9768d6ce81","attributes":{"anon_address":"Riverside Mead, Peterborough, PE2","line_1":"4 Riverside Mead","line_2":"Peterborough","line_3":null,"town":"Peterborough","postcode":"PE2 8JN","inline":"4 Riverside Mead, Peterborough, PE2 8JN","longitude":-0.2305068,"latitude":52.5631968}},{"type":"details","id":"f2b1a173-0611-4014-a980-894257b0bab0","attributes":{"display_property_style":null,"work_required":null,"heating_system":null,"council_tax_band":null,"council_tax_cost":null,"local_authority":null,"service_charge":null,"service_charge_period":"month","service_charge_notes":null,"ground_rent":null,"ground_rent_period":"month","ground_rent_review_period_years":null,"ground_rent_uplift":null,"ground_rent_expiry":null,"full_description":"<p>Here at Tortoise Property, we pride ourselves on doing things differently, by offering a complete partnership and consistent approach to construct a comprehensive marketing package tailored for the single purpose of selling your property as agreed at the initial valuation.<br><br>\"Tortoise provided me with a comprehensive property management service over a four year period, offering a friendly, transparent and consistent relationship.<br><br>When I decided to sell my property I immediately engaged with Tortoise to undertake the action. They actively advertised my property, were proactive with local sale opportunities and provided timely updates on progress. The sale on my property was agreed, exchanged and completed within five weeks. I would highly recommend Tortoise Property for their professional and friendly approach.\" - Tracey Matthews - Testimonial <br><br>Valuations<br><br>We concentrate on the maximum price your house is likely to sell for then agree a sensible timeframe for which the property should be sold whilst clearly explaining how the fee you are charged, is invested in enabling us to find your buyer from across the country.<br><br>\"I recently used Tortoise to sell my house, Chris came round and went through everything there price was better than all others I had received and they seemed a lot more genuine.\" - James Richards - Testimonial <br><br>Relationship management<br><br>Our relationship managers are here to personally look after you offering complete transparency and guidance throughout the sales process, following a 12-week programme that provides regular viewings with prompt feedback.<br><br>\"They say selling your home can be very stressful, not with this team, there was never a time you could not get in touch with these guys. You will be in safe hands all the way from start to finish.\" - Maxine Ambrose - Testimonial <br><br>Facebook<br><br>The growth of our sales portfolio into the wider Peterborough area we believe is the result of our unique strategy to capture maximum exposure. Facebook provides us with the opportunity to target our property marketing and expand our reach beyond the property portals.<br><br>Facebook live<br><br>The potential reach of a digital tour is limitless. Our live feed property tours on facebook are great for potential buyers to not only view the property but to ask relevant questions and get instant replies from wherever they are based.<br><br>Online and traditional auctions<br><br>Our property auction service gives you the ability to sell your property at auction either online or at a live auction. The buyer pays a commission so your house is sold at no cost to you. The buyer must complete within 28 or 56 days meaning your property is sold fast.<br><br>Performance-related fees<br><br>Here at Tortoise we do offer traditional fee structures based on a standard percentage of the purchase price or a fixed fee. However, we are so good at what we do that we are confident enough to offer you performance related fees we believe we should win together.<br><br>24\/7 services<br><br>Property sales can be daunting, especially if it is your first time. Here at Tortoise, we have real people available to talk to 24 hours a day 7 days a week as well as a live web chat so that you can chat to someone at your convenience.<br><br><br><br><br><br>Negotiator awards<br><br>In 2017 Tortoise Property was shortlisted for website of the year in the negotiator awards competing with large national estate agency chains illustrating the quality and presentation of our brand and level of service.<br><br>Our micro-site offers plenty of information so please choose from one of the tabs on the left that is applicable to your requirements and we look forward to seeing you in the near future or to find out more about us and our services visit www.tortoise property.co.uk<br><\/p>","short_description":null,"location_summary":null,"has_parking":null,"has_outdoor_space":null,"virtual_tour":null,"shared_ownership":false,"shared_ownership_notes":null,"shared_ownership_rent":null,"shared_ownership_rent_frequency":null,"shared_ownership_percentage_sold":null,"created_at":"2022-06-17T16:39:20+01:00","updated_at":"2022-06-17T16:45:16+01:00"}},{"type":"sales_listing","id":"be1cec3a-cf2f-40c4-a627-427cf3fbdfa7","attributes":{"status":"Sold STC","price":350000,"price_qualifier":"Fixed Price","display_price":true,"archived":false,"is_low_profile":false,"occupancy_status":1,"new_home":false,"created_at":"2022-07-19T11:38:08+01:00","updated_at":"2022-07-19T11:39:26+01:00"}},{"type":"media","id":"125542ce-27f1-4852-8fb6-b71daaaa70d1","attributes":{"name":"136511_31519016_IMG_00_0000","order":0,"is_featured":true,"feature_index":1,"title":null,"is_image":true,"url":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg","urls":{"thumbnail":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg?tr=pr-true,n-property_thumb","small":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg?tr=pr-true,n-property_small_fill_crop","medium":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg?tr=pr-true,n-property_medium_fill_crop","large":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg?tr=pr-true,n-property_large_fill_crop","hero":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg?tr=pr-true,n-property_hero","full":"https:\/\/ik.imagekit.io\/street\/street-mobile\/properties\/general\/844609\/136511_31519016_IMG_00_0000.jpeg"}}}],"meta":{"pagination":{"total":2,"count":2,"per_page":250,"current_page":1,"total_pages":1}},"links":{"self":"https:\/\/inventorymanchester.co.uk\/api\/property-feed\/sales\/search?page%5Bnumber%5D=1","first":"https:\/\/inventorymanchester.co.uk\/api\/property-feed\/sales\/search?page%5Bnumber%5D=1","last":"https:\/\/inventorymanchester.co.uk\/api\/property-feed\/sales\/search?page%5Bnumber%5D=1"}}
I've used XML before and used XML readers to do the job but my understanding is that JSON is a very different way of working with data.
Could I please have some examples of ways I could deserialise the information and then some stored procedures to store them correctly as currently I've looked up several ways to acheive this but none have made too much sense.
Many thanks
-- Micro update --
I have setup the classes using paste special for JSON and am then running the following:
private static async void UpdateStreetWebApiProperties()
{
var client = new HttpClient();
var request = new HttpRequestMessage
{
Method = HttpMethod.Get,
RequestUri = new Uri("https://inventorymanchestertest.co.uk/api/property-feed/sales/search"),
Headers =
{
{ "ContentType", "application/json" },
{ "Authorization", "Bearer auth_foo" },
},
};
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
var properties = await response.Content.ReadAsStringAsync();
if (response.IsSuccessStatusCode)
{
tortoise_common.JSONModel.Datum streetResponse = JsonConvert.DeserializeObject<tortoise_common.JSONModel.Datum>(properties);
{
Console.WriteLine(streetResponse);
}
}
else
{
Console.WriteLine("Could not get branch");
}
Console.WriteLine(properties);
}
}
However, upon running the code through the deserializer, the values assigned are coming through as "null". I would assume this would mean that the classes are not getting and setting any values.
Not quite sure on why the serializer is returning no values despite the "properties" variable having the full JSON string within.

Answer 1

Ok, so you could do this one of two ways:
use netonsoft - parse the jason.
(just like we do with xml - really a VERY simular approach).
Or, build a class that represents the XML, or json data.
So, in some cases, its easier to pluck out the values of the json string (using parsers and attribute selectors). A few loops.
So, MORE important, what is that json data?
Is it
Simple one reocrd, say FirstName, LastName, City?
or, is the data REPEATING data - a complex master customer invoice form with say one record at the top (customer information), and then repeating data of invoice details?
So, lets take a quick look at the data, see what we get.
Add a new blank class to y our project - ctrl-A - del key (clear out the class).
Now, we grab your sample json (copy).
Now, back to the empty class in Visual Studio, and then this:
when we do this, the base (bottom root) class tends to get a default name of RootObject, and you no doubt been code and slicing and dicing json data all day.
So, we either change that name, but better yet, lets just toss around the whole mess a namespace.
Well, VS now cranks out this:
Namespace MyDataTest
Public Class Rootobject
Public Property data() As Datum
Public Property included() As Included
Public Property meta As Meta
Public Property links As Links
End Class
Public Class Meta
Public Property pagination As Pagination
End Class
Public Class Pagination
Public Property total As Integer
Public Property count As Integer
Public Property per_page As Integer
Public Property current_page As Integer
Public Property total_pages As Integer
End Class
Public Class Links
Public Property self As String
Public Property first As String
Public Property last As String
End Class
Public Class Attributes
Public Property branch_uuid As String
Public Property inline_address As String
Public Property public_address As String
Public Property postcode As String
Public Property bedrooms As Integer
Public Property bathrooms As Integer
Public Property receptions As Integer
Public Property floor_area As Object
Public Property plot_area As Object
Public Property land_area As Object
Public Property property_type As String
Public Property property_age_bracket As Object
Public Property construction_year As Object
Public Property status As String
Public Property sale_status As String
Public Property lettings_status As Object
Public Property owner_label As String
Public Property tenure As Object
Public Property tenure_notes As Object
Public Property lease_expiry_year As Object
Public Property lease_expiry_date As Object
Public Property public_url As String
Public Property created_at As Date
Public Property updated_at As Date
Public Property custom_meta_data() As Object
Public Property property_urls() As Object
Public Property viewing_booking_url As String
End Class
(and more - it too rude to post more code - stop here)
End Class
End Namespace
Hum, not a super simple structure, and it does show/have some "repeating" data.
(and with repeating data, we care because then that SIGNIFICALY increases the difficult in taking that data to sql server.
One of the interview type of questions that google, Microsoft ask you?
They are questions of how high, how far, how big.
In other words, they want developers that have a sense of "scope".
I mean, you can walk with ease to a store down the block. But, if it is 5 miles away, then you really starting to solve that issue by adopting some form of motorized transportation - its too far to walk (so, how far, how big, how high type of question).
Ok, so we generated the class. lets look at it.
Right click -> view class diagram.
We get this:
Hum, ok, lets expand a few of the objects now:
Ok, I'm going for coffee. So, at least we asked the how high, how far, and how big question then right?
You not just adding a few simple rows to some database.
You asking me to come to your house, clear out the garage, then cook you dinner, and then clean up the kitchen afterwards.
But, as least you can see how Visual Studio has some built in tools, and some things that let you get a quick and easy feel for what you up against.
Looking at above? I think you probably take the class road, and let Visual Studio create the class for you. You can then send that string to newtonsoft, and it will then peal it out like layers of a onion to the above class. (and its only 4 lines of code to do this).
At that point, then you can start to take that data, and send it to SQL server. but, it not just a simple User name and address here, but a grouping of multiple tables and data - all of which would have to be added to a database, and no doubt highly relational database that has all that information.
If you have that existing database schema? Then hey, this is not too bad then.
but, do the fields and columns and existing database you have match the json data?
So, then we ask another how high how far, how big question:
If the columns and database structure you have now does NOT match the json data, then we not really adding rows to a database, but are in fact doing a type of data migration - and now we have to introduce "mapping" for what amounts to 100+ columns or more. So, that's what I have that 2nd question:
Do you have a matching database structure (schema) now that follows the field names used in this json data ? (are field names 100% exact???).
Since, if the columns don't match, and that database you already have does not match the incoming data? Then you just increased the workload here by significant amounts - we now not adding rows, but having to translate from one data schema to another - and that going to cost you even more time and efforts here.
Edit2: parse out using test data
Ok, so I don't have the web service, so for this I'll just paste the sample data into a notepad.txt and read it. No problem.
So, we take the above data - use the paste speical->json.
I changed the name space around the class - due to having other test json objects. But, no changes - just a extra name space.
So, the start of the class looks like this:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
namespace jdat
{
public class Rootobject
{
public Datum[] data { get; set; }
public Included[] included { get; set; }
public Meta meta { get; set; }
public Links links { get; set; }
}
public class Meta
{
public Pagination pagination { get; set; }
}
public class Pagination
{
public int total { get; set; }
public int count { get; set; }
public int per_page { get; set; }
etc. etc. etc.
Ok, so now our code to test this:
drop a button on a web form, and this code:
protected void cmdPARSE_Click(object sender, EventArgs e)
{
// don't have web service - paste test data into notepage
string strBufJSON = File.ReadAllText(#"c:\test7\jdata.txt");
// pretend above is web service results
jdat.Rootobject jData;
jData = JsonConvert.DeserializeObject<jdat.Rootobject>(strBufJSON);
// now display some values
Debug.Print(jData.data[0].attributes.public_url);
Debug.Print(jData.data[0].attributes.property_type);
Debug.Print(jData.data[0].attributes.sale_status);
}
So a few things:
Try tagging the language - I am fluent in vb/c#, but will post as vb.net if lanaguage not speicfed.
So, since the data is a "array", then you have to either loop over all data tiems, or just pull out [0] as above.
Intel sense still works shows all items/properties of the class - VERY nice and helpful.