Concatenate/ build Json objects From PostgresSQl database - sql

Im trying to build Json array from data existing in a database.
I shall build a Json file that shall match the following Json-file with PostgresSQL.
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Student iformation",
"type": "object",
"required": [
"student",
"name",
"login",
"program",
"branch",
"finished",
],
"properties": {
"student": {
"type": "string",
"minLength": 10,
"maxLength": 10,
"title": "A national identification number, 10 digits"
},
"name": {
"type": "string",
"title": "The name of the student"
},
"login": {
"type": "string",
"title": "The univerity issued computer login"
},
"program": {
"type": "string",
},
"branch": {
"anyOf":[{"type": "string"},{"type": "null"}],
},
"finished": {
"type": "array",
"title": "A list of read courses",
"items": {
"type": "object",
"required": [
"course",
"code",
"credits",
"grade"
],
"properties": {
"course": {
"type": "string",
"title": "Course name"
},
"code": {
"type": "string",
"minLength": 6,
"maxLength": 6,
"title": "Course code"
},
"credits": {
"type": "number",
"title": "Academic credits"
},
"grade": {
"enum" : ["U", "3", "4", "5"]
}
}
}
}
I have tried to do the following to get a better understanding of how to concatenate, build and arrange data that exists in the database:
SELECT array_to_json(array_agg(row_to_json(t))) FROM (
SELECT idnr, name, login, program from students) t;
and
select json_build_object('properties',
json_build_object('student',
json_build_object('idnr',idnr),'name',
json_build_object('name',name),'login',
json_build_object('login',login),'program',
json_build_object('program',program),'branch',
json_build_object('branch',branch)))
from Basicinformation;
How do I build, concatenate objects with PostgresSQL?

Related

Avro Schema: multiple records reference same data type issue: Unknown union branch

I have Avro Schema: customer record import the CustomerAddress subset.
[
{
"type": "record",
"namespace": "com.example",
"name": "CustomerAddress",
"fields": [
{ "name": "address", "type": "string" },
{ "name": "city", "type": "string" },
{ "name": "postcode", "type": ["string", "int"] },
{ "name": "type","type": {"type": "enum","name": "type","symbols": ["POBOX","RESIDENTIAL","ENTERPRISE"]}}
]
},
{
"type": "record",
"namespace": "com.example",
"name": "Customer",
"fields": [
{ "name": "first_name", "type": "string" },
{ "name": "middle_name", "type": ["null", "string"], "default": null },
{ "name": "last_name", "type": "string" },
{ "name": "age", "type": "int" },
{ "name": "height", "type": "float" },
{ "name": "weight", "type": "float" },
{ "name": "automated_email", "type": "boolean", "default": true },
{ "name": "customer_emails", "type": {"type": "array","items": "string"},"default": []},
{ "name": "customer_address", "type": "com.example.CustomerAddress" }
]
}
]
i have JSON payload:
{
"Customer" : {
"first_name": "John",
"middle_name": null,
"last_name": "Smith",
"age": 25,
"height": 177.6,
"weight": 120.6,
"automated_email": true,
"customer_emails": ["ning.chang#td.com", "test#td.com"],
"customer_address":
{
"address": "21 2nd Street",
"city": "New York",
"postcode": "10021",
"type": "RESIDENTIAL"
}
}
}
when i runt the command: java -jar avro-tools-1.8.2.jar fromjson --schema-file customer.avsc customer.json
got the following exception:
Exception in thread "main" org.apache.avro.AvroTypeException: Unknown union branch Customer
In your JSON data you use the key Customer but you have to use the fully qualified name. So it should be com.example.Customer.

Export BigQuery table schema to JSON Schema

It is possible to export a bigquery table schema to a JSON file but the resulting JSON file is a bigquery table schema and not a JSON schema.
I am looking for a way to generate a JSON schema using a bigquery table based on the standard available here: https://json-schema.org/
This looks something like this:
{
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "http://example.com/root.json",
"type": "object",
"title": "The Root Schema",
"required": [
"glossary"
],
"properties": {
"glossary": {
"$id": "#/properties/glossary",
"type": "object",
"title": "The Glossary Schema",
"required": [
"title",
"GlossDiv"
],
"properties": {
"title": {
"$id": "#/properties/glossary/properties/title",
"type": "string",
"title": "The Title Schema",
"default": "",
"examples": [
"example glossary"
],
"pattern": "^(.*)$"
},
"GlossDiv": {
"$id": "#/properties/glossary/properties/GlossDiv",
"type": "object",
"title": "The Glossdiv Schema",
"required": [
"title",
"GlossList"
],
"properties": {
"title": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/title",
"type": "string",
"title": "The Title Schema",
"default": "",
"examples": [
"S"
],
"pattern": "^(.*)$"
},
"GlossList": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList",
"type": "object",
"title": "The Glosslist Schema",
"required": [
"GlossEntry"
],
"properties": {
"GlossEntry": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry",
"type": "object",
"title": "The Glossentry Schema",
"required": [
"ID",
"SortAs",
"GlossTerm",
"Acronym",
"Abbrev",
"GlossDef",
"GlossSee"
],
"properties": {
"ID": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/ID",
"type": "string",
"title": "The Id Schema",
"default": "",
"examples": [
"SGML"
],
"pattern": "^(.*)$"
},
"SortAs": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/SortAs",
"type": "string",
"title": "The Sortas Schema",
"default": "",
"examples": [
"SGML"
],
"pattern": "^(.*)$"
},
"GlossTerm": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/GlossTerm",
"type": "string",
"title": "The Glossterm Schema",
"default": "",
"examples": [
"Standard Generalized Markup Language"
],
"pattern": "^(.*)$"
},
"Acronym": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/Acronym",
"type": "string",
"title": "The Acronym Schema",
"default": "",
"examples": [
"SGML"
],
"pattern": "^(.*)$"
},
"Abbrev": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/Abbrev",
"type": "string",
"title": "The Abbrev Schema",
"default": "",
"examples": [
"ISO 8879:1986"
],
"pattern": "^(.*)$"
},
"GlossDef": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/GlossDef",
"type": "object",
"title": "The Glossdef Schema",
"required": [
"para",
"GlossSeeAlso"
],
"properties": {
"para": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/GlossDef/properties/para",
"type": "string",
"title": "The Para Schema",
"default": "",
"examples": [
"A meta-markup language, used to create markup languages such as DocBook."
],
"pattern": "^(.*)$"
},
"GlossSeeAlso": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/GlossDef/properties/GlossSeeAlso",
"type": "array",
"title": "The Glossseealso Schema",
"items": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/GlossDef/properties/GlossSeeAlso/items",
"type": "string",
"title": "The Items Schema",
"default": "",
"examples": [
"GML",
"XML"
],
"pattern": "^(.*)$"
}
}
}
},
"GlossSee": {
"$id": "#/properties/glossary/properties/GlossDiv/properties/GlossList/properties/GlossEntry/properties/GlossSee",
"type": "string",
"title": "The Glosssee Schema",
"default": "",
"examples": [
"markup"
],
"pattern": "^(.*)$"
}
}
}
}
}
}
}
}
}
}
}
BigQuery does not use the json-schema standard for the tables schema. I found two projects that have the code available to go from json-schema to BigQuery schema:
jsonschema-bigquery
jsonschema-transpiler
You could try using those projects as reference to create the opposite transformation. Also, you could create a feature request to the BigQuery team, asking to include the json-schema standard as an output format option.
No this is not possible without writing a program to do so for you.
There is a feature request made by me that requests this functionality.
https://issuetracker.google.com/issues/145308573

How to display json schema using Dataweave in Mule 4?

in Mule 4 I just want to display the JSONschema using DataWeave but I get an error for reference ids or any field started with '$' in JSON schema. The mime/type is application/json. The goal is to display the schema, I'll appreciate any suggestions. Thanks!
SAMPLE SCHEMA
{
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "http://example.com/example.json",
"type": "object",
"title": "The Root Schema",
"properties": {
"checked": {
"$id": "/properties/checked",
"type": "boolean",
"title": "The Checked Schema",
"default": false,
"examples": [
false
]
},
"dimensions": {
"$id": "/properties/dimensions",
"type": "object",
"title": "The Dimensions Schema",
"required": [
"width",
"height"
],
"properties": {
"width": {
"$id": "/properties/dimensions/properties/width",
"type": "integer",
"title": "The Width Schema",
"default": 0,
"examples": [
5
]
},
"height": {
"$id": "/properties/dimensions/properties/height",
"type": "integer",
"title": "The Height Schema",
"default": 0,
"examples": [
10
]
}
}
},
"id": {
"$id": "/properties/id",
"type": "integer",
"title": "The Id Schema",
"default": 0,
"examples": [
1
]
}
}
}
Use blackslash \ before $, this will print your schema with $ in column names. Unfortuanely, $ as a prefix is not treated as part of string in Mule
Example
"\$id": "/properties/dimensions/properties/width"
will print
"$id": "/properties/dimensions/properties/width"

How to prevent certain fields based on another fields value with JSON Schema validator

Depending on the salaryRange the user selects I need to validate differently by requiring some fields and rejecting others. I feel like its a combination of allOf and not but I can't seem to quite get it.
Scenario #1
User selects salaryRange(Hourly)
Require hourlyRate
Prevent the submission of fields feeOne and feeTwo
Scenario #2
User selects salaryRange(0-50k OR 50-100k)
Require feeOne and feeTwo
Prevent the submission of field hourlyRate
Here is my schema
{
"schema": "http://json-schema.org/draft-04/schema#",
"$id": "http://mysite/schemas/job.json#",
"title": "Job",
"description": "Create job",
"type": "object",
"properties": {
"title": { "type": "string" },
"description": { "type": "string" },
"salaryRange": { "enum": ["0-50k", "50-100k", "100-150k", "150-200k", "200-300k", "300k+", "nonExempt", "Hourly"] },
"hourlyRate": {
"type": "number",
"minimum": 0,
"maximum": 300
},
"feeOne": {
"type": "number",
"minimum": 0
},
"feeTwo": {
"type": "number",
"minimum": 0
}
} ,
"additionalProperties": false,
"required": [
"title",
"description",
"salaryRange"
]
}
You can use oneOf and not required to model all possible combinations.
Here is an example in js:
https://runkit.com/embed/cf8cra1mwvx3/
{
"schema": "http://json-schema.org/draft-04/schema#",
"$id": "http://mysite/schemas/job.json#",
"title": "Job",
"description": "Create job",
"type": "object",
"properties": {
"title": { "type": "string" },
"description": { "type": "string" },
"salaryRange": { "enum": ["0-50k", "50-100k", "100-150k", "150-200k", "200-300k", "300k+", "nonExempt", "Hourly"] },
"hourlyRate": {
"type": "number",
"minimum": 0,
"maximum": 300
},
"feeOne": {
"type": "number",
"minimum": 0
},
"feeTwo": {
"type": "number",
"minimum": 0
}
},
"oneOf": [
{
"description": "Disallow fees for hourly salary",
"properties": {
"salaryRange": { "enum": ["Hourly"] }
},
"required": ["hourlyRate"],
"allOf": [
{"not":{"required":["feeOne"]}},
{"not":{"required":["feeTwo"]}}
]
},
{
"description": "Disallow hourly rate for 0-50k, 50-100k salaries",
"properties": {
"salaryRange": { "enum": ["0-50k", "50-100k"] }
},
"required": ["feeOne", "feeTwo"],
"not":{"required":["hourlyRate"]}
},
{
"description": "Allow other cases",
"properties": {
"salaryRange": { "not" : {"enum": ["Hourly", "0-50k", "50-100k"] } }
}
}
],
"additionalProperties": false,
"required": [
"title",
"description",
"salaryRange"
]
}

Resolving error: returned "Output field used as input"

I'm trying to create a BigQuery table using Python. Other operations (queries, retrieving table bodies etc.) are working fine, but when trying to create a table I'm stuck with an error:
apiclient.errors.HttpError: https://www.googleapis.com/bigquery/v2/projects/marechal-consolidation/datasets/marechal_results/tables?alt=json
returned "Output field used as input">
Here's the command I'm executing:
projectId = 'xxxx'
dataSet = 'marechal_results'
with open(filePath+'tableStructure.json') as data_file:
structure = json.load(data_file)
table_result = tables.insert(projectId=projectId, datasetId=dataSet, body=structure).execute()
JSON table:
{
"kind": "bigquery#table",
"tableReference": {
"projectId": "xxxx",
"tableId": "xxxx",
"datasetId": "xxxx"
},
"type": "table",
"schema": {
"fields": [
{
"mode": "REQUIRED",
"type": "STRING",
"description": "Company",
"name": "COMPANY"
},
{
"mode": "REQUIRED",
"type": "STRING",
"description": "Currency",
"name": "CURRENCY"
}
// bunch of other fields follow...
]
}
}
Why am I receiving this error?
EDIT: Here's the JSON object I'm passing as parameter:
{
"kind": "bigquery#table",
"type": "TABLE",
"tableReference": {
"projectId": "xxxx",
"tableId": "xxxx",
"datasetId": "xxxx"
},
"schema": {
"fields": [
{
"type": "STRING",
"name": "COMPANY"
},
{
"type": "STRING",
"name": "YEAR"
},
{
"type": "STRING",
"name": "COUNTRY_ISO"
},
{
"type": "STRING",
"name": "COUNTRY"
},
{
"type": "STRING",
"name": "COUNTRY_GROUP"
},
{
"type": "STRING",
"name": "REGION"
},
{
"type": "STRING",
"name": "AREA"
},
{
"type": "STRING",
"name": "BU"
},
{
"type": "STRING",
"name": "REFERENCE"
},
{
"type": "FLOAT",
"name": "QUANTITY"
},
{
"type": "FLOAT",
"name": "NET_SALES"
},
{
"type": "FLOAT",
"name": "GROSS_SALES"
},
{
"type": "STRING",
"name": "FAM_GRP"
},
{
"type": "STRING",
"name": "FAMILY"
},
{
"type": "STRING",
"name": "PRESENTATION"
},
{
"type": "STRING",
"name": "ORIG_FAMILY"
},
{
"type": "FLOAT",
"name": "REF_PRICE"
},
{
"type": "STRING",
"name": "CODE1"
},
{
"type": "STRING",
"name": "CODE4"
}
]
}
}
This is probably too late to help you but hopefully it helps the next poor soul like me. It took me a while figure out what "Output field used as input" meant.
Though the API specifies the same object for the request (input) and response (output), some fields are only allowed in the response. In the docs you will see their descriptions prefixed with "Output only". From looking at your table definition I see that you have "type": "TABLE" and "type" is listed as an "Output only" property. So I would gander that if you remove it then that error will go away. Here is the link to the docs: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables
It would help if they told you what field the violation was on.