Related
enter image description hereI am trying to create a JSON schema which is expecting inputs from 3 sources (file/Table/Kafka message), at a time any one of them will get the data from external source.
Issue :
I am using "anyOf" keyword in JSON but when I am applying "anyOf" and testing the negative scenario that is removing one of required field from the input properties, it's not giving any validation error, which ideally should have given that required field is missing.
**Expectation: **
Need to get input from any one of the source (file/table/message). If external party sends data in the file format, file portion from JSON schema should get executed, similarly if gets data from table or kafka message then relevant portion from JSON schema should get executed.
JSON SCHEMA CODE:
{
"$schema": "https://json-schema.org/draft/2019-09/schema",
"type": "object",
"title": "Job",
"properties": {
"unique_name": {
"type": "string",
"format": "regex",
"pattern": "[a-zA-Z0-9_]+"
},
"processing_instructions_file": {
"type": "string",
"examples": [
"/path/to/some/file/processing123.yaml"
]
},
"variables": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"environment_prefix"
]
},
"value": {
"type": "string",
"examples": [
"DEV"
]
}
},
"additionalProperties": true,
"required": [
"name",
"value"
]
}
},
"inputs": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"is_stream": {
"type": "boolean",
"examples": [
true
],
"default": false
}
},
"additionalProperties": true,
"anyOf": [
{
"type": "object",
"properties": {
"file": {
"type": "object",
"properties": {
"folder_structure": {
"type": "string",
"examples": [
"$environment_prefix\\$$yyyy-MMM-dd"
]
},
"naming_convention": {
"type": "string",
"examples": [
"STATIC_FILE_NAME:$$yyyy_MMM-dd"
]
},
"format type": {
"type": "string",
"enum": [
"CSV",
"EBCDIC"
],
"examples": [
"CSV"
]
},
"format_options": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"option_name": {
"type": "string",
"examples": [
"generate_record_id"
]
},
"option_value": {
"type": [
"string",
"boolean"
],
"examples": [
true
]
}
},
"additionalProperties": true,
"required": [
"option_name",
"option_value"
]
}
}
},
"additionalProperties": true,
"if": {
"properties": {
"format type": {
"type": "string",
"enum": [
"CSV",
"EBCDIC"
],
"examples": [
"CSV"
],
"const": "EBCDIC"
}
},
"required": [
"format type"
]
},
"then": {
"properties": {
"schema_location": {
"type": "string",
"examples": [
"some_path"
]
}
},
"required": [
"schema_location"
]
},
"else": {
"properties": {
"schema_location": {
"type": "string",
"examples": [
"some_path"
]
}
}
},
"required": [
"folder_structure",
"naming_convention",
"format type",
"format_options"
]
}
},
"additionalProperties": true
},
{
"type": "object",
"properties": {
"table": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"tbl_employees"
]
},
"jdbc_url": {
"type": "string",
"examples": [
"orcl#thinXXX"
]
},
"jdbc_library_jar": {
"type": "string",
"examples": [
"orcl.jar"
]
},
"schema": {
"type": "string",
"examples": [
"xxxxx"
]
},
"info_date_from_column_name": {
"type": "string",
"examples": [
"ASOF_DATE"
]
},
"info_date_to_column_name": {
"type": "string",
"examples": [
"END_DATE"
]
},
"primary_key_column_name": {
"type": "array",
"additionalItems": true,
"items": {
"type": "string"
}
}
},
"additionalProperties": true,
"required": [
"name",
"jdbc_url",
"schema"
]
}
},
"additionalProperties": true
},
{
"type": "object",
"properties": {
"message": {
"type": "object",
"properties": {
"topic_name": {
"type": "string",
"examples": [
"XXXXXX"
]
},
"group_id": {
"type": "string",
"examples": [
"XXXXX"
]
},
"port": {
"type": "integer",
"examples": [
8081
]
},
"kafka_cluster_ip": {
"type": "string",
"examples": [
"1.1.1.1"
]
},
"kafka_cluster_schema_registry": {
"type": "string",
"examples": [
"1.1.1.1"
]
}
},
"additionalProperties": true,
"required": [
"topic_name",
"group_id",
"port",
"kafka_cluster_ip",
"kafka_cluster_schema_registry"
]
}
},
"additionalProperties": true
}
]
}
},
"outputs": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"output_3"
]
},
"is_stream": {
"type": "boolean",
"examples": [
true
]
},
"spark_save_mode": {
"type": "string",
"examples": [
"append"
]
},
"file": {
"type": "object",
"properties": {
"root_path": {
"type": "string",
"examples": [
"some_path"
]
},
"folder_structure": {
"type": "string",
"examples": [
"$environment_prefix\\$$yyyy-MMM-dd"
]
},
"naming_convention": {
"type": "string",
"examples": [
"STATIC_FILE_NAME:$$yyyy_MMM-dd"
]
},
"partition_by": {
"type": "array",
"additionalItems": true,
"items": {
"type": "string"
}
},
"format": {
"type": "object",
"properties": {
"format_type": {
"type": "string",
"examples": [
"CSV"
]
},
"format_options": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"option_name": {
"type": "string",
"examples": [
"include_header"
]
},
"option_value": {
"type": [
"string",
"boolean"
],
"examples": [
true
]
}
},
"additionalProperties": true,
"required": [
"option_name",
"option_value"
]
}
}
},
"additionalProperties": true,
"required": [
"format_type",
"format_options"
]
}
},
"additionalProperties": true,
"required": [
"root_path",
"folder_structure",
"naming_convention",
"partition_by",
"format"
]
},
"mongo": {
"type": "object",
"properties": {
"mongo_spark_connector_options": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"option_name": {
"type": "string",
"examples": [
"format"
]
},
"option_value": {
"type": "string"
}
},
"additionalProperties": true,
"required": [
"option_name",
"option_value"
]
}
}
},
"additionalProperties": true,
"required": [
"mongo_spark_connector_options"
]
},
"table": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"tbl_employees"
]
},
"jdbc_url": {
"type": "string",
"examples": [
"orcl#thin:XXX"
]
},
"jdbc_library_jar": {
"type": "string",
"examples": [
"orcl.jar"
]
},
"schema": {
"type": "string",
"examples": [
"xxxxx"
]
},
"info_date_from_column_name": {
"type": "string",
"examples": [
"ASOF_DATE"
]
},
"info_date_to_column_name": {
"type": "string",
"examples": [
"END_DATE"
]
},
"primary_key_column_name": {
"type": "array",
"additionalItems": true,
"items": {
"type": "string"
}
}
},
"additionalProperties": true,
"required": [
"name",
"jdbc_url",
"jdbc_library_jar",
"schema",
"info_date_from_column_name",
"info_date_to_column_name",
"primary_key_column_name"
]
},
"message": {
"type": "object",
"properties": {
"topic_name": {
"type": "string",
"examples": [
"XXXXXX"
]
},
"group_id": {
"type": "string",
"examples": [
"XXXXX"
]
},
"port": {
"type": "integer",
"examples": [
8081
]
},
"kafka_cluster_ip": {
"type": "string",
"examples": [
"1.1.1.1"
]
},
"kafka_cluster_schema_registry": {
"type": "string",
"examples": [
"1.1.1.1"
]
}
},
"additionalProperties": true,
"required": [
"topic_name",
"group_id",
"port",
"kafka_cluster_ip",
"kafka_cluster_schema_registry"
]
}
},
"additionalProperties": true,
"required": [
"name",
"is_stream"
]
}
}
},
"additionalProperties": true,
"required": [
"unique_name",
"processing_instructions_file",
"inputs",
"outputs"
]
}
JSON DATA
{
"unique_name": "aXeTGImcM6PrCalRpGjLigbj1puXXvK",
"processing_instructions_file": "/path/to/some/file/processing123.yaml",
"variables": [
{
"name": "environment_prefix",
"value": "DEV"
}
],
"inputs": [
{
"is_stream": false,
"file": {
"folder_structure": "$environment_prefix\\$$yyyy-MMM-dd",
"naming_convention": "STATIC_FILE_NAME:$$yyyy_MMM-dd",
"format type": "CSV",
"format_options": [
{
"option_name": "generate_record_id",
"option_value": "Lorem"
}
],
"schema_location": "some_path"
}
}
],
"outputs": [
{
"name": "output_3",
"is_stream": true,
"spark_save_mode": "append",
"file": {
"root_path": "some_path",
"folder_structure": "$environment_prefix\\$$yyyy-MMM-dd",
"naming_convention": "STATIC_FILE_NAME:$$yyyy_MMM-dd",
"partition_by": [
"Lorem"
],
"format": {
"format_type": "CSV",
"format_options": [
{
"option_name": "include_header",
"option_value": "Lorem"
}
]
}
},
"mongo": {
"mongo_spark_connector_options": [
{
"option_name": "format",
"option_value": "Lorem"
}
]
},
"table": {
"name": "tbl_employees",
"jdbc_url": "orcl#thin:XXX",
"jdbc_library_jar": "orcl.jar",
"schema": "xxxxx",
"info_date_from_column_name": "ASOF_DATE",
"info_date_to_column_name": "END_DATE",
"primary_key_column_name": [
"Lorem"
]
},
"message": {
"topic_name": "XXXXXX",
"group_id": "XXXXX",
"port": 8081,
"kafka_cluster_ip": "1.1.1.1",
"kafka_cluster_schema_registry": "1.1.1.1"
}
}
]
}
cant build validation for simple case:
if sources field contains "OTHER" in values then "sourceOtherDescription" must be required.
Shall pass validation
{
"sources": ["RENTS"]
}
{
"sources": ["RENTS", "OTHER"],
"sourceOtherDescription": "other income"
}
This should not pass validation since sources contains "OTHER"
{
"sources": ["RENTS", "OTHER"]
}
The schema that I was able to produce. Does not really work
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "money-sources",
"title": "Money Sources",
"description": "Money Sources definitions",
"type": "object",
"required": ["sources"],
"properties": {
"sources": {
"type": "array",
"items": {
"type": "string",
"enum": [
"RENTS",
"MEMBER_FEES",
"PROFIT",
"SALES_SECURITIES",
"INTERNAL_GROUP_TRANSFERS",
"OTHER"
]
},
"uniqueItems": true
},
"sourceOtherDescription": { "type": "string", "minLength": 3}
},
"additionalProperties": false,
"oneOf": [
{
"properties": {
"sources": {
"type": "array",
"contains": {"const": "OTHER"}
}
},
"required": ["sourceOtherDescription"]
},
{
"properties": {
"sources": {
"type": "array",
"contains": {
"enum": [
"RENTS",
"MEMBER_FEES",
"PROFIT",
"SALES_SECURITIES"
]
}
}
}
}
, false
]
}
Using if-then it works for me this way:
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "money-sources",
"title": "Money Sources",
"description": "Money Sources definitions",
"type": "object",
"required": [ "sources" ],
"properties": {
"sources": {
"type": "array",
"items": {
"type": "string",
"enum": [
"RENTS",
"MEMBER_FEES",
"PROFIT",
"SALES_SECURITIES",
"INTERNAL_GROUP_TRANSFERS",
"OTHER"
]
},
"uniqueItems": true
},
"sourceOtherDescription": {
"type": "string",
"minLength": 3
}
},
"additionalProperties": false,
"if": {
"properties": {
"sources": {
"type": "array",
"contains": {
"const": "OTHER"
}
}
}
},
"then": {
"required": [ "sourceOtherDescription" ]
}
}
I have an API returning response based on input type.
For a single filter I get
{
"intervalData": [
{
"timestamp": "2021-05-25T06:00:00.000Z",
"result": {
"Attempts": 309194
}
}
],
"intervalTotals": {
"Attempts": 4719471
}
}
For multiple filter I get
{
"multiFilterData": [
{
"filter": {
"id": 111111
},
"intervalData": {
"timestamp": "2021-05-25T20:41:39.000Z",
"result": {
"Attempts": 7902
}
}
},
{...}
]
}
That means the response will either have (intervalData,intervalTotals) OR multiFilterData.
Question:
Is it possible to have a generic schema to handle this or must I have two separate schemas?
How to make sure that if one combination is present other should not be there?
I tried with below schema on https://www.jsonschemavalidator.net/. The validation fails if there is some other keys listed under the required, unfortunately the validation succeeds if one of the entry from oneOf is missing. EDIT1 below
{
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"oneOf": [
{
"required": [
"intervalData",
"intervalTotals",
]
},
{
"required": [
"multiFilterData",
]
}
],
"properties": {
"intervalData": {
"type": "array",
"additionalItems": false,
"items": {
"type": "object",
"required": [
"timestamp",
"result"
],
"properties": {
...
},
"additionalProperties": false
}
},
"intervalTotals": {
"type": "object",
"required": [
"Attempts"
],
"properties": {
"Attempts": {
"type": "integer"
}
},
"additionalProperties": false
},
"multiFilterData": {
"type": "array",
"additionalItems": false,
"items": {
"type": "object",
"required": [
"filter",
"intervalData"
],
"properties": {
"filter": {
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"type": "integer"
}
},
"additionalProperties": false
},
"intervalData": {
"type": "object",
"required": [
"timestamp",
"result"
],
"properties": {
"timestamp": {
"type": "string"
},
"result": {
"type": "object",
"required": [
"Attempts"
],
"properties": {
...
},
"additionalProperties": false
}
},
"additionalProperties": false
}
},
"additionalProperties": false
}
}
},
"additionalProperties": false
}
EDIT 1: This schema also passes even though the first required is missing an element on the list.
{
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"oneOf": [
{
"required": [
"intervalData"
]
},
{
"required": [
"multiFilterData",
]
}
],
"properties": {
"intervalData": {...},
"intervalTotals": {...},
"multiFilterData": {...}
},
"additionalProperties": false
}
Depending on the salaryRange the user selects I need to validate differently by requiring some fields and rejecting others. I feel like its a combination of allOf and not but I can't seem to quite get it.
Scenario #1
User selects salaryRange(Hourly)
Require hourlyRate
Prevent the submission of fields feeOne and feeTwo
Scenario #2
User selects salaryRange(0-50k OR 50-100k)
Require feeOne and feeTwo
Prevent the submission of field hourlyRate
Here is my schema
{
"schema": "http://json-schema.org/draft-04/schema#",
"$id": "http://mysite/schemas/job.json#",
"title": "Job",
"description": "Create job",
"type": "object",
"properties": {
"title": { "type": "string" },
"description": { "type": "string" },
"salaryRange": { "enum": ["0-50k", "50-100k", "100-150k", "150-200k", "200-300k", "300k+", "nonExempt", "Hourly"] },
"hourlyRate": {
"type": "number",
"minimum": 0,
"maximum": 300
},
"feeOne": {
"type": "number",
"minimum": 0
},
"feeTwo": {
"type": "number",
"minimum": 0
}
} ,
"additionalProperties": false,
"required": [
"title",
"description",
"salaryRange"
]
}
You can use oneOf and not required to model all possible combinations.
Here is an example in js:
https://runkit.com/embed/cf8cra1mwvx3/
{
"schema": "http://json-schema.org/draft-04/schema#",
"$id": "http://mysite/schemas/job.json#",
"title": "Job",
"description": "Create job",
"type": "object",
"properties": {
"title": { "type": "string" },
"description": { "type": "string" },
"salaryRange": { "enum": ["0-50k", "50-100k", "100-150k", "150-200k", "200-300k", "300k+", "nonExempt", "Hourly"] },
"hourlyRate": {
"type": "number",
"minimum": 0,
"maximum": 300
},
"feeOne": {
"type": "number",
"minimum": 0
},
"feeTwo": {
"type": "number",
"minimum": 0
}
},
"oneOf": [
{
"description": "Disallow fees for hourly salary",
"properties": {
"salaryRange": { "enum": ["Hourly"] }
},
"required": ["hourlyRate"],
"allOf": [
{"not":{"required":["feeOne"]}},
{"not":{"required":["feeTwo"]}}
]
},
{
"description": "Disallow hourly rate for 0-50k, 50-100k salaries",
"properties": {
"salaryRange": { "enum": ["0-50k", "50-100k"] }
},
"required": ["feeOne", "feeTwo"],
"not":{"required":["hourlyRate"]}
},
{
"description": "Allow other cases",
"properties": {
"salaryRange": { "not" : {"enum": ["Hourly", "0-50k", "50-100k"] } }
}
}
],
"additionalProperties": false,
"required": [
"title",
"description",
"salaryRange"
]
}
I'm helping to build an interface that works with Json Schema, and I have a question about interface generation based on that schema. There are two display types - one for internal users and one for external users. Both are dealing with the same data, but the external users should see a smaller subset of fields than the internal users.
For example, here is one schema, it defines an obituary:
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "",
"type": "object",
"required": [
"id",
"deceased"
],
"properties": {
"id": { "type": "string" },
"account": {
"type": "object",
"required": [
"name"
],
"properties": {
"id": { "type": "number" },
"name": { "type": "string" },
"website": {
"anyOf": [
{
"type": "string",
"format": "uri"
},
{
"type": "string",
"maxLength": 0
}
]
},
"email": {
"anyOf": [
{
"type": "string",
"format": "email"
},
{
"type": "string",
"maxLength": 0
}
]
},
"address": {
"type": "object",
"properties": {
"address1": { "type": "string" },
"address2": { "type": "string" },
"city": { "type": "string" },
"state": { "type": "string" },
"postalCode": { "type": "string" },
"country": { "type": "string" }
}
},
"phoneNumber": {
"anyOf": [
{
"type": "string",
"format": "phone"
},
{
"type": "string",
"maxLength": 0
}
]
},
"faxNumber": {
"anyOf": [
{
"type": "string",
"format": "phone"
},
{
"type": "string",
"maxLength": 0
}
]
},
"type": { "type": "string" }
}
},
"deceased": {
"type": "object",
"required": [
"fullName"
],
"properties": {
"fullName": { "type": "string" },
"prefix": { "type": "string" },
"firstName": { "type": "string" },
"middleName": { "type": "string" },
"nickName": { "type": "string" },
"lastName1": { "type": "string" },
"lastName2": { "type": "string" },
"maidenName": { "type": "string" },
"suffix": { "type": "string" }
}
},
"description": { "type": "string" },
"photos": {
"type": "array",
"items": { "type": "string" }
}
}
}
Internal users would be able to access all the fields, but external users shouldn't be able to read/write the account fields.
Should I make a second schema for the external users, or is there a way to indicate different display levels or public/private on each field?
You cannot restrict acess to the fields defined in a schema, but you can have 2 schema files, one defining the "public" fields, and the other one defining the restricted fields plus including the restricted fields.
So
public-schema.json:
{
"properties" : {
"id" : ...
}
}
restricted-schema.json:
{
"allOf" : [
{
"$ref" : "./public-schema.json"
},
{
"properties" : {
"account": ...
}
}
]
}