aws pipeline parameter error - amazon-s3

I have created a pipeline to load data from S3 to RDS mysql instance.I can save the pipeline without any errors but on activation I get the error "No value specified for parameter 1". My online search so far has suggested that the insert statement parameters need to be defined somewhere. If this is correct then how to do so?
The following is the script generated in the process
{
"objects": [
{
"output": {
"ref": "DestinationRDSTable"
},
"input": {
"ref": "S3InputDataLocation"
},
"dependsOn": {
"ref": "RdsMySqlTableCreateActivity"
},
"name": "DataLoadActivity",
"id": "DataLoadActivity",
"runsOn": {
"ref": "Ec2Instance"
},
"type": "CopyActivity"
},
{
"*password": "#{*myRDSPassword}",
"name": "rds_mysql",
"jdbcProperties": "allowMultiQueries=true",
"id": "rds_mysql",
"type": "RdsDatabase",
"rdsInstanceId": "#{myRDSInstanceId}",
"username": "#{myRDSUsername}"
},
{
"instanceType": "t1.micro",
"name": "Ec2Instance",
"actionOnTaskFailure": "terminate",
"securityGroups": "#{myEc2RdsSecurityGrps}",
"id": "Ec2Instance",
"type": "Ec2Resource",
"terminateAfter": "2 Hours"
},
{
"database": {
"ref": "rds_mysql"
},
"name": "RdsMySqlTableCreateActivity",
"runsOn": {
"ref": "Ec2Instance"
},
"id": "RdsMySqlTableCreateActivity",
"type": "SqlActivity",
"script": "#{myRDSTableInsertSql}"
},
{
"database": {
"ref": "rds_mysql"
},
"name": "DestinationRDSTable",
"insertQuery": "#{myRDSTableInsertSql}",
"id": "DestinationRDSTable",
"type": "SqlDataNode",
"table": "#{myRDSTableName}",
"selectQuery": "select * from #{table}"
},
{
"escapeChar": "\\",
"name": "DataFormat1",
"columnSeparator": "|",
"id": "DataFormat1",
"type": "TSV",
"recordSeparator": "\\n"
},
{
"directoryPath": "#{myInputS3Loc}",
"dataFormat": {
"ref": "DataFormat1"
},
"name": "S3InputDataLocation",
"id": "S3InputDataLocation",
"type": "S3DataNode"
},
{
"failureAndRerunMode": "CASCADE",
"resourceRole": "DataPipelineDefaultResourceRole",
"role": "DataPipelineDefaultRole",
"pipelineLogUri": "s3://logs3tords/",
"scheduleType": "ONDEMAND",
"name": "Default",
"id": "Default"
}
],
"parameters": [
{
"description": "RDS MySQL password",
"id": "*myRDSPassword",
"type": "String"
},
{
"watermark": "security group name",
"helpText": "The names of one or more EC2 security groups that have access to the RDS MySQL cluster.",
"description": "RDS MySQL security group(s)",
"isArray": "true",
"optional": "true",
"id": "myEc2RdsSecurityGrps",
"type": "String"
},
{
"description": "RDS MySQL username",
"id": "myRDSUsername",
"type": "String"
},
{
"description": "Input S3 file path",
"id": "myInputS3Loc",
"type": "AWS::S3::ObjectKey"
},
{
"helpText": "The SQL statement to insert data into the RDS MySQL table.",
"watermark": "INSERT INTO #{table} (col1, col2, col3) VALUES(?, ?, ?) ;",
"description": "Insert SQL query",
"id": "myRDSTableInsertSql",
"type": "String"
},
{
"helpText": "The name of an existing table or a new table that will be created based on the create table SQL query parameter below.",
"description": "RDS MySQL table name",
"id": "myRDSTableName",
"type": "String"
},
{
"watermark": "CREATE TABLE pet IF NOT EXISTS (name VARCHAR(20), owner VARCHAR(20), species VARCHAR(20), gender CHAR(1), birth DATE, death DATE);",
"helpText": "The idempotent SQL statement to create the RDS MySQL table if it does not already exist.",
"description": "Create table SQL query",
"optional": "true",
"id": "myRDSCreateTableSql",
"type": "String"
},
{
"watermark": "DB Instance",
"description": "RDS Instance ID",
"id": "myRDSInstanceId",
"type": "String"
}
],
"values": {
"myRDSInstanceId": "instance name",
"myRDSUsername": "user",
"myRDSTableInsertSql": "Insert into Ten.MD_ip_hp (ID, NAME, ADDRESS1, ADDRESS2, CITY, STATE, ZIP, DS ) VALUES(?,?,?,?,?,?,?,?);",
"*myRDSPassword": "password",
"myInputS3Loc": "log location",
"myRDSTableName": "MD_ip_hp"
}
}
UPDATE:
So I specified 'script argument' 1 to 8 on the sql activity node which resulted in my error to change to "No value specified for parameter 2". How to now read each number as a different parameter? >:x

Such a silly thing!
I was able to resolve it by creating separate script argument corresponding to each parameter in my query. In layman words, a script argument for each of the ? in my query.

Related

JSON Schema validation for typos in JSON

How to validate the schema properties for typos, when the property is not required value.
Ex JSON Schema:
{
"$id": "https://example.com/person.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Person",
"type": "object",
"properties": {
"firstName": {
"type": "string",
"description": "The person's first name."
},
"lastName": {
"type": "string",
"description": "The person's last name."
},
"age": {
"description": "Age in years which must be equal to or greater than zero.",
"type": "integer",
"minimum": 0
}
}
}
If the following is the JSON how can we catch the typo for "age" field which has typo as "aged".
{
"firstName": "John",
"lastName": "Doe",
"aged": 21
}
If you add "additionalProperties": false, any properties not declared in properties will be considered an error. In more complex cases, you might need "unevaluatedProperties": false instead, but that's not necessary in this case. The other option is to be explicit about what you fields you allow with "propertyNames": { "enum": ["firstName", "lastName", "aged"] }.

Sql Database Elastic pool and sku combination is invalid

I am able to create Sql Server, Sql database, sql elastic Pool Successfully using ARM templates. But when I trying to create new database with existing elastic pool name. I am getting below error.
Without elastic pool id, database is creating successfully.
Both Sql database Elastic Pool and database are using same location, tier, edition etc.Also When tried in azure portal it created successfully.
"error": {
"code": "ResourceDeploymentFailure",
"message": "The resource operation completed with terminal provisioning state 'Failed'.",
"details": [
{
"code": "ElasticPoolSkuCombinationInvalid",
"message": "Elastic pool 'sqlsamplepool' and sku 'Basic' combination is invalid."
}
]
ARM Template:
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"collation": {
"type": "string",
"metadata": {
"description": "The collation of the database."
},
"defaultValue": "SQL_Latin1_General_CP1_CI_AS"
},
"skutier": {
"type": "string",
"metadata": {
"description": "The edition of the database. The DatabaseEditions enumeration contains all the
valid editions. e.g. Basic, Premium."
},
"allowedValues": [ "Basic", "Standard", "Premium" ],
"defaultValue": "Basic"
},
"resourcelocation": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Location for all resources."
}
},
"sqlservername": {
"type": "string",
"metadata": {
"description": "The name of the sql server."
}
},
"zoneRedundant": {
"type": "bool",
"metadata": {
"description": "Whether or not this database is zone redundant, which means the replicas of this database will be spread across multiple availability zones."
},
"defaultValue": false
},
"sqlElasticPoolName": {
"type": "string",
"metadata": {
"description": "The Elastic Pool name."
}
},
"databaseName": {
"type": "string"
}
},
"functions": [],
"variables": { },
"resources": [
{
"type": "Microsoft.Sql/servers/databases",
"apiVersion": "2020-08-01-preview",
"name": "[concat(parameters('sqlservername'),'/',parameter('databaseName'))]",
"location": "[parameters('resourcelocation')]",
"sku": {
"name": "[parameters('skutier')]",
"tier": "[parameters('skutier')]"
},
"properties": {
"collation": "[parameters('collation')]",
"zoneRedundant": "[parameters('zoneRedundant')]",
"elasticPoolId":"[concat('/subscriptions/',subscription().subscriptionId,'/resourceGroups/',resourceGroup().name,'/providers/Microsoft.Sql/servers/',parameters('sqlservername'),'/elasticPools/',parameters('sqlElasticPoolName'))]"
}
}
]
}
I am not sure what wrong with "2020-08-01-preview" version but its working fine with stable version. below is my partial arm template code that working.
I changed to 2014-04-01 api version.
"comments": "If Elastic Pool Name is defined, then curent database will be added to elastic pool.",
"type": "Microsoft.Sql/servers/databases",
"apiVersion": "2014-04-01",
"name": "[concat(parameters('sqlservername'),'/',variables('dbname'))]",
"location": "[parameters('resourcelocation')]",
"properties": {
"collation": "[parameters('collation')]",
"zoneRedundant": "[parameters('zoneRedundant')]",
"elasticPoolName":"[if(not(empty(parameters('sqlElasticPoolName'))),parameters('sqlElasticPoolName'),'')]",
"edition": "[parameters('skutier')]"
}

Error registering Avro schema: "string" RestClientException: Schema being registered is incompatible with an earlier schema;

I'm trying to send a message to my broker, using Avro schema, but "im always getting error:
2020-02-01 11:24:37.189 [nioEventLoopGroup-4-1] ERROR Application -
Unhandled: POST - /api/orchestration/
org.apache.kafka.common.errors.SerializationException: Error
registering Avro schema: "string" Caused by:
io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException:
Schema being registered is incompatible with an earlier schema; error
code: 409
Here my docker container:
connect:
image: confluentinc/cp-kafka-connect:5.4.0
hostname: confluentinc-connect
container_name: confluentinc-connect
depends_on:
- zookeeper
- broker
- schema-registry
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: confluentinc-connect
CONNECT_CONFIG_STORAGE_TOPIC: confluentinc-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: confluentinc-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: confluentinc-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/extras"
My producer (written in Kolin)
val prop: HashMap<String, Any> = HashMap()
prop[BOOTSTRAP_SERVERS_CONFIG] = bootstrapServers
prop[KEY_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java.name
prop[VALUE_SERIALIZER_CLASS_CONFIG] = KafkaAvroSerializer::class.java.name
prop[SCHEMA_REGISTRY_URL] = schemaUrl
prop[ENABLE_IDEMPOTENCE_CONFIG] = idempotence
prop[ACKS_CONFIG] = acks.value
prop[RETRIES_CONFIG] = retries
prop[MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION] = requestPerConnection
prop[COMPRESSION_TYPE_CONFIG] = compression.value
prop[LINGER_MS_CONFIG] = linger
prop[BATCH_SIZE_CONFIG] = batchSize.value
return KafkaProducer(prop)
My Avro Schema:
{
"type": "record",
"namespace": "com.rjdesenvolvimento",
"name": "create_client_value",
"doc": "Avro Schema for Kafka Command",
"fields": [
{
"name": "id",
"type": "string",
"logicalType": "uuid",
"doc": "UUID for indentifaction command"
},
{
"name": "status",
"type": {
"name": "status",
"type": "enum",
"symbols": [
"Open",
"Closed",
"Processing"
],
"doc": "Can be only: Open, Closed or Processing"
},
"doc": "Status of the command"
},
{
"name": "message",
"type": {
"type": "record",
"name": "message",
"doc": "Avro Schema for insert new client",
"fields": [
{
"name": "id",
"type": "string",
"logicalType": "uuid",
"doc": "UUID for indentifaction client transaction"
},
{
"name": "active",
"type": "boolean",
"doc": "Soft delete for client"
},
{
"name": "name",
"type": "string",
"doc": "Name of the client"
},
{
"name": "email",
"type": "string",
"doc": "Email of the client"
},
{
"name": "document",
"type": "string",
"doc": "CPF or CPNJ of the client"
},
{
"name": "phones",
"doc": "A list of phone numbers",
"type": {
"type": "array",
"items": {
"name": "phones",
"type": "record",
"fields": [
{
"name": "id",
"type": "string",
"logicalType": "uuid",
"doc": "UUID for indentifaction of phone transaction"
},
{
"name": "active",
"type": "boolean",
"doc": "Soft delete for phone number"
},
{
"name": "number",
"type": "string",
"doc": "The phone number with this regex +xx xx xxxx xxxx"
}
]
}
}
},
{
"name": "address",
"type": "string",
"logicalType": "uuid",
"doc": "Adrres is an UUID for a other address-microservice"
}
]
}
}
]
}
And my post:
{
"id" : "9ec818da-6ee0-4634-9ed8-c085248cae12",
"status" : "Open",
"message": {
"id" : "9ec818da-6ee0-4634-9ed8-c085248cae12",
"active" : true,
"name": "name",
"email": "email#com",
"document": "document",
"phones": [
{
"id" : "9ec818da-6ee0-4634-9ed8-c085248cae12",
"active" : true,
"number": "+xx xx xxxx xxxx"
},
{
"id" : "9ec818da-6ee0-4634-9ed8-c085248cae12",
"active" : true,
"number": "+xx xx xxxx xxxx"
}
],
"address": "9ec818da-6ee0-4634-9ed8-c085248cae12"
}
}
What am I doing wrong?
github project: https://github.com/rodrigodevelms/kafka-registry
UPDATE =====
Briefly:
I'm not generating my classes using the Gradle Avro plugin.
In this example, my POST sends an Client object. And in service, it assembles a Command-type object as follows:
id: same client id
status: open
message: the POST that was sent.
So I send this to KAFKA, and in the connect (jdbc sink postgres) I put as fields.whitelist only the attributes of the message (the client) and I don't get either the command id or the status.
on github the only classes that matter to understand the code are:
1
-https://github.com/rodrigodevelms/kafka-registry/blob/master/kafka/src/main/kotlin/com/rjdesenvolvimento/messagebroker/producer/Producer.kt
2 -
https://github.com/rodrigodevelms/kafka-registry/blob/master/kafka/src/main/kotlin/com/rjdesenvolvimento/messagebroker/commnad/Command.kt
3 -
https://github.com/rodrigodevelms/kafka-registry/blob/master/src/client/Controller.kt
4
-https://github.com/rodrigodevelms/kafka-registry/blob/master/src/client/Service.kt
5 - docker-compose.yml, insert-client-value.avsc, postgresql.json,
if i set the compatibility mode of the avro scheme to "none", i can send a message, but some unknown characters will be shown, as shown in the photo below.
I suspect that you're trying to do multiple things and you've not been cleaning up state after previous attempts. You should not get that error in a fresh installation
Schema being registered is incompatible with an earlier schema
Your data has changed in a way that the schema in the registry is not compatible with the one you're sending.
You can send an HTTP DELETE request to http://registry:8081/subjects/[name]/ to delete all versions of the schema, then you can restart your connector

Load Avro file to GCS with nested record using customized column name

I was trying to load an Avro file with nested record. One of the record was having a union of schema. When loaded to BigQuery, it created a very long name like com_mycompany_data_nestedClassname_value on each union element. That name is long. Wondering if there is a way to specify name without having the full package name prefixed.
For example. The following Avro schema
{
"type": "record",
"name": "EventRecording",
"namespace": "com.something.event",
"fields": [
{
"name": "eventName",
"type": "string"
},
{
"name": "eventTime",
"type": "long"
},
{
"name": "userId",
"type": "string"
},
{
"name": "eventDetail",
"type": [
{
"type": "record",
"name": "Network",
"namespace": "com.something.event",
"fields": [
{
"name": "hostName",
"type": "string"
},
{
"name": "ipAddress",
"type": "string"
}
]
},
{
"type": "record",
"name": "DiskIO",
"namespace": "com.something.event",
"fields": [
{
"name": "path",
"type": "string"
},
{
"name": "bytesRead",
"type": "long"
}
]
}
]
}
]
}
Came up with
Is that possible to make the long field name like eventDetail.com_something_event_Network_value to be something like eventDetail.Network
Avro loading is not as flexible as it should be in BigQuery (basic example is that it does not support load a subset of the fields (reader schema). Also, renaming of the columns is not supported today in BigQuery refer here. Only options are recreate your table with the proper names (create a new table from your existing table) or recreate the table from your previous table

How to generate HTML table for test execution summary in Email (Outlook) body

Background:
In our project[selenium-cucumber-java-Mvn based] we have test execution reporting done through- master thoughts Cucumber report (mvn dependency). This Report get generated in the test>target directory and we are sharing this folders Link embed in an email to everyone automatically for reporting. Here recipient has to open this link to see the status. Link looks like-
http://localhost:63342/Automation/TestExecutionReports/02-08-2018/Run-1/cucumber-html-reports/overview-features.html
Question:
Now we have a need to send Execution summary also in the email so that one can see high level status without even drilling through Link.
So we need to insert a Tabular format- with columns -TC Name, Status, Execution Time, Overall Count etc. in email body itself in addition to the above Link.
Can someone plz help me doing this?
Other info:
I don't know much how internally master though works, but it gathers the execution data via a JSON file. And we are generating this report in the onExecutionfinish method of TestNGExecutionListener.
our current report looks like an attached snap.
Our json is contains the following info-
[
{
"line": 1,
"elements": [
{
"before": [
{
"result": {
"duration": 3849355155,
"status": "passed"
},
"match": {
"location": "CucumberHooks.InitBrowser()"
}
}
],
"line": 3,
"name": "Add Account and verify details",
"description": "",
"id": "add-account-and-verify-details;add-account-and-verify-details",
"after": [
{
"result": {
"duration": 129904,
"status": "passed"
},
"match": {
"location": "CucumberHooks.TearDownTest(Scenario)"
}
}
],
"type": "scenario",
"keyword": "Scenario",
"steps": [
{
"result": {
"duration": 13810592966,
"status": "passed"
},
"line": 5,
"name": "I have Logged in to Grid application",
"match": {
"location": "leaseSearchSteps.i_have_Logged_in_to_Grid_application()"
},
"keyword": "Given "
},
{
"result": {
"duration": 4659413417,
"status": "passed"
},
"line": 6,
"name": "I select Accounts and Contacts option from the Homepage",
"match": {
"location": "createAccountSteps.selectAccountsAndContactsOptionFromTheHomepage()"
},
"keyword": "And "
},
{
"result": {
"duration": 3470670155,
"status": "passed"
},
"line": 7,
"name": "Click on Add Account button",
"match": {
"location": "createAccountSteps.clickOnAddAccountButton()"
},
"keyword": "Then "
},
{
"result": {
"duration": 28732919407,
"status": "passed"
},
"line": 8,
"name": "Create Account with Account name as \"Pacific UAT Test Account\"",
"match": {
"arguments": [
{
"val": "Pacific UAT Test Account",
"offset": 37
}
],
"location": "createAccountSteps.createAccountWithAccountName(String)"
},
"keyword": "Then "
},
{
"result": {
"duration": 2243415294,
"status": "passed"
},
"line": 9,
"name": "Click on View button",
"match": {
"location": "createAccountSteps.clickOnViewButton()"
},
"keyword": "Then "
},
{
"result": {
"duration": 121318771,
"status": "passed"
},
"line": 10,
"name": "Verify the newly created Account name",
"match": {
"location": "createAccountSteps.verifyTheNewlyCreatedAccountName()"
},
"keyword": "Then "
}
],
"tags": [
{
"line": 2,
"name": "#Test"
}
]
}
],
"name": "Add Account and verify details",
"description": "",
"id": "add-account-and-verify-details",
"keyword": "Feature",
"uri": "src/test/resources/cucumber/featureFiles/AU/addAccountAndVerify.feature"
}
]
I dont have code but I think a probable approach can be like below-
First write some code to generate Table Template with its Header, Main Table, Rows and Columns and Footer.
Then insert the Data into this table by reading it from the features.html page you have (as mentioned in url ). You should be able to read data for pass, fail or whatever needed from this html web page through standard methods [like- getText()] of Selenium .
In the end get this whole table appended to a StringBuilder and send in email message as bu using setting content of thmlSourceEmbed as text/html..hope this help