documents not auto indexing elasticsearch - indexing

I have created an elasticsearch index. It searches everything correctly but when any new document is added it does not index it automatically. Everytime I have to restart the service for it to index the new document.
PUT /_river/mydocum1/_meta
{
"type": "fs",
"fs": {
"url": "\\\\file\\datum\\Depart\\Inet\\",
"update_rate": 3600,
"includes": [
"*.docx",
"*.xlsx",
"*.pdf",
"*.txt",
"*.doc",
"*.xls"
]
},
"index": {
"index": "test",
"type": "docum",
"bulk_size": 500
}
}
EDIT:
Here are the settings
{
"test": {
"settings": {
"index": {
"uuid": "9rofjAZySByrtehnjemkjudg",
"analysis": {
"filter": {
"synonym_filter": {
"type": "synonym",
"synonyms_path": "analysis/main_synonym.txt"
},
"nGram_filter": {
"max_gram": "20",
"min_gram": "2",
"type": "nGram",
"token_chars": [
"letter",
"digit",
"punctuation",
"symbol"
]
}
},
"analyzer": {
"search_analyzer_2": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding",
"synonym_filter"
],
"tokenizer": "standard"
},
"index_analyzer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding",
"nGram_filter"
],
"tokenizer": "whitespace"
},
"search_analyzer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding",
"synonym_filter"
],
"tokenizer": "whitespace"
},
"index_analyzer_2": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding"
],
"tokenizer": "whitespace"
},
"index_analyzer_3": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding"
],
"tokenizer": "standard"
},
"nGram_analyzer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding",
"nGram_filter"
],
"tokenizer": "whitespace"
},
"index_analyzer_4": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding"
],
"tokenizer": "keyword"
},
"whitespace_analyzer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding"
],
"tokenizer": "whitespace"
}
}
}
}
}
Does anyone know how do I solve it?

Related

Validation issue on JSON schema #jsonschema

enter image description hereI am trying to create a JSON schema which is expecting inputs from 3 sources (file/Table/Kafka message), at a time any one of them will get the data from external source.
Issue :
I am using "anyOf" keyword in JSON but when I am applying "anyOf" and testing the negative scenario that is removing one of required field from the input properties, it's not giving any validation error, which ideally should have given that required field is missing.
**Expectation: **
Need to get input from any one of the source (file/table/message). If external party sends data in the file format, file portion from JSON schema should get executed, similarly if gets data from table or kafka message then relevant portion from JSON schema should get executed.
JSON SCHEMA CODE:
{
"$schema": "https://json-schema.org/draft/2019-09/schema",
"type": "object",
"title": "Job",
"properties": {
"unique_name": {
"type": "string",
"format": "regex",
"pattern": "[a-zA-Z0-9_]+"
},
"processing_instructions_file": {
"type": "string",
"examples": [
"/path/to/some/file/processing123.yaml"
]
},
"variables": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"environment_prefix"
]
},
"value": {
"type": "string",
"examples": [
"DEV"
]
}
},
"additionalProperties": true,
"required": [
"name",
"value"
]
}
},
"inputs": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"is_stream": {
"type": "boolean",
"examples": [
true
],
"default": false
}
},
"additionalProperties": true,
"anyOf": [
{
"type": "object",
"properties": {
"file": {
"type": "object",
"properties": {
"folder_structure": {
"type": "string",
"examples": [
"$environment_prefix\\$$yyyy-MMM-dd"
]
},
"naming_convention": {
"type": "string",
"examples": [
"STATIC_FILE_NAME:$$yyyy_MMM-dd"
]
},
"format type": {
"type": "string",
"enum": [
"CSV",
"EBCDIC"
],
"examples": [
"CSV"
]
},
"format_options": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"option_name": {
"type": "string",
"examples": [
"generate_record_id"
]
},
"option_value": {
"type": [
"string",
"boolean"
],
"examples": [
true
]
}
},
"additionalProperties": true,
"required": [
"option_name",
"option_value"
]
}
}
},
"additionalProperties": true,
"if": {
"properties": {
"format type": {
"type": "string",
"enum": [
"CSV",
"EBCDIC"
],
"examples": [
"CSV"
],
"const": "EBCDIC"
}
},
"required": [
"format type"
]
},
"then": {
"properties": {
"schema_location": {
"type": "string",
"examples": [
"some_path"
]
}
},
"required": [
"schema_location"
]
},
"else": {
"properties": {
"schema_location": {
"type": "string",
"examples": [
"some_path"
]
}
}
},
"required": [
"folder_structure",
"naming_convention",
"format type",
"format_options"
]
}
},
"additionalProperties": true
},
{
"type": "object",
"properties": {
"table": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"tbl_employees"
]
},
"jdbc_url": {
"type": "string",
"examples": [
"orcl#thinXXX"
]
},
"jdbc_library_jar": {
"type": "string",
"examples": [
"orcl.jar"
]
},
"schema": {
"type": "string",
"examples": [
"xxxxx"
]
},
"info_date_from_column_name": {
"type": "string",
"examples": [
"ASOF_DATE"
]
},
"info_date_to_column_name": {
"type": "string",
"examples": [
"END_DATE"
]
},
"primary_key_column_name": {
"type": "array",
"additionalItems": true,
"items": {
"type": "string"
}
}
},
"additionalProperties": true,
"required": [
"name",
"jdbc_url",
"schema"
]
}
},
"additionalProperties": true
},
{
"type": "object",
"properties": {
"message": {
"type": "object",
"properties": {
"topic_name": {
"type": "string",
"examples": [
"XXXXXX"
]
},
"group_id": {
"type": "string",
"examples": [
"XXXXX"
]
},
"port": {
"type": "integer",
"examples": [
8081
]
},
"kafka_cluster_ip": {
"type": "string",
"examples": [
"1.1.1.1"
]
},
"kafka_cluster_schema_registry": {
"type": "string",
"examples": [
"1.1.1.1"
]
}
},
"additionalProperties": true,
"required": [
"topic_name",
"group_id",
"port",
"kafka_cluster_ip",
"kafka_cluster_schema_registry"
]
}
},
"additionalProperties": true
}
]
}
},
"outputs": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"output_3"
]
},
"is_stream": {
"type": "boolean",
"examples": [
true
]
},
"spark_save_mode": {
"type": "string",
"examples": [
"append"
]
},
"file": {
"type": "object",
"properties": {
"root_path": {
"type": "string",
"examples": [
"some_path"
]
},
"folder_structure": {
"type": "string",
"examples": [
"$environment_prefix\\$$yyyy-MMM-dd"
]
},
"naming_convention": {
"type": "string",
"examples": [
"STATIC_FILE_NAME:$$yyyy_MMM-dd"
]
},
"partition_by": {
"type": "array",
"additionalItems": true,
"items": {
"type": "string"
}
},
"format": {
"type": "object",
"properties": {
"format_type": {
"type": "string",
"examples": [
"CSV"
]
},
"format_options": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"option_name": {
"type": "string",
"examples": [
"include_header"
]
},
"option_value": {
"type": [
"string",
"boolean"
],
"examples": [
true
]
}
},
"additionalProperties": true,
"required": [
"option_name",
"option_value"
]
}
}
},
"additionalProperties": true,
"required": [
"format_type",
"format_options"
]
}
},
"additionalProperties": true,
"required": [
"root_path",
"folder_structure",
"naming_convention",
"partition_by",
"format"
]
},
"mongo": {
"type": "object",
"properties": {
"mongo_spark_connector_options": {
"type": "array",
"additionalItems": true,
"items": {
"type": "object",
"properties": {
"option_name": {
"type": "string",
"examples": [
"format"
]
},
"option_value": {
"type": "string"
}
},
"additionalProperties": true,
"required": [
"option_name",
"option_value"
]
}
}
},
"additionalProperties": true,
"required": [
"mongo_spark_connector_options"
]
},
"table": {
"type": "object",
"properties": {
"name": {
"type": "string",
"examples": [
"tbl_employees"
]
},
"jdbc_url": {
"type": "string",
"examples": [
"orcl#thin:XXX"
]
},
"jdbc_library_jar": {
"type": "string",
"examples": [
"orcl.jar"
]
},
"schema": {
"type": "string",
"examples": [
"xxxxx"
]
},
"info_date_from_column_name": {
"type": "string",
"examples": [
"ASOF_DATE"
]
},
"info_date_to_column_name": {
"type": "string",
"examples": [
"END_DATE"
]
},
"primary_key_column_name": {
"type": "array",
"additionalItems": true,
"items": {
"type": "string"
}
}
},
"additionalProperties": true,
"required": [
"name",
"jdbc_url",
"jdbc_library_jar",
"schema",
"info_date_from_column_name",
"info_date_to_column_name",
"primary_key_column_name"
]
},
"message": {
"type": "object",
"properties": {
"topic_name": {
"type": "string",
"examples": [
"XXXXXX"
]
},
"group_id": {
"type": "string",
"examples": [
"XXXXX"
]
},
"port": {
"type": "integer",
"examples": [
8081
]
},
"kafka_cluster_ip": {
"type": "string",
"examples": [
"1.1.1.1"
]
},
"kafka_cluster_schema_registry": {
"type": "string",
"examples": [
"1.1.1.1"
]
}
},
"additionalProperties": true,
"required": [
"topic_name",
"group_id",
"port",
"kafka_cluster_ip",
"kafka_cluster_schema_registry"
]
}
},
"additionalProperties": true,
"required": [
"name",
"is_stream"
]
}
}
},
"additionalProperties": true,
"required": [
"unique_name",
"processing_instructions_file",
"inputs",
"outputs"
]
}
JSON DATA
{
"unique_name": "aXeTGImcM6PrCalRpGjLigbj1puXXvK",
"processing_instructions_file": "/path/to/some/file/processing123.yaml",
"variables": [
{
"name": "environment_prefix",
"value": "DEV"
}
],
"inputs": [
{
"is_stream": false,
"file": {
"folder_structure": "$environment_prefix\\$$yyyy-MMM-dd",
"naming_convention": "STATIC_FILE_NAME:$$yyyy_MMM-dd",
"format type": "CSV",
"format_options": [
{
"option_name": "generate_record_id",
"option_value": "Lorem"
}
],
"schema_location": "some_path"
}
}
],
"outputs": [
{
"name": "output_3",
"is_stream": true,
"spark_save_mode": "append",
"file": {
"root_path": "some_path",
"folder_structure": "$environment_prefix\\$$yyyy-MMM-dd",
"naming_convention": "STATIC_FILE_NAME:$$yyyy_MMM-dd",
"partition_by": [
"Lorem"
],
"format": {
"format_type": "CSV",
"format_options": [
{
"option_name": "include_header",
"option_value": "Lorem"
}
]
}
},
"mongo": {
"mongo_spark_connector_options": [
{
"option_name": "format",
"option_value": "Lorem"
}
]
},
"table": {
"name": "tbl_employees",
"jdbc_url": "orcl#thin:XXX",
"jdbc_library_jar": "orcl.jar",
"schema": "xxxxx",
"info_date_from_column_name": "ASOF_DATE",
"info_date_to_column_name": "END_DATE",
"primary_key_column_name": [
"Lorem"
]
},
"message": {
"topic_name": "XXXXXX",
"group_id": "XXXXX",
"port": 8081,
"kafka_cluster_ip": "1.1.1.1",
"kafka_cluster_schema_registry": "1.1.1.1"
}
}
]
}

Create Delivery Plan styling rules using Azure Devops REST Apis

I am trying to create Delivery plan in the Azure Devops project using Azure Devops REST Apis. I have used following method to create the same.
https://learn.microsoft.com/en-us/rest/api/azure/devops/work/plans/create?view=azure-devops-rest-6.0
POST https://dev.azure.com/{organization}/{project}/_apis/work/plans?api-version=6.0
and I am sending following data in the request body properties
{
"properties": {
"teamBacklogMappings": [
{
"teamId": "09d57738-697f-4433-abdd-b80a2bc6337b",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "5df45eec-4108-474a-8d93-bc09c0b9037e",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "e8ed402b-68e7-4140-96f2-07790a08788b",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "14425694-efa5-454e-811c-e9e03d79198f",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "b02690e9-1f48-421a-a918-3b23cc9a1b73",
"categoryReferenceName": "Microsoft.RequirementCategory"
}
],
"cardSettings": {
"fields": {
"showId": true,
"showAssignedTo": true,
"assignedToDisplayFormat": "avatarOnly",
"showState": true,
"showTags": true,
"showParent": false,
"showEmptyFields": true,
"showChildRollup": true,
"additionalFields": null,
"coreFields": [
{
"referenceName": "System.AssignedTo",
"displayName": "Assigned To",
"fieldType": "string",
"isIdentity": true
},
{
"referenceName": "System.Id",
"displayName": "ID",
"fieldType": "integer",
"isIdentity": false
},
{
"referenceName": "System.State",
"displayName": "State",
"fieldType": "string",
"isIdentity": false
},
{
"referenceName": "System.Tags",
"displayName": "Tags",
"fieldType": "plainText",
"isIdentity": false
}
]
}
},
"markers": [],
"styleSettings": [
{
"name": "BLOCKER",
"isEnabled": "True",
"filter": "[System.Tags] CONTAINS 'BLOCKER'",
"clauses": [
{
"fieldName": "System.Tags",
"logicalOperator": "AND",
"operator": "CONTAINS",
"value": "BLOCKER"
}
],
"settings": {
"background-color": "#E60017",
"title-color": "#000000"
}
},
{
"name": "New",
"isEnabled": "True",
"filter": "[System.State] = 'New'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "New"
}
],
"settings": {
"background-color": "#AAAAAA",
"title-color": "#000000"
}
},
{
"name": "Dev Completed",
"isEnabled": "True",
"filter": "[System.State] = 'Development Completed'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Development Completed"
}
],
"settings": {
"background-color": "#D7E587",
"title-color": "#000000"
}
},
{
"name": "Deployed to QA",
"isEnabled": "True",
"filter": "[System.State] = 'Deployed to QA (SIT)'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Deployed to QA (SIT)"
}
],
"settings": {
"background-color": "#C3D84C",
"title-color": "#000000"
}
},
{
"name": "Deployed to UAT",
"isEnabled": "True",
"filter": "[System.State] = 'Deployed to UAT'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Deployed to UAT"
}
],
"settings": {
"background-color": "#60AF49",
"title-color": "#000000"
}
},
{
"name": "Deployed to PROD",
"isEnabled": "True",
"filter": "[System.State] = 'Completed'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Completed"
}
],
"settings": {
"background-color": "#00643A",
"title-color": "#000000"
}
}
],
"tagStyleSettings": []
}
}
However Styling rules are not getting created in the project.
Got this done by using Update for the Delivery plan immediately after creating the same. Update it with same properties though you will have to add revision property to the request body.
https://learn.microsoft.com/en-us/rest/api/azure/devops/work/plans/update?view=azure-devops-rest-6.0
https://dev.azure.com/{organization}/{project}/_apis/work/plans/{id}?api-version=6.0
Create Delivery Plan doesn't accept style settings, from the UI you can notice this:
Just get the plan id and the response from the Create API and then use them in the Update API, this is the only way.

Transferring ownership of NFT(ERC721)

I'm trying to change the ownership of a requested ERC721(NFT) to another wallet.
I'm using the Mumbai Test Net and ethers.
To do it I need to create the general ERC721 contract using its abi but I can't find it.
In comparison, the ERC20's abi I found easily - link and used it like followed and it's working:
const createERC20Contract = (contractAddress) => {
const provider = new ethers.providers.Web3Provider(ethereum);
const signer = provider.getSigner();
const ERC20Contract = new ethers.Contract(contractAddress, ERC20ABI, signer);
return ERC20Contract;
}
Where can I find the abi of ERC721 to create the ethers.Contract() ?
Here's a generic ERC721 ABI that I built from the OpenZeppelin implementation. (The bare minimum contract - none of their extensions such as mint and burn tokens.)
[
{
"inputs": [
{
"internalType": "string",
"name": "name_",
"type": "string"
},
{
"internalType": "string",
"name": "symbol_",
"type": "string"
}
],
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "approved",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "Approval",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"indexed": false,
"internalType": "bool",
"name": "approved",
"type": "bool"
}
],
"name": "ApprovalForAll",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "from",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "to",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "Transfer",
"type": "event"
},
{
"inputs": [
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "approve",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "owner",
"type": "address"
}
],
"name": "balanceOf",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "getApproved",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"internalType": "address",
"name": "operator",
"type": "address"
}
],
"name": "isApprovedForAll",
"outputs": [
{
"internalType": "bool",
"name": "",
"type": "bool"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "name",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "ownerOf",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "safeTransferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
},
{
"internalType": "bytes",
"name": "_data",
"type": "bytes"
}
],
"name": "safeTransferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"internalType": "bool",
"name": "approved",
"type": "bool"
}
],
"name": "setApprovalForAll",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "bytes4",
"name": "interfaceId",
"type": "bytes4"
}
],
"name": "supportsInterface",
"outputs": [
{
"internalType": "bool",
"name": "",
"type": "bool"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "symbol",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "tokenURI",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "transferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
}
]

Azure Data Factory Pipeline errorCode

I have a relatively simple process set up in Azure Data Factory to copy, cleanse and process some log files from a chatbot which has been running fine until I recently started getting the following errorCode:
"errorCode": "InvalidTemplate",
"message": "Unable to process expressions for action 'EvaluatefinaliseTSCRPTS': 'The function 'bool' was invoked with a parameter that is not valid. The value cannot be converted to the target type",
"failureType": "UserError",
"target": "finaliseTSCRPTS",
"details": ""
I can't seem to identify the error in the ADF process despite going through the code for my process below:
"name": "SearchBot dailyTranscripts",
"properties": {
"activities": [
{
"name": "MST Validation",
"type": "Validation",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "teamsLogs",
"type": "DatasetReference"
},
"timeout": "0.00:00:30",
"sleep": 10,
"childItems": true
}
},
{
"name": "Get MST-TSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "MST Validation",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"mappings": [
{
"source": {
"path": "$['type']"
},
"sink": {
"path": "$['type']"
}
},
{
"source": {
"path": "$['timestamp']"
},
"sink": {
"path": "$['timestamp']"
}
},
{
"source": {
"path": "$['id']"
},
"sink": {
"path": "$['id']"
}
},
{
"source": {
"path": "$['channelId']"
},
"sink": {
"path": "$['channelId']"
}
},
{
"source": {
"path": "$['serviceUrl']"
},
"sink": {
"path": "$['serviceUrl']"
}
},
{
"source": {
"path": "$['from']['id']"
},
"sink": {
"path": "$['from']['id']"
}
},
{
"source": {
"path": "$['from']['aadObjectId']"
},
"sink": {
"path": "$['from']['aadObjectId']"
}
},
{
"source": {
"path": "$['from']['role']"
},
"sink": {
"path": "$['from']['role']"
}
},
{
"source": {
"path": "$['from']['name']"
},
"sink": {
"path": "$['from']['name']"
}
},
{
"source": {
"path": "$['conversation']['conversationType']"
},
"sink": {
"path": "$['conversation']['conversationType']"
}
},
{
"source": {
"path": "$['conversation']['tenantId']"
},
"sink": {
"path": "$['conversation']['tenantId']"
}
},
{
"source": {
"path": "$['conversation']['id']"
},
"sink": {
"path": "$['conversation']['id']"
}
},
{
"source": {
"path": "$['recipient']['id']"
},
"sink": {
"path": "$['recipient']['id']"
}
},
{
"source": {
"path": "$['recipient']['name']"
},
"sink": {
"path": "$['recipient']['name']"
}
},
{
"source": {
"path": "$['recipient']['aadObjectId']"
},
"sink": {
"path": "$['recipient']['aadObjectId']"
}
},
{
"source": {
"path": "$['recipient']['role']"
},
"sink": {
"path": "$['recipient']['role']"
}
},
{
"source": {
"path": "$['channelData']['tenant']['id']"
},
"sink": {
"path": "$['channelData']['tenant']['id']"
}
},
{
"source": {
"path": "$['text']"
},
"sink": {
"path": "$['text']"
}
},
{
"source": {
"path": "$['inputHint']"
},
"sink": {
"path": "$['inputHint']"
}
},
{
"source": {
"path": "$['replyToId']"
},
"sink": {
"path": "$['replyToId']"
}
},
{
"source": {
"path": "$['textFormat']"
},
"sink": {
"path": "$['textFormat']"
}
},
{
"source": {
"path": "$['localTimestamp']"
},
"sink": {
"path": "$['localTimestamp']"
}
},
{
"source": {
"path": "$['locale']"
},
"sink": {
"path": "$['locale']"
}
},
{
"source": {
"path": "$['value']"
},
"sink": {
"path": "$['value']"
}
},
{
"source": {
"path": "$['valueType']"
},
"sink": {
"path": "$['valueType']"
}
},
{
"source": {
"path": "$['name']"
},
"sink": {
"path": "$['name']"
}
},
{
"source": {
"path": "$['label']"
},
"sink": {
"path": "$['label']"
}
}
]
}
},
"inputs": [
{
"referenceName": "teamsLogs",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
}
]
},
{
"name": "finaliseTSCRPTS",
"type": "IfCondition",
"dependsOn": [
{
"activity": "MST Validation",
"dependencyConditions": [
"Completed"
]
},
{
"activity": "Get MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Skipped"
]
}
],
"userProperties": [],
"typeProperties": {
"expression": {
"value": "activity('MST Validation').output.exists",
"type": "Expression"
},
"ifTrueActivities": [
{
"name": "Combine TSCRPTS",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "SearchBotDailyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#concat(formatDateTime(utcnow(), 'yyyy-MM-dd'),'.json')"
}
}
]
},
{
"name": "Delete Staging",
"type": "Delete",
"dependsOn": [
{
"activity": "Combine TSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
}
},
{
"name": "Get monthlyTSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Combine TSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "SearchBotDailyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#concat('2019-',formatDateTime(utcnow(), 'MM'),'-??.json')"
}
}
],
"outputs": [
{
"referenceName": "SearchBotMonthlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#dataset().sourceFileName"
}
}
]
},
{
"name": "Get yearlyTSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Get monthlyTSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "SearchBotMonthlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": {
"value": "#concat(formatDateTime(utcnow(), 'yyyy'),'-??.json')",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "SearchBotYearlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#dataset().sourceFileName"
}
}
]
},
{
"name": "Copy MST-TSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Delete Staging",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "0.00:01:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "teamsLogs",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "transcriptsHistory",
"type": "DatasetReference"
}
]
},
{
"name": "Delete MST-TSCRPTS",
"type": "Delete",
"dependsOn": [
{
"activity": "Copy MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "teamsLogs",
"type": "DatasetReference"
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
}
},
{
"name": "run learningList",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Delete MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Failed",
"Skipped"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "runLearningList",
"type": "PipelineReference"
},
"waitOnCompletion": true
}
}
]
}
}
],
"parameters": {
"sourceFileName": {
"type": "string",
"defaultValue": "#concat(formatDateTime(utcnow(),'yyyy-MM-dd'),'.json')"
}
},
"annotations": []
},
"type": "Microsoft.DataFactory/factories/pipelines"
There is no expression 'EvaluatefinaliseTSCRPTS' and I can't find a 'bool' function either. The only hint I've found was a previous question here
You have an IF activity named "finaliseTSCRPTS", so "EvaluatefinaliseTSCRPTS" is most likely the internal name of the function that performs the IF condition check. The message seems to indicate that it cannot evaluate your expression "activity('MST Validation').output.exists".

Deploying a marketplace Connector from Azure Resource Group template

I'm using the Azure Resource Group project template in Visual studio to deploy two API Apps and a Logic App. I want one of those API Apps to be a Blob Connector from the marketplace. What I need is the uri of the .zip package for the connector, as shown here:
{
"apiVersion": "2014-06-01",
"name": "MSDeploy",
"type": "Extensions",
"dependsOn": [
//........
],
"properties": {
"packageUri": "https://auxmktplceprod.blob.core.windows.net/packages/UmbracoCms.WebPI.7.2.5.zip",
"dbType": "SQL",
(source)
I tried this solution, but that cmdlet is now deprecated. Is there any way to get these URIs?
-Thanks!
I found a way of deploying custom api app with Marketplace apps.
Below is a sample script to just guide you
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"blobConnectorName": {
"type": "string",
"minLength": 1,
"defaultValue" : "mytestblobconnector"
},
"blobStorageAccount": {
"type": "string",
"minLength": 1,
"defaultValue" : "mystorage.blob.core.windows.net"
},
"blobStorageKey": {
"type": "securestring",
"minLength": 1,
"defaultValue" : "storgekey"
},
"blobContainerName": {
"type": "string",
"minLength": 1,
"defaultValue" : "mycontainer"
},
"gatewayName": {
"type": "string",
"minLength": 1,
"defaultValue" : "myblobconnectorgateway"
},
"logicAppName": {
"type": "string",
"minLength": 1,
"defaultValue" : "testinglogicapp"
},
"svcPlanName": {
"type": "string",
"minLength": 1,
"defaultValue" : "myresourcegrpserviceplan"
},
"sku": {
"type": "string",
"defaultValue": "Basic",
"allowedValues": [
"Free",
"Basic",
"Standard",
"Premium"
]
},
"svcPlanSize": {
"defaultValue": "0",
"type": "string",
"allowedValues": [
"0",
"1",
"2"
]
},
"gatewayToApiAppSecret": {
"defaultValue": "0000000000000000000000000000000000000000000000000000000000000000",
"type": "securestring"
}
},
"variables": {
"$packageId": "Microsoft.ApiApp",
"$nugetFeed": "http://apiapps-preview.nuget.org/api/v2/"
},
"resources": [
{
"type": "Microsoft.Web/serverfarms",
"apiVersion": "2015-04-01",
"name": "[parameters('svcPlanName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "AppServicePlan"
},
"properties": {
"name": "[parameters('svcPlanName')]",
"sku": "[parameters('sku')]",
"workerSize": "[parameters('svcPlanSize')]",
"numberOfWorkers": 1
}
},
{
"type": "Microsoft.Web/sites",
"apiVersion": "2015-04-01",
"name": "[parameters('gatewayName')]",
"location": "[resourceGroup().location]",
"kind": "gateway",
"tags": {
"displayName": "GatewayHost"
},
"resources": [
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/gateway",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('gatewayName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
}
}
],
"dependsOn": [
"[concat(resourceGroup().id, '/providers/Microsoft.Web/serverfarms/', parameters('svcPlanName'))]"
],
"properties": {
"name": "[parameters('gatewayName')]",
"gatewaySiteName": "[parameters('gatewayName')]",
"serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('svcPlanName'))]",
"siteConfig": {
"appSettings": [
{
"name": "ApiAppsGateway_EXTENSION_VERSION",
"value": "latest"
},
{
"name": "EmaStorage",
"value": "D:\\home\\data\\apiapps"
},
{
"name": "WEBSITE_START_SCM_ON_SITE_CREATION",
"value": "1"
}
]
}
}
},
{
"type": "Microsoft.AppService/gateways",
"apiVersion": "2015-03-01-preview",
"name": "[parameters('gatewayName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "Gateway"
},
"resources": [
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/gatewaySite",
"dependsOn": [
"[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.Web/sites', parameters('gatewayName'))]"
}
},
{
"type": "tokens",
"apiVersion": "2015-03-01-preview",
"location": "[resourceGroup().location]",
"name": "[parameters('logicAppName')]",
"tags": {
"displayName": "AuthenticationToken"
},
"dependsOn": [
"[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
]
}
],
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('gatewayName'))]"
],
"properties": {
"host": {
"resourceName": "[parameters('gatewayName')]"
}
}
},
{
"type": "Microsoft.Web/sites",
"apiVersion": "2015-04-01",
"name": "[parameters('blobConnectorName')]",
"location": "[resourceGroup().location]",
"kind": "apiApp",
"tags": {
"displayName": "APIAppHost",
"packageId": "AzureStorageBlobConnector"
},
"dependsOn": [
"[resourceId('Microsoft.Web/serverfarms', parameters('svcPlanName'))]",
"[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
],
"resources": [
{
"type": "siteextensions",
"tags": {
"displayName": "APIAppExtension"
},
"apiVersion": "2015-02-01",
"name": "AzureStorageBlobConnector",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('blobConnectorName'))]"
],
"properties": {
"type": "WebRoot",
"feed_url": "[variables('$nugetFeed')]"
}
},
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/apiApp",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('blobConnectorName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.AppService/apiapps', parameters('blobConnectorName'))]"
}
}
],
"properties": {
"gatewaySiteName": "[parameters('gatewayName')]",
"serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('svcPlanName'))]",
"siteConfig": {
"appSettings": [
{
"name": "EMA_MicroserviceId",
"value": "[parameters('blobConnectorName')]"
},
{
"name": "EMA_Secret",
"value": "[parameters('gatewayToAPIappSecret')]"
},
{
"name": "EMA_RuntimeUrl",
"value": "[concat('https://', reference(resourceId('Microsoft.Web/sites', parameters('gatewayName'))).hostNames[0])]"
},
{
"name": "WEBSITE_START_SCM_ON_SITE_CREATION",
"value": "1"
},
{
"name": "BlobConnector_ContainerUrl",
"value": "[concat('https://', parameters('blobStorageAccount'),'/',parameters('blobContainerName'))]"
},
{
"name": "BlobConnector_AccessKey",
"value": "[parameters('blobStorageKey')]"
}
],
"applicationLogs": {
"filesystem": {
"level": "Verbose"
},
"azureTableStorage": {
"level": "Off",
"sasUrl": null
},
"azureBlobStorage": {
"level": "Off",
"sasUrl": null,
"retentionInDays": null
}
}
}
}
},
{
"type": "Microsoft.AppService/apiapps",
"apiVersion": "2015-03-01-preview",
"name": "[parameters('blobConnectorName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "BlobConnector"
},
"resources": [
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/apiAppSite",
"dependsOn": [
"[resourceId('Microsoft.AppService/apiapps', parameters('blobConnectorName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.Web/sites', parameters('blobConnectorName'))]"
}
}
],
"dependsOn": [
"[resourceId('Microsoft.Web/sites/siteextensions', parameters('blobConnectorName'), 'AzureStorageBlobConnector')]"
],
"properties": {
"package": {
"id": "AzureStorageBlobConnector"
},
"host": {
"resourceName": "[parameters('blobConnectorName')]"
},
"gateway": {
"resourceName": "[parameters('gatewayName')]"
},
"dependencies": [ ]
}
},
{
"type": "Microsoft.Logic/workflows",
"apiVersion": "2015-02-01-preview",
"name": "[parameters('logicAppName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "LogicApp"
},
"dependsOn": [
"[resourceId('Microsoft.AppService/apiApps', parameters('blobConnectorName'))]"
],
"properties": {
"sku": {
"name": "[parameters('sku')]",
"plan": {
"id": "[concat(resourceGroup().id, '/providers/Microsoft.Web/serverfarms/',parameters('svcPlanName'))]"
}
},
"definition": {
"$schema": "http://schema.management.azure.com/providers/Microsoft.Logic/schemas/2014-12-01-preview/workflowdefinition.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"token": {
"defaultValue": "[reference(resourceId('Microsoft.AppService/gateways/tokens', parameters('gatewayName'), parameters('logicAppName'))).token]",
"type": "String",
"metadata": {
"token": {
"name": "token"
}
}
},
"runworkflowmanually": {
"defaultValue": true,
"type": "Bool"
}
},
"triggers": { },
"actions": {
"azurestorageblobconnector": {
"type": "ApiApp",
"inputs": {
"apiVersion": "2015-01-14",
"host": {
"id": "[concat(resourceGroup().id, '/providers/Microsoft.AppService/apiApps/',parameters('blobConnectorName'))]",
"gateway": "[concat('https://', reference(resourceId('Microsoft.Web/sites', parameters('gatewayName'))).hostNames[0])]"
},
"operation": "UploadBlob",
"parameters": {
"BlobPath": "myfolder/test.txt",
"BlobContent": {
"Content": "TestMessage",
"ContentTransferEncoding": "None"
},
"Overwrite": true
},
"authentication": {
"type": "Raw",
"scheme": "Zumo",
"parameter": "#parameters('token')"
}
},
"conditions": [ ]
}
},
"outputs": {
}
},
"parameters": { }
}
}
]
}
Just search for "AzureStorageBlobConnector" in the above json to observe the usage which is the package id of the blob connector from marketplace. I found the package id for the blob connector from azure portal by deploying one manually and then checking its settings. Please feel free to post a comment for package id for other market place apps if you face any difficulty.