Azure Data Factory Pipeline errorCode - azure-data-factory-2

Question

I have a relatively simple process set up in Azure Data Factory to copy, cleanse and process some log files from a chatbot which has been running fine until I recently started getting the following errorCode:
"errorCode": "InvalidTemplate",
"message": "Unable to process expressions for action 'EvaluatefinaliseTSCRPTS': 'The function 'bool' was invoked with a parameter that is not valid. The value cannot be converted to the target type",
"failureType": "UserError",
"target": "finaliseTSCRPTS",
"details": ""
I can't seem to identify the error in the ADF process despite going through the code for my process below:
"name": "SearchBot dailyTranscripts",
"properties": {
"activities": [
{
"name": "MST Validation",
"type": "Validation",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "teamsLogs",
"type": "DatasetReference"
},
"timeout": "0.00:00:30",
"sleep": 10,
"childItems": true
}
},
{
"name": "Get MST-TSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "MST Validation",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"mappings": [
{
"source": {
"path": "$['type']"
},
"sink": {
"path": "$['type']"
}
},
{
"source": {
"path": "$['timestamp']"
},
"sink": {
"path": "$['timestamp']"
}
},
{
"source": {
"path": "$['id']"
},
"sink": {
"path": "$['id']"
}
},
{
"source": {
"path": "$['channelId']"
},
"sink": {
"path": "$['channelId']"
}
},
{
"source": {
"path": "$['serviceUrl']"
},
"sink": {
"path": "$['serviceUrl']"
}
},
{
"source": {
"path": "$['from']['id']"
},
"sink": {
"path": "$['from']['id']"
}
},
{
"source": {
"path": "$['from']['aadObjectId']"
},
"sink": {
"path": "$['from']['aadObjectId']"
}
},
{
"source": {
"path": "$['from']['role']"
},
"sink": {
"path": "$['from']['role']"
}
},
{
"source": {
"path": "$['from']['name']"
},
"sink": {
"path": "$['from']['name']"
}
},
{
"source": {
"path": "$['conversation']['conversationType']"
},
"sink": {
"path": "$['conversation']['conversationType']"
}
},
{
"source": {
"path": "$['conversation']['tenantId']"
},
"sink": {
"path": "$['conversation']['tenantId']"
}
},
{
"source": {
"path": "$['conversation']['id']"
},
"sink": {
"path": "$['conversation']['id']"
}
},
{
"source": {
"path": "$['recipient']['id']"
},
"sink": {
"path": "$['recipient']['id']"
}
},
{
"source": {
"path": "$['recipient']['name']"
},
"sink": {
"path": "$['recipient']['name']"
}
},
{
"source": {
"path": "$['recipient']['aadObjectId']"
},
"sink": {
"path": "$['recipient']['aadObjectId']"
}
},
{
"source": {
"path": "$['recipient']['role']"
},
"sink": {
"path": "$['recipient']['role']"
}
},
{
"source": {
"path": "$['channelData']['tenant']['id']"
},
"sink": {
"path": "$['channelData']['tenant']['id']"
}
},
{
"source": {
"path": "$['text']"
},
"sink": {
"path": "$['text']"
}
},
{
"source": {
"path": "$['inputHint']"
},
"sink": {
"path": "$['inputHint']"
}
},
{
"source": {
"path": "$['replyToId']"
},
"sink": {
"path": "$['replyToId']"
}
},
{
"source": {
"path": "$['textFormat']"
},
"sink": {
"path": "$['textFormat']"
}
},
{
"source": {
"path": "$['localTimestamp']"
},
"sink": {
"path": "$['localTimestamp']"
}
},
{
"source": {
"path": "$['locale']"
},
"sink": {
"path": "$['locale']"
}
},
{
"source": {
"path": "$['value']"
},
"sink": {
"path": "$['value']"
}
},
{
"source": {
"path": "$['valueType']"
},
"sink": {
"path": "$['valueType']"
}
},
{
"source": {
"path": "$['name']"
},
"sink": {
"path": "$['name']"
}
},
{
"source": {
"path": "$['label']"
},
"sink": {
"path": "$['label']"
}
}
]
}
},
"inputs": [
{
"referenceName": "teamsLogs",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
}
]
},
{
"name": "finaliseTSCRPTS",
"type": "IfCondition",
"dependsOn": [
{
"activity": "MST Validation",
"dependencyConditions": [
"Completed"
]
},
{
"activity": "Get MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Skipped"
]
}
],
"userProperties": [],
"typeProperties": {
"expression": {
"value": "activity('MST Validation').output.exists",
"type": "Expression"
},
"ifTrueActivities": [
{
"name": "Combine TSCRPTS",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "SearchBotDailyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#concat(formatDateTime(utcnow(), 'yyyy-MM-dd'),'.json')"
}
}
]
},
{
"name": "Delete Staging",
"type": "Delete",
"dependsOn": [
{
"activity": "Combine TSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
}
},
{
"name": "Get monthlyTSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Combine TSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "SearchBotDailyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#concat('2019-',formatDateTime(utcnow(), 'MM'),'-??.json')"
}
}
],
"outputs": [
{
"referenceName": "SearchBotMonthlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#dataset().sourceFileName"
}
}
]
},
{
"name": "Get yearlyTSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Get monthlyTSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "SearchBotMonthlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": {
"value": "#concat(formatDateTime(utcnow(), 'yyyy'),'-??.json')",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "SearchBotYearlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#dataset().sourceFileName"
}
}
]
},
{
"name": "Copy MST-TSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Delete Staging",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "0.00:01:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "teamsLogs",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "transcriptsHistory",
"type": "DatasetReference"
}
]
},
{
"name": "Delete MST-TSCRPTS",
"type": "Delete",
"dependsOn": [
{
"activity": "Copy MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "teamsLogs",
"type": "DatasetReference"
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
}
},
{
"name": "run learningList",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Delete MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Failed",
"Skipped"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "runLearningList",
"type": "PipelineReference"
},
"waitOnCompletion": true
}
}
]
}
}
],
"parameters": {
"sourceFileName": {
"type": "string",
"defaultValue": "#concat(formatDateTime(utcnow(),'yyyy-MM-dd'),'.json')"
}
},
"annotations": []
},
"type": "Microsoft.DataFactory/factories/pipelines"
There is no expression 'EvaluatefinaliseTSCRPTS' and I can't find a 'bool' function either. The only hint I've found was a previous question here

Answer 1

You have an IF activity named "finaliseTSCRPTS", so "EvaluatefinaliseTSCRPTS" is most likely the internal name of the function that performs the IF condition check. The message seems to indicate that it cannot evaluate your expression "activity('MST Validation').output.exists".