I have JMS inbound endpoint which subscribes to the topic. once there is a message transformer splits the payload into list of records and then insert into database using batch-commit. If there is any error while inserting into the database I want roll back the entire payload to JMS. How to achieve this using transactions?
<batch:job name="lockboxBatch" max-failed-records="-1">
<batch:input>
<jms:inbound-endpoint topic="lockbox" connector-ref="Active_MQ1" doc:name="JMS">
</jms:inbound-endpoint>
<custom-transformer class="transformers.PaymentsTransformer" doc:name="Java"/>
<logger level="INFO" doc:name="Logger"/>
</batch:input>
<batch:process-records>
<batch:step name="Batch_Step">
<expression-component doc:name="Expression"><![CDATA[payload[2].batchAmount='hghghfghfhgf']]></expression-component>
<batch:commit size="4" doc:name="Batch Commit">
<db:insert config-ref="Oracle_Configuration" doc:name="Database" bulkMode="true" >
<db:parameterized-query><![CDATA[INSERT INTO TIB_INT_AR_PAYMENT_IFACE (TRANSMISSION_REQUEST_ID,DESTINATION_ACCOUNT,ORIGINATION,TRANSMISSION_RECORD_COUNT,TRANSMISSION_AMOUNT,LOCKBOX_NUMBER,LOCKBOX_BATCH_COUNT,LOCKBOX_RECORD_COUNT,LOCKBOX_AMOUNT,BATCH_NAME,BATCH_AMOUNT,BATCH_RECORD_COUNT,ITEM_NUMBER,CURRENCY_CODE,REMITTANCE_AMOUNT,TRANSIT_ROUTING_NUMBER,ACCOUNT,CHECK_NUMBER,CUSTOMER_NUMBER,OVERFLOW_INDICATOR,OVERFLOW_SEQUENCE,INVOICE1,AMOUNT_APPLIED1) VALUES (#[payload.?transmissiosnRequestID],#[payload.?destinastionAccount],#[payload.?origination],#[payload.?transmissionSrecordCount],#[payload.?transmisssionAmount],#[payload.lockboxNumber],#[payload.lockboxBatchCount],#[payload.lockboxRecordCount],#[payload.lockboxAmount],#[payload.batchName],#[payload.batchAmount],#[payload.batchRecordCount],#[payload.itemNumber],#[payload.currencyCode],#[payload.remittanceAmount],#[payload.transitRoutingNumber],#[payload.account],#[payload.checkNumber],#[payload.customerNumber],#[payload.overflowIndicator],#[payload.overflowSequence],#[payload.invoice1],#[payload.amountApplied1])]]></db:parameterized-query>
</db:insert>
</batch:commit>
</batch:step>
<batch:step name="Batch_Step1" accept-policy="ONLY_FAILURES">
<set-payload value="#[getStepExceptions()]" doc:name="Set Payload"/>
<foreach collection="#[payload.values()]" doc:name="For Each">
<jms:outbound-endpoint queue="Invalid_Transmission" connector-ref="Active_MQ" doc:name="JMS"/>
</foreach>
</batch:step>
</batch:process-records>
<batch:on-complete>
<logger message="Completed the insert" level="INFO" doc:name="Logger"/>
</batch:on-complete>
</batch:job>
You can try two things.
1.Make max-failed-records="0",this will rollback in case of any failure in batch step.
2.Encomapss the DB connector inside transaction scope,and handle the exception scenario as required.
<transactional action="ALWAYS_BEGIN" doc:name="Transactional">
<db:insert...>........</db:insert>
</transactional>
Please consider below updated code,you can make changes to suit your requirements.
<batch:job name="lockboxBatch" max-failed-records="0">
<batch:input>
<jms:inbound-endpoint topic="lockbox" connector-ref="Active_MQ1" doc:name="JMS">
</jms:inbound-endpoint>
<custom-transformer class="transformers.PaymentsTransformer" doc:name="Java"/>
<logger level="INFO" doc:name="Logger"/>
</batch:input>
<batch:process-records>
<batch:step name="Batch_Step">
<expression-component doc:name="Expression"><![CDATA[payload[2].batchAmount='hghghfghfhgf']]></expression-component>
<batch:commit size="4" doc:name="Batch Commit">
<transactional action="ALWAYS_BEGIN" doc:name="Transactional">
<db:insert config-ref="Oracle_Configuration" bulkMode="true" doc:name="Database">
<db:parameterized-query><![CDATA[INSERT INTO TIB_INT_AR_PAYMENT_IFACE (TRANSMISSION_REQUEST_ID,DESTINATION_ACCOUNT,ORIGINATION,TRANSMISSION_RECORD_COUNT,TRANSMISSION_AMOUNT,LOCKBOX_NUMBER,LOCKBOX_BATCH_COUNT,LOCKBOX_RECORD_COUNT,LOCKBOX_AMOUNT,BATCH_NAME,BATCH_AMOUNT,BATCH_RECORD_COUNT,ITEM_NUMBER,CURRENCY_CODE,REMITTANCE_AMOUNT,TRANSIT_ROUTING_NUMBER,ACCOUNT,CHECK_NUMBER,CUSTOMER_NUMBER,OVERFLOW_INDICATOR,OVERFLOW_SEQUENCE,INVOICE1,AMOUNT_APPLIED1) VALUES (#[payload.?transmissiosnRequestID],#[payload.?destinastionAccount],#[payload.?origination],#[payload.?transmissionSrecordCount],#[payload.?transmisssionAmount],#[payload.lockboxNumber],#[payload.lockboxBatchCount],#[payload.lockboxRecordCount],#[payload.lockboxAmount],#[payload.batchName],#[payload.batchAmount],#[payload.batchRecordCount],#[payload.itemNumber],#[payload.currencyCode],#[payload.remittanceAmount],#[payload.transitRoutingNumber],#[payload.account],#[payload.checkNumber],#[payload.customerNumber],#[payload.overflowIndicator],#[payload.overflowSequence],#[payload.invoice1],#[payload.amountApplied1])]]></db:parameterized-query>
</db:insert>
</transactional>
</batch:commit>
</batch:step>
<batch:step name="Batch_Step1" accept-policy="ONLY_FAILURES">
<set-payload value="#[getStepExceptions()]" doc:name="Set Payload"/>
<foreach collection="#[payload.values()]" doc:name="For Each">
<jms:outbound-endpoint queue="Invalid_Transmission" connector-ref="Active_MQ" doc:name="JMS"/>
</foreach>
</batch:step>
</batch:process-records>
<batch:on-complete>
<logger message="Completed the insert" level="INFO" doc:name="Logger"/>
</batch:on-complete>
</batch:job>
Related
Please find below is the code of two batches, the payload from ma-sfdc-marketing-activities-batch-executionBatch commit is reaching properly to the ma-sfdc-marketing-activities-batch-executionBatch1 input phase but when it tries to enter process phase batch step getting the error
<batch:job name="ma-sfdc-marketing-activities-batch-executionBatch" max-failed-records="-1" block-size="10">
<batch:process-records>
<batch:step name="Batch_Step" accept-policy="ALL">
........
</batch:step>
<batch:step name="Batch_Step3" accept-expression="#[payload.ListId != null]">
<batch:commit size="10000" doc:name="Batch Commit">
<enricher target="#[flowVars['jobInfo_contact_pref_payload']]" doc:name="Message Enricher">
<processor-chain doc:name="Processor Chain">
<dw:transform-message doc:name="Transform Message" metadata:id="264ca131-fd48-425a-ab79-c990728d9997">
<dw:input-payload/>
<dw:set-payload resource="classpath:elq-to-sfdc-task-v1.dwl"/>
</dw:transform-message>
<dw:transform-message doc:name="Transform Message">
<dw:set-payload><![CDATA[%dw 1.0
%output application/java
---
flatten payload.wholepl]]></dw:set-payload>
</dw:transform-message>
<async doc:name="Async">
<batch:execute name="ma-sfdc-marketing-activities-batch-executionBatch1" doc:name="ma-sfdc-marketing-activities-batch-executionBatch1"/>
</async>
</processor-chain>
</enricher>
</batch:commit>
</batch:step>
</batch:process-records>
<batch:on-complete>
.......
</batch:on-complete>
</batch:job>
<batch:job name="ma-sfdc-marketing-activities-batch-executionBatch1">
<batch:input>
.......
</batch:input>
<batch:process-records>
<batch:step name="Batch_Step1">
<logger message="subflow payload............. #[payload]" level="INFO" doc:name="Logger"/>
<batch:commit size="5000" doc:name="Batch Commit">
<logger message="#['updating sfdc with payload: ' + message.payloadAs(java.lang.String)]" level="INFO" doc:name="Logger"/>
<sfdc:create-batch config-ref="Salesforce__Basic_Authentication" doc:name="Salesforce">
<sfdc:job-info ref="#[flowVars.jobInfo_contact_pref]"/>
<sfdc:objects ref="#[payload]"/>
</sfdc:create-batch>
</batch:commit>
</batch:step>
</batch:process-records>
<batch:on-complete>
.......
</batch:on-complete>
</batch:job>
I'm calling batch from other batchcommit asynchronously but getting below error, control is not entering into second batch commit, getting below error.
Exception in thread "[ma-sfdc-batch-interface-v1.1].Batch Dispatcher thread" org.mule.api.serialization.SerializationException: Could not deserialize object
at org.mule.serialization.internal.AbstractObjectSerializer.deserialize(AbstractObjectSerializer.java:139)
at org.mule.serialization.internal.AbstractObjectSerializer.deserialize(AbstractObjectSerializer.java:112)
at org.mule.serialization.internal.AbstractObjectSerializer.deserialize(AbstractObjectSerializer.java:102)
at com.mulesoft.module.batch.engine.DefaultBatchJobInstanceStore.doRetrieve(DefaultBatchJobInstanceStore.java:370)
at com.mulesoft.module.batch.engine.DefaultBatchJobInstanceStore.getJobInstance(DefaultBatchJobInstanceStore.java:249)
at com.mulesoft.module.batch.engine.DefaultBatchEngine.refresh(DefaultBatchEngine.java:932)
at com.mulesoft.module.batch.engine.DefaultBatchEngine.getSteppingQueueSize(DefaultBatchEngine.java:826)
at com.mulesoft.module.batch.engine.BatchRecordDispatcherDelegate.dispatch(BatchRecordDispatcherDelegate.java:135)
at com.mulesoft.module.batch.engine.BatchRecordDispatcherDelegate.dispatch(BatchRecordDispatcherDelegate.java:122)
at com.mulesoft.module.batch.engine.BatchRecordDispatcherDelegate.run(BatchRecordDispatcherDelegate.java:90)
at com.google.common.util.concurrent.AbstractExecutionThreadService$1$2.run(AbstractExecutionThreadService.java:60)
at com.google.common.util.concurrent.Callables$3.run(Callables.java:95)
at java.lang.Thread.run(Unknown Source)
SFTP connector does not delete the ssource file after reading in batch processing, while run as normal flow the file gets deleted
I am setting autoDelete="true" condition in the SFTP connector. Can anyone please suggest how to handle this scenario
<context:property-placeholder location="dev.properties"/>
<encryption:config name="Encryption" defaultEncrypter="PGP_ENCRYPTER" doc:name="Encryption">
<encryption:pgp-encrypter-config publicKeyRingFileName="keys/dev_pgp_wd_ecc_public.key.gpg" secretKeyRingFileName="keys/dev_pgp_wd_ecc_private.key.gpg" secretAliasId="${key.AliasId}" secretPassphrase="${key.Passphrase}" principal="${key.principal}"/>
</encryption:config>
<amqp:connector name="AMQP_Connector" validateConnections="true" host="${amqp.host}" doc:name="AMQP Connector" virtualHost="${amqp.virtualhost}" password="${amqp.password}" port="${amqp.port}" username="${amqp.user}"/>
<http:request-config name="HTTP_Request_PI" host="${pi.endpoint}" port="${pi.port}" doc:name="HTTP Request Configuration" basePath="${pi.path}" responseTimeout="30000">
<http:basic-authentication username="${pi.username}" password="${pi.password}"/>
</http:request-config>
<smtp:connector name="SMTP_Alert" contentType="text/html" validateConnections="true" doc:name="SMTP"/>
<sftp:connector name="SFTP_Inbound_Connector" validateConnections="true" autoDelete="true" pollingFrequency="120000" doc:name="SFTP"/>
<batch:job name="TestBatch" max-failed-records="-1">
<batch:input>
<sftp:inbound-endpoint connector-ref="SFTP_Inbound_Connector" host="${sftp.host}" port="${sftp.port}" path="/test/incoming/lt" user="${sftp.user}" password="${sftp.password}" responseTimeout="10000" doc:name="SFTP"/>
<encryption:decrypt config-ref="Encryption" using="PGP_ENCRYPTER" doc:name="Encryption"/>
<object-to-string-transformer doc:name="Object to String"/>
<logger message="Decrypted" level="INFO" doc:name="Logger"/>
<splitter expression="#[xpath3('/wd:Report_Data/wd:Report_Entry', payload, 'NODESET')]" doc:name="Splitter"/>
<mulexml:dom-to-xml-transformer doc:name="DOM to XML"/>
<dw:transform-message doc:name="Transform Message" metadata:id="b7cbbbbb-d58b-439b-8684-5a6d0345d48c">
<dw:input-payload doc:sample="empty.xml"/>
<dw:set-payload></dw:set-payload>
</dw:transform-message>
</batch:input>
<batch:process-records>
<batch:step name="Batch_Step1">
<json:object-to-json-transformer doc:name="Object to JSON"/>
<batch:commit size="5" doc:name="Batch Commit">
<amqp:outbound-endpoint exchangeName="${amqp.exchangeName}" queueName="${amqp.queueName}" responseTimeout="10000" encoding="UTF-8" mimeType="application/xml" connector-ref="AMQP_Connector" doc:name="AMQP"/>
</batch:commit>
</batch:step>
</batch:process-records>
<batch:on-complete>
<logger message="#[flowVars.totalRecords.totalRecords]" level="INFO" doc:name="Logger"/>
<logger message="#[flowVars.failedReocrds.failedReocrds]" level="INFO" doc:name="Logger"/>
<set-payload value="${mail.html}" doc:name="Set Payload" mimeType="text/html"/>
<smtp:outbound-endpoint host="${mail.host}" port="${mail.port}" user="${mail.user}" password="${mail.password}" connector-ref="SMTP_Alert" to="${mail.receiver}" from="${mail.from}" subject="${IntegrationName}" responseTimeout="10000" doc:name="SMTP"/>
</batch:on-complete>
</batch:job>
There seemed to be bugs around autoDelete and SFTP:
https://www.mulesoft.org/jira/browse/MULE-9144
So maybe this is the issue you are running into however in your normal flow you say it works.
So an option is to try a normal flow together with the batch execute component, so your FTP inbound endpoint and collection creation is part of a flow and from that flow you execute the batch processing. The input section of your batch element will be empty.
https://docs.mulesoft.com/mule-user-guide/v/3.7/batch-processing#triggering-batch-jobs
I am iterating xml in foreach. Echo is printing correct message but logger is print null.
<foreach collection="#[xpath('//xmlsoap/soap/soapenv:Envelope')]"
doc:name="For Each">
<echo-component />
<logger level="INFO" category="ProTSP Logger" message="#[payload]"
doc:name="Logger" />
</foreach>
I resolved my problem using dom to xml transformer.
<foreach collection="#[xpath('//xmlsoap/soap/soapenv:Envelope')]"
doc:name="For Each">
<echo-component doc:name="Echo"/>
<mulexml:dom-to-xml-transformer doc:name="DOM to XML"/>
<logger level="INFO" category="ProTSP Logger" message="#[payload]"
doc:name="Logger" />
</foreach>
How can I get Mule to download just attachments using pop3? I tried following the example at http://www.mulesoft.org/documentation/display/current/POP3+Transport+Reference as closely as possible, but I keep getting two files: one with the e-mail body and one with the actual attachment. Here's the flow I'm using:
<pop3:connector name="pop3Connector" checkFrequency="5000" doc:name="POP3"/>
<expression-transformer name="returnAttachments" doc:name="Expression">
<return-argument evaluator="attachments-list" expression="*" />
</expression-transformer>
<file:connector name="fileName" doc:name="File">
<file:expression-filename-parser/>
</file:connector>
<flow name="incoming-orders" doc:name="incoming-orders">
<pop3s:inbound-endpoint host="pop.gmail.com" port="995" user="myuser%40mydomain" password="mypassword" responseTimeout="10000" doc:name="POP3" transformer-refs="returnAttachments" />
<collection-splitter doc:name="Collection Splitter"/>
<file:outbound-endpoint path="C:/popthreetest" outputPattern="#[function:datestamp].dat" doc:name="File">
<expression-transformer>
<return-argument expression="payload.inputStream" evaluator="groovy" />
</expression-transformer>
</file:outbound-endpoint>
</flow>
Thanks!
edit:
Here's the final flow based on #David Dossot's answer. I have an added complexity in that I'm reading in a JSON file that specifies attachment names and an arbitrary destinations for the attachment. I included the replaceAll because I was getting an error about an invalid character in the path file:///C:\.
<pop3:connector name="pop3Connector" checkFrequency="5000" doc:name="POP3"/>
<expression-transformer name="returnAttachments" doc:name="Expression">
<return-argument evaluator="attachments-list" expression="*" />
</expression-transformer>
<file:connector name="DestinationsFileConnector" doc:name="File" autoDelete="false" streaming="true" validateConnections="true">
<file:expression-filename-parser/>
</file:connector>
<file:endpoint path="C:/popthreetest/" name="DestinationsFileEndpoint" responseTimeout="10000" doc:name="File" connector-ref="DestinationsFileConnector">
<file:filename-regex-filter pattern="destinations\.json" caseSensitive="true"/>
</file:endpoint>
<mulerequester:config name="DestinationsMuleRequestorConnector" doc:name="Mule Requester"/>
<flow name="incoming-orders" doc:name="incoming-orders">
<pop3s:inbound-endpoint host="pop.gmail.com" port="995" user="myusername%40mydomain" password="mypassword" responseTimeout="10000" doc:name="POP3" transformer-refs="returnAttachments" />
<collection-splitter doc:name="Collection Splitter"/>
<set-variable variableName="MessagePart" value="#[message.payload]" doc:name="MessagePart"/>
<logger message="Got #[message.payload.dataSource.name]." level="INFO" doc:name="Logger"/>
<mulerequester:request config-ref="DestinationsMuleRequestorConnector" resource="DestinationsFileEndpoint" doc:name="GetDestinations"/>
<json:json-to-object-transformer returnClass="java.util.HashMap" doc:name="JSON to Object"/>
<choice doc:name="Choice">
<when expression="#[message.payload.get(MessagePart.dataSource.name) != null]">
<set-payload value="#[message.payload.get(MessagePart.dataSource.name)]" doc:name="Destination List"/>
<foreach doc:name="For Each">
<logger message="Saving to #[message.payload]." level="INFO" doc:name="Logger"/>
<set-variable variableName="DestinationPath" value="#[java.nio.file.Paths.get(message.payload).getParent().toString().replaceAll('\\\\', '/')]" doc:name="DestinationPath"/>
<set-variable variableName="DestinationPattern" value="#[java.nio.file.Paths.get(message.payload).getFileName()]" doc:name="DestinationPattern"/>
<logger message="Saving to #[DestinationPattern] in #[DestinationPath]." level="INFO" doc:name="Logger"/>
<set-payload value="#[MessagePart]" doc:name="MessagePart"/>
<file:outbound-endpoint path="#[DestinationPath]" outputPattern="#[DestinationPattern]" doc:name="File">
<expression-transformer>
<return-argument expression="payload.inputStream" evaluator="groovy"/>
</expression-transformer>
</file:outbound-endpoint>
</foreach>
</when>
<otherwise>
<logger message="Did not find destination(s) for #[MessagePart.dataSource.name]." level="INFO" doc:name="Logger"/>
</otherwise>
</choice>
</flow>
For completeness, here's the JSON file:
{
"attachment-name.txt": [
"C:/popthreetest/firstDestination.txt"
, "C:/path/to/secondDestination.txt"
, "C:\\popthreetest\\destination\\using-backslashes.txt"
]
}
An email with attachment is a multi-part email into which attachments are parts but also the body. Hence Mule can only download "the whole package" and give the different parts to you.
You should be able to filter the body part after the collection-splitter based on the name of the part.
Alternatively, you could use a MEL expression to drop the first element of the collection, which is typically the body (Mule uses this technique internally to set the message payload: https://github.com/mulesoft/mule/blob/mule-3.x/transports/email/src/main/java/org/mule/transport/email/transformers/EmailMessageToString.java#L50 )
I am using the following fork and join pattern in my flow.Parallel processing works just fine. However, I have issues with exception handling. I want to implement an exception strategy for the processing happening on VM response inbound endpoint of the router. However, it seems my nothing happens when I am trying to write to response, from within the catch exception strategy block. When there is an exception, I am able to see the log statement from the catch block, but the response on browser just hangs.
Please suggest.
<flow name="forkAndJoinFlow" doc:name="forkAndJoinFlow">
<http:inbound-endpoint exchange-pattern="request-response" host="localhost" port="81" path="lowestprice" doc:name="HTTP"/>
<not-filter doc:name="Not">
<wildcard-filter pattern="*favicon*" caseSensitive="true"/>
</not-filter>
<request-reply>
<all>
<vm:outbound-endpoint path="shop1"/>
<vm:outbound-endpoint path="shop2"/>
</all>
<vm:inbound-endpoint path="response">
<message-properties-transformer>
<add-message-property key="MULE_CORRELATION_GROUP_SIZE" value="2" />
</message-properties-transformer>
<collection-aggregator />
</vm:inbound-endpoint>
</request-reply>
<expression-transformer evaluator="groovy" expression="java.util.Collections.min(payload)" doc:name="Expression"/>
<object-to-string-transformer doc:name="Object to String"/>
<logger level="WARN" message="#[string:Lowest price: #[payload]]" doc:name="Logger"/>
<catch-exception-strategy doc:name="Catch Exception Strategy">
<logger message="EXCEPTION:#[payload]" level="INFO" doc:name="Logger"></logger>
<!--Nothing happens after this-->
<object-to-string-transformer doc:name="Object to String"></object-to-string-transformer></catch-exception-strategy>
</flow>
<flow name="shop1Flow" doc:name="shop1Flow">
<vm:inbound-endpoint path="shop1" doc:name="VM"/>
<logger level="INFO" message="SHOP1 Flow..." doc:name="Logger"/>
<expression-transformer evaluator="groovy" expression="new java.lang.Double(1000.0 * Math.random()).intValue()" doc:name="Expression"/>
<logger level="WARN" message="#[string:Price from shop 1: #[payload]]" doc:name="Logger"/>
</flow>
<flow name="shop2Flow" doc:name="shop2Flow">
<vm:inbound-endpoint path="shop2" doc:name="VM"/>
<logger level="INFO" message="SHOP2 Flow..." doc:name="Logger"/>
<expression-transformer evaluator="groovy" expression="new java.lang.Double(1000.0 * Math.random()).intValue()" doc:name="Expression"/>
<logger level="WARN" message="#[string:Price from shop 2: #[payload]]" doc:name="Logger"/>`enter code here`
</flow>
</mule>