How can I get Mule to download just attachments using pop3? I tried following the example at http://www.mulesoft.org/documentation/display/current/POP3+Transport+Reference as closely as possible, but I keep getting two files: one with the e-mail body and one with the actual attachment. Here's the flow I'm using:
<pop3:connector name="pop3Connector" checkFrequency="5000" doc:name="POP3"/>
<expression-transformer name="returnAttachments" doc:name="Expression">
<return-argument evaluator="attachments-list" expression="*" />
</expression-transformer>
<file:connector name="fileName" doc:name="File">
<file:expression-filename-parser/>
</file:connector>
<flow name="incoming-orders" doc:name="incoming-orders">
<pop3s:inbound-endpoint host="pop.gmail.com" port="995" user="myuser%40mydomain" password="mypassword" responseTimeout="10000" doc:name="POP3" transformer-refs="returnAttachments" />
<collection-splitter doc:name="Collection Splitter"/>
<file:outbound-endpoint path="C:/popthreetest" outputPattern="#[function:datestamp].dat" doc:name="File">
<expression-transformer>
<return-argument expression="payload.inputStream" evaluator="groovy" />
</expression-transformer>
</file:outbound-endpoint>
</flow>
Thanks!
edit:
Here's the final flow based on #David Dossot's answer. I have an added complexity in that I'm reading in a JSON file that specifies attachment names and an arbitrary destinations for the attachment. I included the replaceAll because I was getting an error about an invalid character in the path file:///C:\.
<pop3:connector name="pop3Connector" checkFrequency="5000" doc:name="POP3"/>
<expression-transformer name="returnAttachments" doc:name="Expression">
<return-argument evaluator="attachments-list" expression="*" />
</expression-transformer>
<file:connector name="DestinationsFileConnector" doc:name="File" autoDelete="false" streaming="true" validateConnections="true">
<file:expression-filename-parser/>
</file:connector>
<file:endpoint path="C:/popthreetest/" name="DestinationsFileEndpoint" responseTimeout="10000" doc:name="File" connector-ref="DestinationsFileConnector">
<file:filename-regex-filter pattern="destinations\.json" caseSensitive="true"/>
</file:endpoint>
<mulerequester:config name="DestinationsMuleRequestorConnector" doc:name="Mule Requester"/>
<flow name="incoming-orders" doc:name="incoming-orders">
<pop3s:inbound-endpoint host="pop.gmail.com" port="995" user="myusername%40mydomain" password="mypassword" responseTimeout="10000" doc:name="POP3" transformer-refs="returnAttachments" />
<collection-splitter doc:name="Collection Splitter"/>
<set-variable variableName="MessagePart" value="#[message.payload]" doc:name="MessagePart"/>
<logger message="Got #[message.payload.dataSource.name]." level="INFO" doc:name="Logger"/>
<mulerequester:request config-ref="DestinationsMuleRequestorConnector" resource="DestinationsFileEndpoint" doc:name="GetDestinations"/>
<json:json-to-object-transformer returnClass="java.util.HashMap" doc:name="JSON to Object"/>
<choice doc:name="Choice">
<when expression="#[message.payload.get(MessagePart.dataSource.name) != null]">
<set-payload value="#[message.payload.get(MessagePart.dataSource.name)]" doc:name="Destination List"/>
<foreach doc:name="For Each">
<logger message="Saving to #[message.payload]." level="INFO" doc:name="Logger"/>
<set-variable variableName="DestinationPath" value="#[java.nio.file.Paths.get(message.payload).getParent().toString().replaceAll('\\\\', '/')]" doc:name="DestinationPath"/>
<set-variable variableName="DestinationPattern" value="#[java.nio.file.Paths.get(message.payload).getFileName()]" doc:name="DestinationPattern"/>
<logger message="Saving to #[DestinationPattern] in #[DestinationPath]." level="INFO" doc:name="Logger"/>
<set-payload value="#[MessagePart]" doc:name="MessagePart"/>
<file:outbound-endpoint path="#[DestinationPath]" outputPattern="#[DestinationPattern]" doc:name="File">
<expression-transformer>
<return-argument expression="payload.inputStream" evaluator="groovy"/>
</expression-transformer>
</file:outbound-endpoint>
</foreach>
</when>
<otherwise>
<logger message="Did not find destination(s) for #[MessagePart.dataSource.name]." level="INFO" doc:name="Logger"/>
</otherwise>
</choice>
</flow>
For completeness, here's the JSON file:
{
"attachment-name.txt": [
"C:/popthreetest/firstDestination.txt"
, "C:/path/to/secondDestination.txt"
, "C:\\popthreetest\\destination\\using-backslashes.txt"
]
}
An email with attachment is a multi-part email into which attachments are parts but also the body. Hence Mule can only download "the whole package" and give the different parts to you.
You should be able to filter the body part after the collection-splitter based on the name of the part.
Alternatively, you could use a MEL expression to drop the first element of the collection, which is typically the body (Mule uses this technique internally to set the message payload: https://github.com/mulesoft/mule/blob/mule-3.x/transports/email/src/main/java/org/mule/transport/email/transformers/EmailMessageToString.java#L50 )
Related
SFTP connector does not delete the ssource file after reading in batch processing, while run as normal flow the file gets deleted
I am setting autoDelete="true" condition in the SFTP connector. Can anyone please suggest how to handle this scenario
<context:property-placeholder location="dev.properties"/>
<encryption:config name="Encryption" defaultEncrypter="PGP_ENCRYPTER" doc:name="Encryption">
<encryption:pgp-encrypter-config publicKeyRingFileName="keys/dev_pgp_wd_ecc_public.key.gpg" secretKeyRingFileName="keys/dev_pgp_wd_ecc_private.key.gpg" secretAliasId="${key.AliasId}" secretPassphrase="${key.Passphrase}" principal="${key.principal}"/>
</encryption:config>
<amqp:connector name="AMQP_Connector" validateConnections="true" host="${amqp.host}" doc:name="AMQP Connector" virtualHost="${amqp.virtualhost}" password="${amqp.password}" port="${amqp.port}" username="${amqp.user}"/>
<http:request-config name="HTTP_Request_PI" host="${pi.endpoint}" port="${pi.port}" doc:name="HTTP Request Configuration" basePath="${pi.path}" responseTimeout="30000">
<http:basic-authentication username="${pi.username}" password="${pi.password}"/>
</http:request-config>
<smtp:connector name="SMTP_Alert" contentType="text/html" validateConnections="true" doc:name="SMTP"/>
<sftp:connector name="SFTP_Inbound_Connector" validateConnections="true" autoDelete="true" pollingFrequency="120000" doc:name="SFTP"/>
<batch:job name="TestBatch" max-failed-records="-1">
<batch:input>
<sftp:inbound-endpoint connector-ref="SFTP_Inbound_Connector" host="${sftp.host}" port="${sftp.port}" path="/test/incoming/lt" user="${sftp.user}" password="${sftp.password}" responseTimeout="10000" doc:name="SFTP"/>
<encryption:decrypt config-ref="Encryption" using="PGP_ENCRYPTER" doc:name="Encryption"/>
<object-to-string-transformer doc:name="Object to String"/>
<logger message="Decrypted" level="INFO" doc:name="Logger"/>
<splitter expression="#[xpath3('/wd:Report_Data/wd:Report_Entry', payload, 'NODESET')]" doc:name="Splitter"/>
<mulexml:dom-to-xml-transformer doc:name="DOM to XML"/>
<dw:transform-message doc:name="Transform Message" metadata:id="b7cbbbbb-d58b-439b-8684-5a6d0345d48c">
<dw:input-payload doc:sample="empty.xml"/>
<dw:set-payload></dw:set-payload>
</dw:transform-message>
</batch:input>
<batch:process-records>
<batch:step name="Batch_Step1">
<json:object-to-json-transformer doc:name="Object to JSON"/>
<batch:commit size="5" doc:name="Batch Commit">
<amqp:outbound-endpoint exchangeName="${amqp.exchangeName}" queueName="${amqp.queueName}" responseTimeout="10000" encoding="UTF-8" mimeType="application/xml" connector-ref="AMQP_Connector" doc:name="AMQP"/>
</batch:commit>
</batch:step>
</batch:process-records>
<batch:on-complete>
<logger message="#[flowVars.totalRecords.totalRecords]" level="INFO" doc:name="Logger"/>
<logger message="#[flowVars.failedReocrds.failedReocrds]" level="INFO" doc:name="Logger"/>
<set-payload value="${mail.html}" doc:name="Set Payload" mimeType="text/html"/>
<smtp:outbound-endpoint host="${mail.host}" port="${mail.port}" user="${mail.user}" password="${mail.password}" connector-ref="SMTP_Alert" to="${mail.receiver}" from="${mail.from}" subject="${IntegrationName}" responseTimeout="10000" doc:name="SMTP"/>
</batch:on-complete>
</batch:job>
There seemed to be bugs around autoDelete and SFTP:
https://www.mulesoft.org/jira/browse/MULE-9144
So maybe this is the issue you are running into however in your normal flow you say it works.
So an option is to try a normal flow together with the batch execute component, so your FTP inbound endpoint and collection creation is part of a flow and from that flow you execute the batch processing. The input section of your batch element will be empty.
https://docs.mulesoft.com/mule-user-guide/v/3.7/batch-processing#triggering-batch-jobs
The problem statement is to do multiple things parallely and aggregate the response and store it in a file.
link to the mule flow image as in studio:
image
In this flow, what I was trying to do was to set two constant strings in two branches of scatter and gather and aggregate and store in file. I tried overwriting the payload with a "set payload" with "my response". I am expecting "my response" as the content of the file. But instead the file content is:
¨Ìsr)java.util.concurrent.CopyOnWriteArrayListx]ü’F´ê√xpwtmsg 1tmsg 2x
I did debug and the payload at File endpoint was "my response". How and why is the collection getting written into file.
Can anyone help me to get it working.
Following is the xml:
<flow name="mule-assignFlow21123">
<quartz:inbound-endpoint jobName="dummyflow" repeatInterval="10000" responseTimeout="10000" doc:name="Quartz">
<quartz:event-generator-job/>
</quartz:inbound-endpoint>
<scatter-gather doc:name="Scatter-Gather1" >
<threading-profile maxThreadsActive="1" poolExhaustedAction="RUN"/>
<processor-chain>
<set-payload value="msg 1" doc:name="Set Payload"/>
<logger level="INFO" doc:name="Logger"/>
</processor-chain>
<processor-chain>
<set-payload value="msg 2" doc:name="Set Payload"/>
<logger level="INFO" doc:name="Logger"/>
</processor-chain>
</scatter-gather>
<set-payload value="my response" doc:name="Set Payload"/>
<file:outbound-endpoint path="/Users/premkumar/Desktop" outputPattern="Results.txt" responseTimeout="10000" mimeType="text/plain" doc:name="Save 2 File"/>
</flow>
The flow will automatically determine the processingStrategy from the in-flight event which will be async because of the quartz endpoint, so the file endpoint will fire async also.
Instead explicitly set the flow's processingStrategy to synchronous:
<flow name="mule-assignFlow21123" processingStrategy="synchronous">
<quartz:inbound-endpoint jobName="dummyflow" repeatInterval="10000" responseTimeout="10000" doc:name="Quartz">
<quartz:event-generator-job/>
</quartz:inbound-endpoint>
<scatter-gather doc:name="Scatter-Gather1" >
<threading-profile maxThreadsActive="1" poolExhaustedAction="RUN"/>
<processor-chain>
<set-payload value="msg 1" doc:name="Set Payload"/>
<logger level="INFO" doc:name="Logger"/>
</processor-chain>
<processor-chain>
<set-payload value="msg 2" doc:name="Set Payload"/>
<logger level="INFO" doc:name="Logger"/>
</processor-chain>
</scatter-gather>
<set-payload value="my response" doc:name="Set Payload"/>
<file:outbound-endpoint path="/Users/premkumar/Desktop" outputPattern="Results.txt" responseTimeout="10000" mimeType="text/plain" doc:name="Save 2 File"/>
I want to move files to location D:\target dynamically from the query select filepath from emp where status='y':
This is my table:
emp_Name File Path File Name Status
ABC D:\emp abc.txt y
xyz D:\emp xyz.txt y
bcs D:\emp bcs.txt n
Following is my source code :
<flow name="testdbFlow1">
<http:listener config-ref="HTTP_Listener_Configuration"
path="/" doc:name="HTTP" />
<jdbc-ee:outbound-endpoint queryKey="allEmps"
queryTimeout="-1" connector-ref="JDBCConnector" exchange-pattern="request-response"
doc:name="Database" />
<foreach doc:name="Foreach">
<choice doc:name="Choice">
<when expression="#[payload.status == 'Y']">
<processor-chain doc:name="Processor Chain">
<set-variable variableName="filePath" value="#[payload.filepath]"
doc:name="Variable" />
<set-variable variableName="filename" value="#[payload.filename]"
doc:name="Variable" />
<logger message="#[filePath]" level="INFO"
doc:name="Logger" />
<logger message="#[filename]"
level="INFO" doc:name="Logger" />
<file:inbound-endpoint path="#[filePath]" name="input" doc:name="File"
pollingFrequency="12000" responseTimeout="10000"> <file:filename-wildcard-filter
pattern="#[filename]" /> </file:inbound-endpoint>
<file:outbound-endpoint name="output" path="D:\target" doc:name="File"/>
</processor-chain>
</when>
<otherwise>
<processor-chain doc:name="Processor Chain">
<set-variable variableName="filePath" value="#[payload.filepath]"
doc:name="Variable" />
<set-variable variableName="filename" value="#[payload.filename]"
doc:name="Variable" />
<logger message="#[filePath]" level="INFO"
doc:name="Logger" />
<logger message="#[filename]"
level="INFO" doc:name="Logger" />
</processor-chain>
</otherwise>
</choice>
</foreach>
</flow>
But it's not working.
Inbound doesn't work out of the box in the middle of the flow. I would suggest a simpler solution using Groovy because you know exactly the file you need to consume.
Instead of using file:inbound create a input stream manually:
<set-payload value="#[groovy: new java.io.FileInputStream(new java.io.File(filename))]"/>
You will have to delete the file with another groovy script after consumed.
<flow name="listobjects">
<http:inbound-endpoint exchange-pattern="request-response" host="localhost" port="8081" path="listobjects" contentType="text/plain" doc:name="HTTP"/>
<s3:list-objects config-ref="Amazon_S3" bucketName="demo" doc:name="Amazon S3" maxKeys="5" />
<!-- <payload-type-filter expectedType="java.util.List" doc:name="Payload"/> -->
<foreach collection="#[payload]" doc:name="For Each">
<!-- <foreach doc:name="For Each file"> -->
<logger message=" inside foreach...... #[payload.getKey()] ...." level="INFO" doc:name="Logger" />
<s3:get-object-content config-ref="Amazon_S3" bucketName="demo" key="#[payload.getKey()]" doc:name="Amazon S3"/>
<object-to-byte-array-transformer/>
<file:outbound-endpoint path="C:\output" responseTimeout="10000" doc:name="File" outputPattern="#[payload.getKey()] "></file:outbound-endpoint>
</foreach>
</flow>
I have bucket name called demo.
In that bucket I have 3 pdf files. I want to download all files and put it in c:\output folder.
I hit my url like http://localhost:8081/listobjects.
But I got the error:
Could not find a transformer to transform "CollectionDataType{type=org.mule.module.s3.simpleapi.SimpleAmazonS3AmazonDevKitImpl$S3ObjectSummaryIterable, itemType=com.amazonaws.services.s3.model.S3ObjectSummary, mimeType='/'}" to "SimpleDataType{type=org.mule.api.transport.OutputHandler, mimeType='/'}". (org.mule.api.transformer.TransformerException) (org.mule.api.transformer.TransformerException). Message payload is of type: SimpleAmazonS3AmazonDevKitImpl$S3ObjectSummaryIterable
The error occurs because after the foreach processor the payload is an instance of an S3 class, and you haven't specified any Content-Type to return. So Mule tries to transform the S3 instance to the default SimpleDataType and fails.
One way to solve it is simply to add something like
<set-property propertyName="Content-Type" value="application/json" doc:name="Content-Type" />
<set-payload value="{'result': 'ok'}"/>
at the end to make it explicit.
Also note that in your flow after running:
<object-to-byte-array-transformer/>
the S3 payload is gone, so #[payload.getKey()] will fail in the next processor:
<file:outbound-endpoint path="C:\output" responseTimeout="10000" doc:name="File" outputPattern="#[payload.getKey()] "></file:outbound-endpoint>
I've run this without problems:
<flow name="listobjects">
<http:inbound-endpoint exchange-pattern="request-response" host="localhost" port="8083" path="listobjects" contentType="text/plain" doc:name="HTTP"/>
<s3:list-objects config-ref="Amazon_S3" bucketName="mule_test" doc:name="Amazon S3" maxKeys="5" />
<foreach collection="#[payload]" doc:name="For Each">
<logger message=" inside foreach...... #[payload.getKey()] ...." level="INFO" doc:name="Logger" />
<set-variable variableName="fileKey" value="#[payload.getKey()]" doc:name="Variable" />
<s3:get-object-content config-ref="Amazon_S3" bucketName="#[payload.getBucketName()]" key="#[payload.getKey()]" doc:name="Amazon S3"/>
<object-to-byte-array-transformer/>
<file:outbound-endpoint path="/tmp" responseTimeout="10000" doc:name="File" outputPattern="#[flowVars.fileKey] "></file:outbound-endpoint>
</foreach>
<set-property propertyName="Content-Type" value="application/json" doc:name="Content-Type" />
<set-payload value="{'result': 'ok'}"/>
</flow>
I am using the following fork and join pattern in my flow.Parallel processing works just fine. However, I have issues with exception handling. I want to implement an exception strategy for the processing happening on VM response inbound endpoint of the router. However, it seems my nothing happens when I am trying to write to response, from within the catch exception strategy block. When there is an exception, I am able to see the log statement from the catch block, but the response on browser just hangs.
Please suggest.
<flow name="forkAndJoinFlow" doc:name="forkAndJoinFlow">
<http:inbound-endpoint exchange-pattern="request-response" host="localhost" port="81" path="lowestprice" doc:name="HTTP"/>
<not-filter doc:name="Not">
<wildcard-filter pattern="*favicon*" caseSensitive="true"/>
</not-filter>
<request-reply>
<all>
<vm:outbound-endpoint path="shop1"/>
<vm:outbound-endpoint path="shop2"/>
</all>
<vm:inbound-endpoint path="response">
<message-properties-transformer>
<add-message-property key="MULE_CORRELATION_GROUP_SIZE" value="2" />
</message-properties-transformer>
<collection-aggregator />
</vm:inbound-endpoint>
</request-reply>
<expression-transformer evaluator="groovy" expression="java.util.Collections.min(payload)" doc:name="Expression"/>
<object-to-string-transformer doc:name="Object to String"/>
<logger level="WARN" message="#[string:Lowest price: #[payload]]" doc:name="Logger"/>
<catch-exception-strategy doc:name="Catch Exception Strategy">
<logger message="EXCEPTION:#[payload]" level="INFO" doc:name="Logger"></logger>
<!--Nothing happens after this-->
<object-to-string-transformer doc:name="Object to String"></object-to-string-transformer></catch-exception-strategy>
</flow>
<flow name="shop1Flow" doc:name="shop1Flow">
<vm:inbound-endpoint path="shop1" doc:name="VM"/>
<logger level="INFO" message="SHOP1 Flow..." doc:name="Logger"/>
<expression-transformer evaluator="groovy" expression="new java.lang.Double(1000.0 * Math.random()).intValue()" doc:name="Expression"/>
<logger level="WARN" message="#[string:Price from shop 1: #[payload]]" doc:name="Logger"/>
</flow>
<flow name="shop2Flow" doc:name="shop2Flow">
<vm:inbound-endpoint path="shop2" doc:name="VM"/>
<logger level="INFO" message="SHOP2 Flow..." doc:name="Logger"/>
<expression-transformer evaluator="groovy" expression="new java.lang.Double(1000.0 * Math.random()).intValue()" doc:name="Expression"/>
<logger level="WARN" message="#[string:Price from shop 2: #[payload]]" doc:name="Logger"/>`enter code here`
</flow>
</mule>