Sales force EMP connector, stops receiving notification after some time - api

Question

I am performing a POC to check Streaming API stability, POC is as follows
Program 1 : subscribe to pushtopic created against Account object
Program 2 : create, update & delete single record after every 10 min interval
Both this programs were kept running for more than 12 hours (left overnight), after that I verified if all notification are received or not and found that after sometime (in this case it was nearly ~ 2 hours 45 min ) no notification were received, I repeated this twice and both case it stops getting notification after sometime.
Test code used
Streaming API client (using EMP connector)
public class SFPoc {
static Long count = 0L;
static Long Leadcount = 0L;
public static void main(String[] argv) throws Exception {
String userName = "<user_name>";
String password = "<pwd>";
String pushTopicName = "/topic/AccountPT";
String pushTopicNameLead = "/topic/Leadwhere";
long replayFrom = EmpConnector.REPLAY_FROM_EARLIEST;
String securityToken = "<token>";
BayeuxParameters custom = getBayeuxParamWithSpecifiedAPIVersion("37.0");
BayeuxParameters params = null;
try {
params = login(userName, password + securityToken, custom);
} catch (Exception e) {
e.printStackTrace();
}
Consumer<Map<String, Object>> consumer = event -> System.out.println(String.format("Received:\n%s ** Recieved at %s, event count total %s", event, LocalDateTime.now() , ++count));
Consumer<Map<String, Object>> consumerLead = event -> System.out.println(String.format("****** LEADS ***** Received:\n%s ** Recieved at %s, event count total %s", event, LocalDateTime.now() , ++Leadcount));
EmpConnector connector = new EmpConnector(params);
connector.start().get(10, TimeUnit.SECONDS);
TopicSubscription subscription = connector.subscribe(pushTopicName, replayFrom, consumer).get(10, TimeUnit.SECONDS);
TopicSubscription subscriptionLead = connector.subscribe(pushTopicNameLead, replayFrom, consumerLead).get(10, TimeUnit.SECONDS);
System.out.println(String.format("Subscribed: %s", subscription));
System.out.println(String.format("Subscribed: %s", subscriptionLead));
}
private static BayeuxParameters getBayeuxParamWithSpecifiedAPIVersion(String apiVersion) {
BayeuxParameters params = new BayeuxParameters() {
#Override
public String version() {
return apiVersion;
}
#Override
public String bearerToken() {
return null;
}
};
return params;
}
}
Code which is doing record create/update/delete periodically to generate events
import com.sforce.soap.enterprise.*;
import com.sforce.soap.enterprise.Error;
import com.sforce.soap.enterprise.sobject.Account;
import com.sforce.soap.enterprise.sobject.Contact;
import com.sforce.ws.ConnectionException;
import com.sforce.ws.ConnectorConfig;
import java.time.LocalDateTime;
public class SFDCDataAdjustment {
static final String USERNAME = "<username>";
static final String PASSWORD = "<pwd&securitytoken>";
static EnterpriseConnection connection;
static Long count = 0L;
public static void main(String[] args) {
ConnectorConfig config = new ConnectorConfig();
config.setUsername(USERNAME);
config.setPassword(PASSWORD);
//config.setTraceMessage(true);
try {
connection = Connector.newConnection(config);
// display some current settings
System.out.println("Auth EndPoint: "+config.getAuthEndpoint());
System.out.println("Service EndPoint: "+config.getServiceEndpoint());
System.out.println("Username: "+config.getUsername());
System.out.println("SessionId: "+config.getSessionId());
// run the different examples
while (true) {
createAccounts();
updateAccounts();
deleteAccounts();
Thread.sleep(1 * 10 * 60 * 1000);
}
} catch (ConnectionException e1) {
e1.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// queries and displays the 5 newest contacts
private static void queryContacts() {
System.out.println("Querying for the 5 newest Contacts...");
try {
// query for the 5 newest contacts
QueryResult queryResults = connection.query("SELECT Id, FirstName, LastName, Account.Name " +
"FROM Contact WHERE AccountId != NULL ORDER BY CreatedDate DESC LIMIT 5");
if (queryResults.getSize() > 0) {
for (int i=0;i<queryResults.getRecords().length;i++) {
// cast the SObject to a strongly-typed Contact
Contact c = (Contact)queryResults.getRecords()[i];
System.out.println("Id: " + c.getId() + " - Name: "+c.getFirstName()+" "+
c.getLastName()+" - Account: "+c.getAccount().getName());
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
// create 5 test Accounts
private static void createAccounts() {
System.out.println("Creating a new test Account...");
Account[] records = new Account[1];
try {
// create 5 test accounts
for (int i=0;i<1;i++) {
Account a = new Account();
a.setName("OptyAccount "+i);
records[i] = a;
}
// create the records in Salesforce.com
SaveResult[] saveResults = connection.create(records);
// check the returned results for any errors
for (int i=0; i< saveResults.length; i++) {
if (saveResults[i].isSuccess()) {
System.out.println(i+". Successfully created record - Id: " + saveResults[i].getId() + "At " + LocalDateTime.now());
System.out.println("************Event Count************" + ++count);
} else {
Error[] errors = saveResults[i].getErrors();
for (int j=0; j< errors.length; j++) {
System.out.println("ERROR creating record: " + errors[j].getMessage());
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
// updates the 5 newly created Accounts
private static void updateAccounts() {
System.out.println("Update a new test Accounts...");
Account[] records = new Account[1];
try {
QueryResult queryResults = connection.query("SELECT Id, Name FROM Account ORDER BY " +
"CreatedDate DESC LIMIT 1");
if (queryResults.getSize() > 0) {
for (int i=0;i<queryResults.getRecords().length;i++) {
// cast the SObject to a strongly-typed Account
Account a = (Account)queryResults.getRecords()[i];
System.out.println("Updating Id: " + a.getId() + " - Name: "+a.getName());
// modify the name of the Account
a.setName(a.getName()+" -- UPDATED");
records[i] = a;
}
}
// update the records in Salesforce.com
SaveResult[] saveResults = connection.update(records);
// check the returned results for any errors
for (int i=0; i< saveResults.length; i++) {
if (saveResults[i].isSuccess()) {
System.out.println(i+". Successfully updated record - Id: " + saveResults[i].getId() + "At " + LocalDateTime.now());
System.out.println("************Event Count************" + ++count);
} else {
Error[] errors = saveResults[i].getErrors();
for (int j=0; j< errors.length; j++) {
System.out.println("ERROR updating record: " + errors[j].getMessage());
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
// delete the 5 newly created Account
private static void deleteAccounts() {
System.out.println("Deleting new test Accounts...");
String[] ids = new String[1];
try {
QueryResult queryResults = connection.query("SELECT Id, Name FROM Account ORDER BY " +
"CreatedDate DESC LIMIT 1");
if (queryResults.getSize() > 0) {
for (int i=0;i<queryResults.getRecords().length;i++) {
// cast the SObject to a strongly-typed Account
Account a = (Account)queryResults.getRecords()[i];
// add the Account Id to the array to be deleted
ids[i] = a.getId();
System.out.println("Deleting Id: " + a.getId() + " - Name: "+a.getName());
}
}
// delete the records in Salesforce.com by passing an array of Ids
DeleteResult[] deleteResults = connection.delete(ids);
// check the results for any errors
for (int i=0; i< deleteResults.length; i++) {
if (deleteResults[i].isSuccess()) {
System.out.println(i+". Successfully deleted record - Id: " + deleteResults[i].getId() + "At " + LocalDateTime.now());
System.out.println("************Event Count************" + ++count);
} else {
Error[] errors = deleteResults[i].getErrors();
for (int j=0; j< errors.length; j++) {
System.out.println("ERROR deleting record: " + errors[j].getMessage());
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
Further updates got below mentioned error after which notification were
2017-03-09T19:30:28.346 ERROR [com.salesforce.emp.connector.EmpConnector] - connection failure, reconnecting
org.cometd.common.TransportException: {httpCode=503}
at org.cometd.client.transport.LongPollingTransport$2.onComplete(LongPollingTransport.java:278)
at org.eclipse.jetty.client.ResponseNotifier.notifyComplete(ResponseNotifier.java:193)
After this reconnect also happened and handshake also happened but error seems to be in resubscribe() EMP connector seems to be not able to resubscribe for some reason
Note I am using "resubscribe-on-disconnect" branch of EMP connetor

Answer 1

We have determined there was a bug on the server side in a 403 case. The Streaming API uses a session routing cookie and this cookie periodically expires. When it expires, the session is routed to another server, and this responds with a 403. In the current version, this 403 response does not include connect advice, and the client does not attempt to reconnect. This has been fixed and the fix is currently live. My understanding is that this should fix the reconnect problem exhibited by the clients.