Intermittent Selenium ECONNREFUSED errors on CircleCI - selenium

Question

Our CircleCI tests use selenium, via the selenium-webdriver to run UI tests through PhantomJS. The tests work 100% of the time locally in our vagrant env, but fail about 1 out of 3 times on CircleCI with ECONNREFUSED errors like:
Error: ECONNREFUSED connect ECONNREFUSED 10.0.4.1:59525
at ClientRequest.<anonymous> (node_modules/selenium-webdriver/http/index.js:238:15)
at Socket.socketErrorListener (_http_client.js:310:9)
at emitErrorNT (net.js:1278:8)
at _combinedTickCallback (internal/process/next_tick.js:74:11)
at process._tickCallback (internal/process/next_tick.js:98:9)
From: Task: WebDriver.navigate().to(http://127.0.0.1:8080/src/public/login.php?t=ur&ign=1&ut=ad)
at thenableWebDriverProxy.schedule (node_modules/selenium-webdriver/lib/webdriver.js:816:17)
at Navigation.to (node_modules/selenium-webdriver/lib/webdriver.js:1140:25)
at thenableWebDriverProxy.get (node_modules/selenium-webdriver/lib/webdriver.js:997:28)
at User.logIn (testJs/ui/utils/user.js:9:16)
at Host.logInAsHost (testJs/ui/utils/host.js:13:14)
at Preferences.disableRevenueGeneration (testJs/ui/utils/preferences.js:57:14)
at Context.<anonymous> (testJs/ui/tests/preferences.js:13:22)
These errors happen at random times throughout our tests and they are not being triggered by any particular place in our tests. As far as I can tell, this issue is occurring on the selenium/selenium-webdriver end as the web server remains up and working and isn't generating any errors.
I’ve tried all of the following and none of them have worked:
Upgraded to the latest selenium-webdriver (3.4.0)
Upgraded to a more recent version of nodejs (6.9.2)
Used a different web server
Upgraded to a recent version of PhantomJS (1.9.7-15)
Tried using export DBUS_SESSION_BUS_ADDRESS as per
https://github.com/SeleniumHQ/docker-selenium/issues/87
In node_modules/selenium-webdriver/http/index.js, I modified the code to retry for ECONNREFUSED errors by reusing the retry already there for ECONNRESET, i.e. if (e.code === 'ECONNRESET') { becomes if (e.code === 'ECONNRESET' || e.code === 'ECONNREFUSED') {. This doesn’t work as then selenium-webdriver just retries indefinitely and from this I realized that the issue appears to be that once an ECONNREFUSED error is encountered, selenium/selenium-webdriver are unrecoverable.
Has anyone found a solution?

Answer 1

WARNING: This is merely a hack and is by no means a proper solution, but it is the only one that appears to reliably work for us on CircleCI. Hopefully, someone else will find a better solution.
Our test framework, mocha, appears to have a retry mechanism developed specifically for selenium (go figure): http://mochajs.org/#retry-tests. The first gotcha is that code in after and before is not retried. Therefore, you need to move code to beforeEach and afterEach. The second gotcha is that you need some special logic in beforeEach and afterEach to handle driver errors.
So, in order to utilize this mechanism, we had to refactor our tests so that each test has a beforeEach that sets up a fresh webdriver instance and then an afterEach that quits that driver instance. This way, a crashed driver instance doesn’t stop all subsequent tests. We developed a helper function that we call at the beginning of all describe blocks so that we don’t have to add too much code to our tests.
Then, we added this.retries(10) to our topmost describe block.
Here is the helper class:
var webdriver = require('selenium-webdriver');
// We default to using phantomjs, but in the future, when we run tests in other browsers, e.g. via
// SauceLabs, we'll want to change the browser.
var customPhantom = webdriver.Capabilities.phantomjs();
customPhantom.set('ssl-protocol', 'any');
// For convenience in tests
global.By = webdriver.By;
global.until = webdriver.until;
var SeleniumStabilizer = function () {};
SeleniumStabilizer.prototype.MAX_RETRIES = 10;
SeleniumStabilizer.prototype.createDriver = function () {
global.driver = new webdriver.Builder()
// .forBrowser('phantomjs')
.withCapabilities(customPhantom)
.build();
// Resolves when driver is ready
return global.driver;
};
SeleniumStabilizer.prototype.init = function (opts) {
var self = this,
n = 0;
var beforeEachWithRetries = function () {
// Create a fresh selenium driver for the next test
return self.createDriver().then(function () {
if (opts.onBeforeEach) {
// Execute the test-specific defined onBeforeEach
return opts.onBeforeEach();
}
}).catch(function (err) {
// ECONNREFUSED error and we should retry?
if (err.message.indexOf('ECONNREFUSED') !== -1 && n++ < self.MAX_RETRIES) {
return beforeEachWithRetries();
} else {
throw err;
}
});
};
opts.beforeEach(function () {
n = 0;
return beforeEachWithRetries();
});
var afterEachWithRetries = function () {
return Promise.resolve().then(function () {
if (opts.onAfterEach) {
// Execute the test-specific defined onAfterEach
return opts.onAfterEach();
}
}).then(function () {
// Quit the selenium driver before starting the next test
return driver.quit();
}).catch(function (err) {
// ECONNREFUSED error and we should retry?
if (err.message.indexOf('ECONNREFUSED') !== -1 && n++ < self.MAX_RETRIES) {
// Create a new driver
return self.createDriver().then(function () {
return afterEachWithRetries();
});
} else {
throw err;
}
});
};
opts.afterEach(function () {
n = 0;
return afterEachWithRetries();
});
};
Then, your tests look like:
var seleniumStabilizer = new SeleniumStabilizer();
describe('my tests', function () {
this.retries(seleniumStabilizer.MAX_RETRIES);
describe('search engines', function () {
seleniumStabilizer.init({
beforeEach: beforeEach,
afterEach: afterEach,
onBeforeEach: function () {
// e.g. return doPromiseAfterDriverSetUp();
},
onAfterEach: function () {
// e.g. return doPromiseBeforeDriverQuits();
}
});
it('should get google', function () {
return driver.get('https://www.google.com');
});
it('should get amazon', function () {
return driver.get('https://www.amazon.com');
});
});
});