Functional testing with grunt-webdriver, mocha and chai-as-promised - selenium

I'm trying to build a functional testing system to verify our web site is behaving correctly for our users. I have cobbled together a bunch of Node.js modules and helpers in an attempt to get a framework that provides simple, concise tests without a heap of nested function callbacks and I believe promises can provide that, so my package.json file looks like this:
"dependencies": {
"chai-as-promised": "^4.3.0",
"grunt": "^0.4.5",
"grunt-webdriver": "^0.4.8"
}
My Gruntfile.js looks like this:
module.exports = function(grunt) {
grunt.initConfig({
webdriver: { // for use with webdriver.io
options: {
desiredCapabilities: {
browserName: 'phantomjs' // No Xvfb required
}
},
chrome: {
tests: ['chrome/*.js'],
options: {
desiredCapabilities: {
browserName: 'chrome'
}
}
},
},
});
grunt.loadNpmTasks('grunt-webdriver');
grunt.registerTask('default', ['webdriver']);
};
And finally my test case in chrome/login.js looks like this:
'use strict';
var chai = require('chai'),
chaiAsPromised = require('chai-as-promised'),
assert;
chaiAsPromised.transferPromiseness = browser.transferPromiseness;
chai.use(chaiAsPromised);
assert = chai.assert;
describe('login test', function () {
it('verifies user can log in', function(done) {
browser
.url('https://localhost/')
.setValue('#userauth_username','foo')
.setValue('#userauth_password',"password")
.submitForm('#form_users_login')
.then(function(){
browser.getText('#auth-user-id', function(err, value){
console.log(value);
});
assert.becomes(browser.getText('#auth-user-id'), 'foo');
})//.call(done);
});
});
When I run grunt webdriver:chrome on the command line, I see it start up Chrome and log into the website. The 'auth-user-id' span is correctly displaying the user's id after logging in but for some reason browser.getText() is not returning it and the test is therefore failing. I have tried adding a .pause(100) after the .submitForm() to give me time to interact with the page in Chrome so I know it is a problem in the test case.
What am I doing wrong?

This seems to be the best and most succinct way of doing what I want. I'm not sure I need chai-as-promised yet but maybe I'll move the login function to an included file and use chai-as-promised to assert that the promised login has occurred before entering the tests.
'use strict';
var chai = require('chai'),
chaiAsPromised = require('chai-as-promised'),
assert,
expect;
chaiAsPromised.transferPromiseness = browser.transferPromiseness;
chai.use(chaiAsPromised);
assert = chai.assert;
expect = chai.expect;
describe('login test', function () {
it('verifies user can log in', function(done) {
browser
.url('https://localhost/')
.setValue('#userauth_username','foo')
.setValue('#userauth_password',"password")
.submitForm('#form_users_login')
.waitForExist('#auth-user-id')
.getText('#auth-user-id')
.then(function(text){
//console.log('Username: ' + text);
assert.equal(text, 'foo');
})
.saveScreenshot('out.png')
.call(done)
});
it('should not display logincontrols after login', function(done){
browser
.isVisible('#logincontrols')
.then(function(bool){
expect(bool).to.be.false;
})
.call(done)
});
it('should display loggedin section after login', function(done){
browser
.isVisible('#loggedin')
.then(function(bool){
expect(bool).to.be.true;
})
.call(done)
});
});
and for completeness, this is what I see on the output:
# grunt webdriver:chrome
Running "webdriver:chrome" (webdriver) task
login test
✓ verifies user can log in (7691ms)
✓ should not display logincontrols after login (70ms)
✓ should display loggedin section after login (58ms)
3 passing (8s)
Done, without errors.

Related

How to configure Detox lifecycle hooks in the latest version?

I run detox in version 17.13.2 with jest-circus as the test runner. My main problem is the app is not reset either after or before I run the tests which leads to an inconsistent state of the app.
My test file:
import { by, device, element, expect, waitFor } from "detox"
describe("Login", () => {
beforeEach(async () => {
await device.reloadReactNative()
})
it("should login with correct data", async () => {
await element(by.id("login_email_input")).typeText("ch.tietz#gmail.com")
await element(by.id("login_password_input")).typeText("12345678")
await element(by.id("login_submit_button")).tap()
await waitFor(element(by.id("workout_screen"))).toBeVisible()
// additional test steps
})
})
Now if the login actually works but the test fails at one of the subsequent steps, the state of the app user will still be "logged in".
From what I understand, it's not possible to actually alter the app state, e.g. by clearing the AsyncStorage or interacting directly with the state mgmt tool. Instead, it's recommended to just reinstall the app - but this is exactly where I am struggling.
I have tried numerous approaches and none of them worked. What makes it really hard to understand configuration is that detox completely changed how the configuration works and switched to jest-circus as the main test runner.
My setup is basically the one created by jest init -r jest. From what I understand, this already includes some defaults for detox.init() and detox.cleanup():
{
"detox": {
"behavior": {
"init": {
"reinstallApp": true,
"launchApp": true,
"exposeGlobals": true
},
"cleanup": {
"shutdownDevice": false
}
}
}
}
However, this does not seem to be sufficient to actually wipe the app state after running the tests.
I tried working with an init script as setupFilesAfterEnv, which would call cleanup() after the test suite is run. Actually that works in an older project which still uses jasmine 2 as test runner.
import { cleanup, init } from 'detox';
const adapter = require('detox/runners/jest/adapter');
const specReporter = require('detox/runners/jest/specReporter');
const config = require('../package.json').detox;
// Set the default timeout
jest.setTimeout(120000);
jasmine.getEnv().addReporter(adapter);
// This takes care of generating status logs on a per-spec basis. By default, jest only reports at file-level.
// This is strictly optional.
jasmine.getEnv().addReporter(specReporter);
beforeAll(async () => {
await init(config, { launchApp: false });
}, 300000);
beforeEach(async () => {
await adapter.beforeEach();
});
afterAll(async () => {
await adapter.afterAll();
await cleanup();
});
First off, it complains that jasmine is not defined. I guess that's because actually the adapter in this case should be a DetoxAdapterCircus which it is not, even though in my config file I specify:
{
"preset": "react-native",
"testEnvironment": "./environment.ts",
"testRunner": "jest-circus/runner",
"testTimeout": 120000,
"testRegex": "\\.e2e\\.ts$",
"reporters": ["detox/runners/jest/streamlineReporter"],
"verbose": true
}
"testRunner": "jest-circus/runner"
Another idea would be to alter the CustomDetoxEnvironment but I do not understand how I can get access to the detox lifecycle hooks.
const {
DetoxCircusEnvironment,
SpecReporter,
WorkerAssignReporter,
} = require("detox/runners/jest-circus")
class CustomDetoxEnvironment extends DetoxCircusEnvironment {
constructor(config) {
super(config)
// should I access the hooks here now?
// This takes care of generating status logs on a per-spec basis. By default, Jest only reports at file-level.
// This is strictly optional.
this.registerListeners({
SpecReporter,
WorkerAssignReporter,
})
}
}
module.exports = CustomDetoxEnvironment
Tl;dr: I don't know where to put my reusable lifecycle hooks in the latest version of Detox. Also, I wonder if these custom configurations are even needed to reinstall the app and wipe the app data before each test suite.
You can still use setupFilesAfterEnv, but you're not supposed to call the same hooks/cleanup as before (see https://github.com/wix/Detox/issues/2410#issuecomment-744387707).
here's an example of our init script:
// init.ts
// with "setupFilesAfterEnv": ["./init.ts"] in the conf
beforeAll(async () => {
// to reset the state
await device.clearKeychain();
// we are launching the app manually
await device.launchApp({
permissions: { notifications: 'YES', location: 'inuse' },
});
await device.setURLBlacklist([
// ...
]);
});

Spectron with mocha and chai does not work

I am trying to write a tests with Spectron for our Electron App, but I am running into problems with the setup. I use the classical setup with chai. I have one file which contains setup code:
const path = require("path");
const { Application } = require("spectron");
module.exports = {
initialiseSpectron: () => {
let electronPath = path.join(__dirname, "../../node_modules", ".bin", "electron");
if (process.platform == "win32") {
electronPath += ".cmd";
}
return new Application({
path: electronPath,
args: [path.join(__dirname, "../index.ts"), path.join(__dirname, "../../package.json")],
env: {
ELECTRON_ENABLE_LOGGING: true,
ELECTRON_ENABLE_STACK_DUMPING: true,
NODE_ENV: "development"
},
startTimeout: 10000,
chromeDriverLogPath: "../chromedriverlog.txt"
});
},
sleep: time => new Promise(resolve => setTimeout(resolve, time))
};
And then the test itself:
const chaiAsPromised = require("chai-as-promised");
const chai = require("chai");
chai.should();
chai.use(chaiAsPromised);
const testHelper = require("./initialise");
const app = testHelper.initialiseSpectron();
// Setup Promises and start Application
before(() => app.start());
// Tear down App after Tests are finished
after(() => {
if (app && app.isRunning()) {
return app.stop();
}
});
describe("Login", () => {
it("opens a window", function() {
return app.client
.waitUntilWindowLoaded()
.getWindowCount()
.should.eventually.equal(1);
});
it("tests the title", () =>
app.client
.waitUntilWindowLoaded()
.getTitle()
.should.eventually.equal("VIPFY"));
});
My problem is that I always get this error:
1) "before all" hook in "{root}"
0 passing (2s)
1 failing
1) "before all" hook in "{root}":
Error: Timeout of 2000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves.
So it looks like the App does not start. But that is not true. The App Window opens, but it seems like the test does not recognize that. I have already tried changing the path using all kinds of syntax. But nothing worked. What am I missing?
Have you tried to increase the timeout for mocha?
Sometimes I had it fail first time, then worked on the second try.
See a working sample here with Electron 6:
https://github.com/florin05/electron-spectron-example

Is it possible to run same testcafe fixture with different preconditions

Could somebody help with finding the most adequate way to run the same fixture with different preconditions using testcafe (fixture beforeEach)?
Given I have following fixture:
fixture('[Test] My Page')
.meta({ env: 'aat', mobile: 'true', author: 'me' })
.beforeEach(async t => {
await t.navigateTo(myPage.url)
await waitForReact(10000)
})
test('Page should load all data successfully', async t => {
const { Query: queries } = await t.ctx.graphQLTools.mockManagementClient.getCalls()
for (const q of Object.keys(queries)) {
for (const { args, context } of Object.keys(queries[q])) {
await t.expect(queries[q][args]).typeOf('undefined')
await t.expect(queries[q][context]).typeOf('undefined')
}
}
})
In the code example above I need to run this test for several times:
I'm an anonymous user
I'm the admin user
I'm a default user
I've tried the following code, but it looks cumbersome and will create complexity when I have more than 1 test in the fixture that needs to be checked(taken from https://testcafe-discuss.devexpress.com/t/multiple-execution-of-one-test-with-different-data/219):
const a = ['anonymous', 'logged In']
for (const user of a) {
test.meta({ wip: 'true' })(`Page should load all data successfully for ${user} user`, async t => {
if (R.equals('logged In', user)) {
await helpers.setCookie({ key: 'access_token', value: '123123123' })
await helpers.reloadPage()
}
const { Query: queries } = await t.ctx.graphQLTools.mockManagementClient.getCalls()
await t.expect(helpers.isEqlToEmptyQuery(queries.basket)).eql(true)
await t.expect(helpers.isEqlToEmptyQuery(queries.categories)).eql(true)
if (R.equals('logged In', user)) {
await t.expect(helpers.isEqlToEmptyQuery(queries.customer)).eql(true)
}
})
}
Is there a way I can run the whole fixture for 2-3 times with different beforeEach fixture hooks?
One simple solution could be to add three npm scripts in package.json:
"test": "testcafe chrome <all testcafe options>"
"test-anonymous": "npm test -- --user=anonymous"
"test-admin": "npm test -- --user=admin"
"test-default": "npm test -- --user=default"
Then read this custom command-line option in the beforeEach as I explained in Testcafe - Test command line argument outside test case

Cucumber + Protractor - timed out error while executing the steps

I am using cucumber and protractor to write behavioural driven tests.My scenarios and all the steps will pass but at the end it will show timed out error. The homepage will get loaded for the first step and later it will not perform any steps described in the steps definition file. Once the page is loaded it should click on the tabs. I have mentioned these steps in step definition file.But these steps are not executed and it will show all the steps passed in the console. I followed this link for reference https://semaphoreci.com/community/tutorials/getting-started-with-protractor-and-cucumber
This is the error message
Please find the sample code below.
//sample.feature
Feature: The Dashboard has 2 tabs, Statistics and Results
Scenario: I want to have 2 tabs with Displayed text "Statistics" and "Results" on the homepage
Given I go to Dashboard homepage
And I click on the "#/results"
Then the Results page is displayed
And I click on the "#/Statistics" tab
Then the Statistics page is displayed
//menu.steps.js
var chai = require('chai');
var chaiAsPromised = require('chai-as-promised');
chai.use(chaiAsPromised);
var expect = chai.expect;
module.exports = function() {
this.Given(/^I go to Dashboard homepage$/, function() {
browser.get('http://localhost:8100/#/');
browser.waitForAngular();
});
this.Then(/^I click on the "([^"]*)"$/,function(arg1){
element(by.css('[href="#/results"]')).click();
});
this.Then(/^the results page is displayed$/, () => {
browser.get('http://localhost:8100/#/results');
});
this.When(/^I click on the "([^"]*)" tab$/, function(arg1) {
element(by.css('[href="#/statistics"]')).click();
});
this.Then(/^the statistics page is displayed$/, () =>{
browser.get('http://localhost:8100/#/statistics');
});
//cucumber.conf.js
exports.config = {
framework: 'custom', // set to "custom" instead of cucumber.
frameworkPath: require.resolve('protractor-cucumber-framework'),
seleniumAddress: 'http://localhost:4444/wd/hub',
specs: ['test/e2e/cucumber/*.feature'],
capabilities: {
'browserName': 'firefox',
},
baseUrl: 'http://localhost:8100/#/',
// cucumber command line options
cucumberOpts: {
require: ['test/e2e/cucumber/*.steps.js'], // require step definition files before executing features
tags: [], // <string[]> (expression) only execute the features or scenarios with tags matching the expression
strict: true, // <boolean> fail if there are any undefined or pending steps
format: ["pretty"], // <string[]> (type[:path]) specify the output format, optionally supply PATH to redirect formatter output (repeatable)
dryRun: false, // <boolean> invoke formatters without executing steps
compiler: [] // <string[]> ("extension:module") require files with the given EXTENSION after requiring MODULE (repeatable)
},
onPrepare: function () {
browser.manage().window().maximize(); // maximize the browser before executing the feature files
},
resultJsonOutputFile: './test/e2e/results.json'
}
If you are using CucumberJS, you can choose to use callbacks or promises. In your code, you didn't use one of them. Below, you will find an example of your steps how you can use promises, of callbacks.
Be aware of the fact that you if you implement one of the 2 solutions, your tests could still fail, but that has more to do with the test implementation instead of the solution for callbacks / promises
// With promises
module.exports = function() {
this.Given(/^I go to Dashboard homepage$/, function() {
browser.get('http://localhost:8100/#/');
return browser.waitForAngular();
});
this.Then(/^I click on the "([^"]*)"$/, function(arg1) {
return element(by.css('[href="#/results"]')).click();
});
this.Then(/^the results page is displayed$/, () => {
return browser.get('http://localhost:8100/#/results');
});
this.When(/^I click on the "([^"]*)" tab$/, function(arg1) {
return element(by.css('[href="#/statistics"]')).click();
});
this.Then(/^the statistics page is displayed$/, () => {
return browser.get('http://localhost:8100/#/statistics');
});
}
// With callbacks
module.exports = function() {
this.Given(/^I go to Dashboard homepage$/, function(done) {
browser.get('http://localhost:8100/#/');
browser.waitForAngular().then(done);
});
this.Then(/^I click on the "([^"]*)"$/, function(arg1, done) {
element(by.css('[href="#/results"]')).click().then(done);
});
this.Then(/^the results page is displayed$/, (done) => {
browser.get('http://localhost:8100/#/results').then(done);
});
this.When(/^I click on the "([^"]*)" tab$/, function(arg1, done) {
element(by.css('[href="#/statistics"]')).click().then(done);
});
this.Then(/^the statistics page is displayed$/, (done) => {
browser.get('http://localhost:8100/#/statistics').then(done);
});
}

How to test an Electron app with selenium webdriver

I have read the documentation and I have followed the tutorial step by step and I only have managed to run the app.
Documentation: http://electron.atom.io/docs/tutorial/using-selenium-and-webdriver/
The connection with chromedriver I cannot make it work, when I launch the test and try click a simple button I get this:
Error: ChromeDriver did not start within 5000ms at Error (native)
at node_modules/spectron/lib/chrome-driver.js:58:25 at
Request._callback (node_modules/spectron/lib/chrome-driver.js:116:45)
at Request.self.callback
(node_modules/spectron/node_modules/request/request.js:200:22) at
Request.
(node_modules/spectron/node_modules/request/request.js:1067:10) at
IncomingMessage.
(node_modules/spectron/node_modules/request/request.js:988:12) at
endReadableNT (_stream_readable.js:913:12) at _combinedTickCallback
(internal/process/next_tick.js:74:11) at process._tickCallback
(internal/process/next_tick.js:98:9)
My code:
"use strict";
require("co-mocha");
var Application = require('spectron').Application;
var assert = require('assert');
const webdriver = require('selenium-webdriver');
const driver = new webdriver.Builder()
.usingServer('http://127.0.0.1:9515')
.withCapabilities({
chromeOptions: {
binary: "./appPath/app"
}
})
.forBrowser('electron')
.build();
describe('Application launch', function () {
this.timeout(100000);
var app;
beforeEach(function () {
app = new Application({
path: "./appPath/app"
});
return app.start();
});
afterEach(function () {
if (app && app.isRunning()) {
return app.stop();
}
});
it('click a button', function* () {
yield driver.sleep(5000);
yield driver.findElement(webdriver.By.css(".classSelector")).click();
});
});
Thanks and sorry for my English.
I recommend you to use Spectron. which is a less painful way of testing your electron app. in my opinion perfect combination is using it with Ava test framework, which allows the concurrently test.
async & await is also another big win. which allows you to have so clean test cases.
and also if you have a test which needs to happen serial, you can use test.serial
test.serial('login as new user', async t => {
let app = t.context.app
app = await loginNewUser(app)
await util.screenshotCreateOrCompare(app, t, 'new-user-mission-view-empty')
})
test.serial('Can Navigate to Preference Page', async t => {
let app = t.context.app
await app.client.click('[data-test="preference-button"]')
await util.screenshotCreateOrCompare(app, t, 'new-user-preference-page-empty')
})
and just for reference; my helper test cases.
test.before(async t => {
app = util.createApp()
app = await util.waitForLoad(app, t)
})
test.beforeEach(async t => {
t.context.app = app
})
test.afterEach(async t => {
console.log('test complete')
})
// CleanUp
test.after.always(async t => {
// This runs after each test and other test hooks, even if they
failed
await app.client.localStorage('DELETE', 'user')
console.log('delete all files')
const clean = await exec('rm -rf /tmp/DesktopTest')
await clean.stdout.on('data', data => {
console.log(util.format('clean', data))
})
await app.client.close()
await app.stop()
})
util function,
// Returns a promise that resolves to a Spectron Application once the app has loaded.
// Takes a Ava test. Makes some basic assertions to verify that the app loaded correctly.
function createApp (t) {
return new Application({
path: path.join(__dirname, '..', 'node_modules', '.bin',
'electron' + (process.platform === 'win32' ? '.cmd' : '')),
// args: ['-r', path.join(__dirname, 'mocks.js'), path.join(__dirname, '..')],
env: {NODE_ENV: 'test'},
waitTimeout: 10e3
})
}
First off, Spectron (which is a wrapper for WebdriverIO) and WebdriverJS (which is part of Selenium-Webdriver) are two different frameworks, you only need to use one of them for your tests.
If you are using WebdriverJS, then you need to run ./node_modules/.bin/chromedriver in this step: http://electron.atom.io/docs/tutorial/using-selenium-and-webdriver/#start-chromedriver
I could get ChromeDriver working by adding a proxy exception in my terminal.
export {no_proxy,NO_PROXY}="127.0.0.1"