feat: dashboard orchestration (#14925)

* test orchestration

* bump json-schemas

* fix unit

* fix server-ct build

* add @types/node

* add runtime to resolved config, send resolved config to postInstanceTests

* add missing fixture spec, support sending test config for skipped and proper title

* spec prior., refactor recording tests, update specs

* remove unneded utils, fix e2e_record_spec, rename testConfig, fix unit

* fix bug with spec SKIP not stopping test execution

* bump json-schemas

* bump json schemas 2

* refactor exit early and error logic, add runnerCapabilities + bump json schemas

* update yarn.lock

* fix referenceError

* fix: skipAction -> skipSpecAction, muteAction -> muteTestAction

* exit with non-zero for canceled runs, improve messaging for canceled runs

* fixup unit tests

* colorize run cancelation messages
This commit is contained in:
Ben Kucera
2021-03-15 10:42:11 -04:00
committed by GitHub
parent 614c40955c
commit c0d063c7dd
35 changed files with 1945 additions and 1411 deletions
@@ -28,7 +28,7 @@ describe('src/cy/commands/angular', () => {
cy.state('window').angular = this.angular
})
it('throws when cannot find angular', (done) => {
it('throws when cannot find angular', { retries: 2 }, (done) => {
delete cy.state('window').angular
cy.on('fail', (err) => {
@@ -44,20 +44,22 @@ function mutateConfiguration (testConfigOverride, config, env) {
return restoreConfigFn
}
function getResolvedTestConfigOverride (test) {
// this is called during test onRunnable time
// in order to resolve the test config upfront before test runs
export function getResolvedTestConfigOverride (test) {
let curParent = test.parent
const cfgs = [test.cfg]
const testConfig = [test._testConfig]
while (curParent) {
if (curParent.cfg) {
cfgs.push(curParent.cfg)
if (curParent._testConfig) {
testConfig.push(curParent._testConfig)
}
curParent = curParent.parent
}
return _.reduceRight(cfgs, (acc, cfg) => _.extend(acc, cfg), {})
return _.reduceRight(testConfig, (acc, opts) => _.extend(acc, opts), {})
}
class TestConfigOverride {
@@ -65,7 +67,7 @@ class TestConfigOverride {
restoreAndSetTestConfigOverrides (test, config, env) {
if (this.restoreTestConfigFn) this.restoreTestConfigFn()
const resolvedTestConfig = getResolvedTestConfigOverride(test)
const resolvedTestConfig = test._testConfig || {}
this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env)
}
+1
View File
@@ -134,6 +134,7 @@ class $Cypress {
_.extend(this, browserInfo(config))
this.state = $SetterGetter.create({})
this.originalConfig = _.cloneDeep(config)
this.config = $SetterGetter.create(config)
this.env = $SetterGetter.create(env)
this.getFirefoxGcInterval = $FirefoxForcedGc.createIntervalGetter(this)
+8 -9
View File
@@ -33,9 +33,10 @@ const suiteAfterEach = Suite.prototype.afterEach
delete window.mocha
delete window.Mocha
function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn) {
function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn, _testConfig) {
const ret = fn.apply(ctx, mochaArgs)
ret._testConfig = _testConfig
ret.originalTitle = originalTitle
return ret
@@ -64,13 +65,11 @@ function overloadMochaFnForConfig (fnName, specWindow) {
const origFn = subFn ? _fn[subFn] : _fn
if (args.length > 2 && _.isObject(args[1])) {
const opts = _.defaults({}, args[1], {
browser: null,
})
const _testConfig = _.extend({}, args[1])
const mochaArgs = [args[0], args[2]]
const configMatchesBrowser = opts.browser == null || Cypress.isBrowser(opts.browser, `${fnType} config value \`{ browser }\``)
const configMatchesBrowser = _testConfig.browser == null || Cypress.isBrowser(_testConfig.browser, `${fnType} config value \`{ browser }\``)
if (!configMatchesBrowser) {
// TODO: this would mess up the dashboard since it would be registered as a new test
@@ -84,15 +83,15 @@ function overloadMochaFnForConfig (fnName, specWindow) {
this.skip()
}
return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn)
return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn, _testConfig)
}
return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'])
return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'], _testConfig)
}
const ret = origFn.apply(this, mochaArgs)
ret.cfg = opts
ret._testConfig = _testConfig
return ret
}
@@ -327,7 +326,7 @@ function patchTestClone () {
const ret = testClone.apply(this, arguments)
// carry over testConfigOverrides
ret.cfg = this.cfg
ret._testConfig = this._testConfig
// carry over test.id
ret.id = this.id
+24 -2
View File
@@ -9,6 +9,7 @@ const $Log = require('./log')
const $utils = require('./utils')
const $errUtils = require('./error_utils')
const $stackUtils = require('./stack_utils')
const { getResolvedTestConfigOverride } = require('../cy/testConfigOverrides')
const mochaCtxKeysRe = /^(_runnable|test)$/
const betweenQuotesRe = /\"(.+?)\"/
@@ -18,8 +19,7 @@ const TEST_BEFORE_RUN_EVENT = 'runner:test:before:run'
const TEST_AFTER_RUN_EVENT = 'runner:test:after:run'
const RUNNABLE_LOGS = 'routes agents commands hooks'.split(' ')
const RUNNABLE_PROPS = 'id order title root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
const RUNNABLE_PROPS = '_testConfig id order title _titlePath root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
const debug = require('debug')('cypress:driver:runner')
const fire = (event, runnable, Cypress) => {
@@ -493,6 +493,17 @@ const normalizeAll = (suite, initialTests = {}, setTestsById, setTests, onRunnab
setTests(testsArr)
}
// generate the diff of the config after spec has been executed
// e.g. config changes via Cypress.config('...')
normalizedSuite.runtimeConfig = {}
_.map(Cypress.config(), (v, key) => {
if (_.isEqual(v, Cypress.originalConfig[key])) {
return null
}
normalizedSuite.runtimeConfig[key] = v
})
return normalizedSuite
}
@@ -548,6 +559,17 @@ const normalize = (runnable, tests, initialTests, onRunnable, onLogsById, getRun
// and collections
const wrappedRunnable = wrapAll(runnable)
if (runnable.type === 'test') {
const cfg = getResolvedTestConfigOverride(runnable)
if (_.size(cfg)) {
runnable._testConfig = cfg
wrappedRunnable._testConfig = cfg
}
wrappedRunnable._titlePath = runnable.titlePath()
}
if (prevAttempts) {
wrappedRunnable.prevAttempts = prevAttempts
}
+1
View File
@@ -47,6 +47,7 @@
"@babel/preset-env": "^7.12.1",
"@packages/driver": "0.0.0-development",
"@percy/cypress": "2.3.4",
"@types/node": "12.12.50",
"@types/sockjs-client": "1.1.0",
"babel-loader": "8.1.0",
"clean-webpack-plugin": "^3.0.0",
+7 -5
View File
@@ -269,6 +269,12 @@ const eventManager = {
Cypress.runner.setStartTime(state.startTime)
}
if (config.isTextTerminal && !state.currentId) {
// we are in run mode and it's the first load
// store runnables in backend and maybe send to dashboard
return ws.emit('set:runnables:and:maybe:record:tests', runnables, run)
}
if (state.currentId) {
// if we have a currentId it means
// we need to tell the Cypress to skip
@@ -276,11 +282,7 @@ const eventManager = {
Cypress.runner.resumeAtTest(state.currentId, state.emissions)
}
if (config.isTextTerminal && !state.currentId) {
ws.emit('set:runnables', runnables, run)
} else {
run()
}
run()
})
},
})
+1 -1
View File
@@ -260,7 +260,7 @@ function createCypress (defaultOptions = {}) {
url: opts.visitUrl,
} })
.withArgs('set:runnables')
.withArgs('set:runnables:and:maybe:record:tests')
.callsFake((...args) => {
setRunnablesStub(...args)
_.last(args)()
+7 -5
View File
@@ -340,6 +340,12 @@ const eventManager = {
Cypress.runner.setStartTime(state.startTime)
}
if (config.isTextTerminal && !state.currentId) {
// we are in run mode and it's the first load
// store runnables in backend and maybe send to dashboard
return ws.emit('set:runnables:and:maybe:record:tests', runnables, run)
}
if (state.currentId) {
// if we have a currentId it means
// we need to tell the Cypress to skip
@@ -347,11 +353,7 @@ const eventManager = {
Cypress.runner.resumeAtTest(state.currentId, state.emissions)
}
if (config.isTextTerminal && !state.currentId) {
ws.emit('set:runnables', runnables, run)
} else {
run()
}
run()
})
},
})
+361 -362
View File
@@ -268,232 +268,6 @@ Alternatively, you can create a new project using the Desktop Application.
https://on.cypress.io/dashboard
`
exports['e2e record api interaction errors create run 500 warns and does not create or update instances 1'] = `
Warning: We encountered an error talking to our servers.
This run will not be recorded.
This error will not alter the exit code.
StatusCodeError: 500 - "Internal Server Error"
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass_spec.js) │
│ Searched: cypress/integration/record_pass* │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass_spec.js (1 of 1)
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: true │
│ Duration: X seconds │
│ Spec Ran: record_pass_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022)
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✔ All specs passed! XX:XX 2 1 - 1 -
`
exports['e2e record api interaction errors create instance does not update instance 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass_spec.js) │
│ Searched: cypress/integration/record_pass* │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
Warning: We encountered an error talking to our servers.
This run will not be recorded.
This error will not alter the exit code.
StatusCodeError: 500 - "Internal Server Error"
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass_spec.js (1 of 1)
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: true │
│ Duration: X seconds │
│ Spec Ran: record_pass_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022)
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✔ All specs passed! XX:XX 2 1 - 1 -
───────────────────────────────────────────────────────────────────────────────────────────────────────
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
`
exports['e2e record api interaction errors update instance does not update instance stdout 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass_spec.js) │
│ Searched: cypress/integration/record_pass* │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass_spec.js (1 of 1)
Estimated: 8 seconds
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: true │
│ Duration: X seconds │
│ Estimated: 8 seconds │
│ Spec Ran: record_pass_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022)
(Uploading Results)
Warning: We encountered an error talking to our servers.
This run will not be recorded.
This error will not alter the exit code.
StatusCodeError: 500 - "Internal Server Error"
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✔ All specs passed! XX:XX 2 1 - 1 -
───────────────────────────────────────────────────────────────────────────────────────────────────────
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
`
exports['e2e record api interaction errors update instance stdout warns but proceeds 1'] = `
@@ -774,26 +548,6 @@ exports['e2e record api interaction errors uploading assets warns but proceeds 1
`
exports['e2e record misconfiguration errors and exits when no browser found 1'] = `
Can't run because you've entered an invalid browser name.
Browser: 'browserDoesNotExist' was not found on your system or is not supported by Cypress.
Cypress supports the following browsers:
- chrome
- chromium
- edge
- electron
- firefox
You can also use a custom browser: https://on.cypress.io/customize-browsers
Available browsers found on your system are:
- browser1
- browser2
- browser3
`
exports['e2e record misconfiguration errors and exits when no specs found 1'] = `
Can't run because no spec files were found.
@@ -1199,75 +953,6 @@ StatusCodeError: 422
"message": "An unknown message here from the server."
}
`
exports['e2e record api interaction errors create run 500 warns but proceeds when grouping without parallelization 1'] = `
Warning: We encountered an error talking to our servers.
This run will not be recorded.
This error will not alter the exit code.
StatusCodeError: 500 - "Internal Server Error"
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass_spec.js) │
│ Searched: cypress/integration/record_pass* │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass_spec.js (1 of 1)
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: true │
│ Duration: X seconds │
│ Spec Ran: record_pass_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022)
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✔ All specs passed! XX:XX 2 1 - 1 -
`
exports['e2e record api interaction errors create run 500 does not proceed and exits with error when parallelizing 1'] = `
@@ -2185,13 +1870,8 @@ exports['e2e record passing passes 2'] = [
},
"tests": [
{
"testId": "r3",
"title": [
"record fails",
"fails 1"
],
"clientId": "r3",
"state": "failed",
"body": "function () {}",
"displayError": "Error: foo\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `record fails`\n [stack trace lines]",
"attempts": [
{
@@ -2228,13 +1908,8 @@ exports['e2e record passing passes 2'] = [
]
},
{
"testId": "r4",
"title": [
"record fails",
"is skipped"
],
"clientId": "r4",
"state": "skipped",
"body": "function () {}",
"displayError": null,
"attempts": [
{
@@ -2249,18 +1924,8 @@ exports['e2e record passing passes 2'] = [
]
}
],
"error": null,
"exception": null,
"video": true,
"hooks": [
{
"hookId": "h1",
"hookName": "before each",
"title": [
"\"before each\" hook"
],
"body": "function () {\n throw new Error('foo');\n }"
}
],
"screenshots": [
{
"screenshotId": "some-random-id",
@@ -2272,7 +1937,6 @@ exports['e2e record passing passes 2'] = [
"width": 1280
}
],
"cypressConfig": {},
"reporterStats": {
"suites": 1,
"tests": 1,
@@ -2298,13 +1962,8 @@ exports['e2e record passing passes 2'] = [
},
"tests": [
{
"testId": "r3",
"title": [
"record pass",
"passes"
],
"clientId": "r3",
"state": "passed",
"body": "function () {\n cy.visit('/scrollable.html');\n cy.viewport(400, 400);\n cy.get('#box');\n cy.screenshot('yay it passes');\n }",
"displayError": null,
"attempts": [
{
@@ -2325,13 +1984,8 @@ exports['e2e record passing passes 2'] = [
]
},
{
"testId": "r4",
"title": [
"record pass",
"is pending"
],
"clientId": "r4",
"state": "pending",
"body": "",
"displayError": null,
"attempts": [
{
@@ -2346,9 +2000,8 @@ exports['e2e record passing passes 2'] = [
]
}
],
"error": null,
"exception": null,
"video": true,
"hooks": [],
"screenshots": [
{
"screenshotId": "some-random-id",
@@ -2360,7 +2013,6 @@ exports['e2e record passing passes 2'] = [
"width": 400
}
],
"cypressConfig": {},
"reporterStats": {
"suites": 1,
"tests": 2,
@@ -2386,12 +2038,8 @@ exports['e2e record passing passes 2'] = [
},
"tests": [
{
"testId": "r2",
"title": [
"An uncaught error was detected outside of a test"
],
"clientId": "r2",
"state": "failed",
"body": "() => {\n throw err;\n }",
"displayError": "Error: The following error originated from your test code, not from Cypress.\n\n > instantly fails\n\nWhen Cypress detects uncaught errors originating from your test code it will automatically fail the current test.\n\nCypress could not associate this error to any specific test.\n\nWe dynamically generated a new test to display this failure.\n [stack trace lines]",
"attempts": [
{
@@ -2425,9 +2073,8 @@ exports['e2e record passing passes 2'] = [
]
}
],
"error": null,
"exception": null,
"video": true,
"hooks": [],
"screenshots": [
{
"screenshotId": "some-random-id",
@@ -2439,7 +2086,6 @@ exports['e2e record passing passes 2'] = [
"width": 1280
}
],
"cypressConfig": {},
"reporterStats": {
"suites": 0,
"tests": 1,
@@ -2532,3 +2178,356 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing
`
exports['e2e record api interaction errors create run 500 errors and exits 1'] = `
We encountered an unexpected error talking to our servers.
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api interaction errors create run 500 when grouping without parallelization errors and exits 1'] = `
We encountered an unexpected error talking to our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api interaction errors create instance 500 without parallelization - does not proceed 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 2 found (a_record.spec.js, b_record.spec.js) │
│ Searched: cypress/integration/*_record.spec.js │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
We encountered an unexpected error talking to our servers.
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api interaction errors create instance errors and exits on createInstance error 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 2 found (a_record_instantfail.spec.js, config_record_spec.js) │
│ Searched: cypress/integration/*_record_* │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
We encountered an unexpected error talking to our servers.
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api interaction errors postInstanceTests without parallelization errors and exits 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 2 found (a_record.spec.js, b_record.spec.js) │
│ Searched: cypress/integration/*_record.spec* │
│ Params: Tag: false, Group: foo, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: a_record.spec.js (1 of 2)
Estimated: 8 seconds
We encountered an unexpected error talking to our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: 1
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api interaction errors postInstanceTests with parallelization errors and exits 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 2 found (a_record.spec.js, b_record.spec.js) │
│ Searched: cypress/integration/*_record.spec.js │
│ Params: Tag: false, Group: foo, Parallel: true │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: a_record.spec.js (1 of 2)
Estimated: 8 seconds
We encountered an unexpected error talking to our servers.
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api interaction errors postInstanceResults errors and exits in serial 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass_spec.js) │
│ Searched: cypress/integration/record_pass* │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass_spec.js (1 of 1)
Estimated: 8 seconds
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: true │
│ Duration: X seconds │
│ Estimated: 8 seconds │
│ Spec Ran: record_pass_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022)
(Uploading Results)
We encountered an unexpected error talking to our servers.
The server's response was:
StatusCodeError: 500 - "Internal Server Error"
`
exports['e2e record api skips specs records tests and exits without executing 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 2 found (a_record_instantfail.spec.js, b_record.spec.js) │
│ Searched: cypress/integration/a_record_instantfail.spec.js, cypress/integration/b_record.spe │
│ c.js │
│ Params: Tag: false, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: a_record_instantfail.spec.js (1 of 2)
Estimated: 8 seconds
This spec and its tests were skipped because the run has been canceled.
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: b_record.spec.js (2 of 2)
Estimated: 8 seconds
b spec
✓ b test
1 passing
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 1 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 0 │
│ Skipped: 0 │
│ Screenshots: 0 │
│ Video: false │
│ Duration: X seconds │
│ Estimated: 8 seconds │
│ Spec Ran: b_record.spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Uploading Results)
- Nothing to Upload
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ - a_record_instantfail.spec.js SKIPPED - - - - - │
├────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ✔ b_record.spec.js XX:XX 1 1 - - - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
- The run was canceled XX:XX 1 1 - - -
───────────────────────────────────────────────────────────────────────────────────────────────────────
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
Exiting with non-zero exit code because the run was canceled.
`
exports['e2e record api skips specs records tests and exits without executing in parallel 1'] = `
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 2 found (a_record_instantfail.spec.js, b_record.spec.js) │
│ Searched: cypress/integration/a_record_instantfail.spec.js, cypress/integration/b_record.spe │
│ c.js │
│ Params: Tag: false, Group: abc, Parallel: true │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: a_record_instantfail.spec.js (1 of 2)
Estimated: 8 seconds
This spec and its tests were skipped because the run has been canceled.
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: b_record.spec.js (2 of 2)
Estimated: 8 seconds
b spec
✓ b test
1 passing
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 1 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 0 │
│ Skipped: 0 │
│ Screenshots: 0 │
│ Video: false │
│ Duration: X seconds │
│ Estimated: 8 seconds │
│ Spec Ran: b_record.spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Uploading Results)
- Nothing to Upload
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ - a_record_instantfail.spec.js SKIPPED - - - - - │
├────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ✔ b_record.spec.js XX:XX 1 1 - - - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
- The run was canceled XX:XX 1 1 - - -
───────────────────────────────────────────────────────────────────────────────────────────────────────
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
Exiting with non-zero exit code because the run was canceled.
`
+42 -21
View File
@@ -22,6 +22,11 @@ let DELAYS = [
TWO_MINUTES,
]
const runnerCapabilities = {
'dynamicSpecsInSerialMode': true,
'skipSpecAction': true,
}
let responseCache = {}
intervals = process.env.API_RETRY_INTERVALS
@@ -199,19 +204,22 @@ module.exports = {
},
createRun (options = {}) {
const body = _.pick(options, [
'ci',
'specs',
'commit',
'group',
'platform',
'parallel',
'ciBuildId',
'projectId',
'recordKey',
'specPattern',
'tags',
])
const body = {
..._.pick(options, [
'ci',
'specs',
'commit',
'group',
'platform',
'parallel',
'ciBuildId',
'projectId',
'recordKey',
'specPattern',
'tags',
]),
runnerCapabilities,
}
return rp.post({
body,
@@ -249,6 +257,22 @@ module.exports = {
.catch(tagError)
},
postInstanceTests (options = {}) {
const { instanceId, ...body } = options
return rp.post({
url: apiRoutes.instanceTests(instanceId),
json: true,
timeout: SIXTY_SECONDS,
headers: {
'x-route-version': '1',
},
body,
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
updateInstanceStdout (options = {}) {
return rp.put({
url: apiRoutes.instanceStdout(options.instanceId),
@@ -262,23 +286,20 @@ module.exports = {
.catch(tagError)
},
updateInstance (options = {}) {
return rp.put({
url: apiRoutes.instance(options.instanceId),
postInstanceResults (options = {}) {
return rp.post({
url: apiRoutes.instanceResults(options.instanceId),
json: true,
timeout: options.timeout != null ? options.timeout : SIXTY_SECONDS,
headers: {
'x-route-version': '3',
'x-route-version': '1',
},
body: _.pick(options, [
'stats',
'tests',
'error',
'exception',
'video',
'hooks',
'stdout',
'screenshots',
'cypressConfig',
'reporterStats',
]),
})
+10
View File
@@ -755,6 +755,16 @@ module.exports = {
, {})
},
getResolvedRuntimeConfig (config, runtimeConfig) {
const resolvedRuntimeFields = _.mapValues(runtimeConfig, (v) => ({ value: v, from: 'runtime' }))
return {
...config,
...runtimeConfig,
resolved: { ...config.resolved, ...resolvedRuntimeFields },
}
},
getNameFromRoot (root = '') {
return path.basename(root)
},
+15 -1
View File
@@ -13,6 +13,7 @@ const R = require('ramda')
const Promise = require('bluebird')
const debug = require('debug')('cypress:server:cypress')
const argsUtils = require('./util/args')
const chalk = require('chalk')
const warning = (code, args) => {
return require('./errors').warning(code, args)
@@ -283,7 +284,20 @@ module.exports = {
// run headlessly and exit
// with num of totalFailed
return this.runElectron(mode, options)
.get('totalFailed')
.then((results) => {
if (results.runs) {
const isCanceled = results.runs.filter((run) => run.skippedSpec).length
if (isCanceled) {
// eslint-disable-next-line no-console
console.log(chalk.magenta('\n Exiting with non-zero exit code because the run was canceled.'))
return 1
}
}
return results.totalFailed
})
.then(exit)
.catch(exitErr)
+15
View File
@@ -166,6 +166,8 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) {
return `Timed out waiting for the browser to connect. ${arg1}`
case 'TESTS_DID_NOT_START_FAILED':
return 'The browser never connected. Something is wrong. The tests cannot run. Aborting...'
case 'DASHBOARD_CANCEL_SKIPPED_SPEC':
return '\n This spec and its tests were skipped because the run has been canceled.'
case 'DASHBOARD_API_RESPONSE_FAILED_RETRYING':
return stripIndent`\
We encountered an unexpected error talking to our servers.
@@ -190,6 +192,19 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) {
The server's response was:
${arg1.response}`
case 'DASHBOARD_CANNOT_PROCEED_IN_SERIAL':
return stripIndent`\
We encountered an unexpected error talking to our servers.
${displayFlags(arg1.flags, {
group: '--group',
ciBuildId: '--ciBuildId',
})}
The server's response was:
${arg1.response}`
case 'DASHBOARD_UNKNOWN_INVALID_REQUEST':
return stripIndent`\
+149 -69
View File
@@ -11,12 +11,14 @@ const logger = require('../logger')
const errors = require('../errors')
const capture = require('../capture')
const upload = require('../upload')
const Config = require('../config')
const env = require('../util/env')
const keys = require('../util/keys')
const terminal = require('../util/terminal')
const humanTime = require('../util/human_time')
const ciProvider = require('../util/ci_provider')
const settings = require('../util/settings')
const testsUtils = require('../util/tests_utils')
const onBeforeRetry = (details) => {
return errors.warning(
@@ -60,6 +62,23 @@ const warnIfProjectIdButNoRecordOption = (projectId, options) => {
}
}
const throwDashboardCannotProceed = ({ parallel, ciBuildId, group, err }) => {
const errMsg = parallel ? 'DASHBOARD_CANNOT_PROCEED_IN_PARALLEL' : 'DASHBOARD_CANNOT_PROCEED_IN_SERIAL'
const errToThrow = errors.get(errMsg, {
response: err,
flags: {
group,
ciBuildId,
},
})
// tells error handler to exit immediately without running anymore specs
errToThrow.isFatalApiErr = true
throw errToThrow
}
const throwIfIndeterminateCiBuildId = (ciBuildId, parallel, group) => {
if ((!ciBuildId && !ciProvider.provider()) && (parallel || group)) {
errors.throw(
@@ -190,29 +209,33 @@ const updateInstanceStdout = (options = {}) => {
}).finally(capture.restore)
}
const updateInstance = (options = {}) => {
const postInstanceResults = (options = {}) => {
const { instanceId, results, group, parallel, ciBuildId } = options
let { stats, tests, hooks, video, screenshots, reporterStats, error } = results
let { stats, tests, video, screenshots, reporterStats, error } = results
video = Boolean(video)
const cypressConfig = options.config
// get rid of the path property
screenshots = _.map(screenshots, (screenshot) => {
return _.omit(screenshot, 'path')
})
tests = tests && _.map(tests, (test) => {
return _.omit({
clientId: test.testId,
...test,
}, 'title', 'body', 'testId')
})
const makeRequest = () => {
return api.updateInstance({
return api.postInstanceResults({
instanceId,
stats,
tests,
error,
exception: error,
video,
hooks,
instanceId,
screenshots,
reporterStats,
cypressConfig,
screenshots,
})
}
@@ -222,25 +245,7 @@ const updateInstance = (options = {}) => {
stack: err.stack,
})
if (parallel) {
return errors.throw('DASHBOARD_CANNOT_PROCEED_IN_PARALLEL', {
response: err,
flags: {
group,
ciBuildId,
},
})
}
errors.warning('DASHBOARD_CANNOT_CREATE_RUN_OR_INSTANCE', err)
// dont log exceptions if we have a 503 status code
if (err.statusCode !== 503) {
return logException(err)
.return(null)
}
return null
throwDashboardCannotProceed({ parallel, ciBuildId, group, err })
})
}
@@ -524,23 +529,7 @@ const createRun = Promise.method((options = {}) => {
}
}
default:
if (parallel) {
return errors.throw('DASHBOARD_CANNOT_PROCEED_IN_PARALLEL', {
response: err,
flags: {
group,
ciBuildId,
},
})
}
// warn the user that assets will be not recorded
errors.warning('DASHBOARD_CANNOT_CREATE_RUN_OR_INSTANCE', err)
// report on this exception
// and return null
return logException(err)
.return(null)
throwDashboardCannotProceed({ parallel, ciBuildId, group, err })
}
})
})
@@ -566,30 +555,41 @@ const createInstance = (options = {}) => {
stack: err.stack,
})
if (parallel) {
return errors.throw('DASHBOARD_CANNOT_PROCEED_IN_PARALLEL', {
response: err,
flags: {
group,
ciBuildId,
},
})
}
throwDashboardCannotProceed({
err,
group,
ciBuildId,
parallel,
})
})
}
errors.warning('DASHBOARD_CANNOT_CREATE_RUN_OR_INSTANCE', err)
const _postInstanceTests = ({
instanceId,
config,
tests,
hooks,
parallel,
ciBuildId,
group,
}) => {
const makeRequest = () => {
return api.postInstanceTests({
instanceId,
config,
tests,
hooks,
})
}
// dont log exceptions if we have a 503 status code
if (err.statusCode !== 503) {
return logException(err)
.return(null)
}
return null
return api.retryWithBackoff(makeRequest, { onBeforeRetry })
.catch((err) => {
throwDashboardCannotProceed({ parallel, ciBuildId, group, err })
})
}
const createRunAndRecordSpecs = (options = {}) => {
const { specPattern, specs, sys, browser, projectId, projectRoot, runAllSpecs, parallel, ciBuildId, group } = options
const { specPattern, specs, sys, browser, projectId, config, projectRoot, runAllSpecs, parallel, ciBuildId, group, project, onError } = options
const recordKey = options.key
// we want to normalize this to an array to send to API
@@ -636,8 +636,7 @@ const createRunAndRecordSpecs = (options = {}) => {
let instanceId = null
const beforeSpecRun = (spec) => {
debug('before spec run %o', { spec })
project.setOnTestsReceived(onTestsReceived)
capture.restore()
captured = capture.stdout()
@@ -653,7 +652,6 @@ const createRunAndRecordSpecs = (options = {}) => {
machineId,
})
.then((resp = {}) => {
resp = resp || {}
instanceId = resp.instanceId
// pull off only what we need
@@ -670,7 +668,7 @@ const createRunAndRecordSpecs = (options = {}) => {
const afterSpecRun = (spec, results, config) => {
// dont do anything if we failed to
// create the instance
if (!instanceId) {
if (!instanceId || results.skippedSpec) {
return
}
@@ -686,7 +684,7 @@ const createRunAndRecordSpecs = (options = {}) => {
// eslint-disable-next-line no-console
console.log('')
return updateInstance({
return postInstanceResults({
group,
config,
results,
@@ -720,9 +718,89 @@ const createRunAndRecordSpecs = (options = {}) => {
})
}
const onTestsReceived = (async (runnables, cb) => {
// we failed createInstance earlier, nothing to do
if (!instanceId) {
return
}
const r = testsUtils.flattenSuiteIntoRunnables(runnables)
const runtimeConfig = runnables.runtimeConfig
const resolvedRuntimeConfig = Config.getResolvedRuntimeConfig(config, runtimeConfig)
const tests = _.chain(r[0])
.uniqBy('id')
.map((v) => {
if (v.originalTitle) {
v._titlePath.splice(-1, 1, v.originalTitle)
}
return _.pick({
...v,
clientId: v.id,
config: v._testConfig || null,
title: v._titlePath,
hookIds: v.hooks.map((hook) => hook.hookId),
},
'clientId', 'body', 'title', 'config', 'hookIds')
})
.value()
const hooks = _.chain(r[1])
.uniqBy('hookId')
.map((v) => {
return _.pick({
...v,
clientId: v.hookId,
title: [v.title],
type: v.hookName,
},
'clientId',
'type',
'title',
'body')
})
.value()
const responseDidFail = {}
const response = await _postInstanceTests({
instanceId,
config: resolvedRuntimeConfig,
tests,
hooks,
parallel,
ciBuildId,
group,
})
.catch((err) => {
onError(err)
return responseDidFail
})
if (response === responseDidFail) {
// dont call the cb, let the browser hang until it's killed
return
}
if (_.some(response.actions, { type: 'SPEC', action: 'SKIP' })) {
errors.warning('DASHBOARD_CANCEL_SKIPPED_SPEC')
// set a property on the response so the browser runner
// knows not to start executing tests
project.emit('end', { skippedSpec: true, stats: {} })
// dont call the cb, let the browser hang until it's killed
return
}
return cb(response)
})
return runAllSpecs({
runUrl,
parallel,
onTestsReceived,
beforeSpecRun,
afterSpecRun,
})
@@ -735,7 +813,9 @@ module.exports = {
createInstance,
updateInstance,
postInstanceResults,
_postInstanceTests,
updateInstanceStdout,
+60 -35
View File
@@ -44,7 +44,7 @@ const gray = (val) => {
}
const colorIf = function (val, c) {
if (val === 0) {
if (val === 0 || val == null) {
val = '-'
c = 'gray'
}
@@ -82,10 +82,16 @@ const formatBrowser = (browser) => {
const formatFooterSummary = (results) => {
const { totalFailed, runs } = results
const isCanceled = _.some(results.runs, { skippedSpec: true })
// pass or fail color
const c = totalFailed ? 'red' : 'green'
const c = isCanceled ? 'magenta' : totalFailed ? 'red' : 'green'
const phrase = (() => {
if (isCanceled) {
return 'The run was canceled'
}
// if we have any specs failing...
if (!totalFailed) {
return 'All specs passed!'
@@ -100,7 +106,7 @@ const formatFooterSummary = (results) => {
})()
return [
formatSymbolSummary(totalFailed),
isCanceled ? '-' : formatSymbolSummary(totalFailed),
color(phrase, c),
gray(duration.format(results.totalDuration)),
colorIf(results.totalTests, 'reset'),
@@ -339,11 +345,21 @@ const renderSummaryTable = (runUrl) => {
_.each(runs, (run) => {
const { spec, stats } = run
const ms = duration.format(stats.wallClockDuration)
const ms = duration.format(stats.wallClockDuration || 0)
const formattedSpec = formatPath(spec.name, getWidth(table2, 1))
if (run.skippedSpec) {
return table2.push([
'-',
formattedSpec, color('SKIPPED', 'gray'),
'-', '-', '-', '-', '-',
])
}
return table2.push([
formatSymbolSummary(stats.failures),
formatPath(spec.name, getWidth(table2, 1)),
formattedSpec,
color(ms, 'gray'),
colorIf(stats.tests, 'reset'),
colorIf(stats.passes, 'green'),
@@ -381,28 +397,27 @@ const renderSummaryTable = (runUrl) => {
}
const iterateThroughSpecs = function (options = {}) {
const { specs, runEachSpec, parallel, beforeSpecRun, afterSpecRun, config } = options
const { specs, runEachSpec, beforeSpecRun, afterSpecRun, config } = options
const serial = () => {
return Promise.mapSeries(specs, runEachSpec)
}
const serialWithRecord = () => {
return Promise
.mapSeries(specs, (spec, index, length) => {
return beforeSpecRun(spec)
.then(({ estimated }) => {
return runEachSpec(spec, index, length, estimated)
})
.tap((results) => {
return afterSpecRun(spec, results, config)
})
})
}
const parallelWithRecord = (runs) => {
const ranSpecs = []
const parallelAndSerialWithRecord = (runs) => {
return beforeSpecRun()
.then(({ spec, claimedInstances, totalInstances, estimated }) => {
.then(({ spec, claimedInstances, totalInstances, estimated, shouldFallbackToOfflineOrder }) => {
// if (!parallel) {
// // NOTE: if we receive the old API which always sends {spec: null},
// // that would instantly end the run with a 0 exit code if we act like parallel mode.
// // so instead we check length of ran specs just to make sure we have run all the specs.
// // However, this means the api can't end a run early for us without some other logic being added.
// if (shouldFallbackToOfflineOrder) {
// spec = _.without(specs, ...ranSpecs)[0]?.relative
// }
// }
// no more specs to run?
if (!spec) {
// then we're done!
@@ -413,6 +428,7 @@ const iterateThroughSpecs = function (options = {}) {
// our specs array since the API sends us
// the relative name
spec = _.find(specs, { relative: spec })
ranSpecs.push(spec)
return runEachSpec(
spec,
@@ -426,22 +442,21 @@ const iterateThroughSpecs = function (options = {}) {
return afterSpecRun(spec, results, config)
})
.then(() => {
// // no need to make an extra request if we know we've run all the specs
// if (!parallel && ranSpecs.length === specs.length) {
// return runs
// }
// recurse
return parallelWithRecord(runs)
return parallelAndSerialWithRecord(runs)
})
})
}
if (parallel) {
if (beforeSpecRun) {
// if we are running in parallel
// then ask the server for the next spec
return parallelWithRecord([])
}
if (beforeSpecRun) {
// else iterate serialially and record
// the results of each spec
return serialWithRecord()
return parallelAndSerialWithRecord([])
}
// else iterate in serial
@@ -967,7 +982,7 @@ module.exports = {
},
listenForProjectEnd (project, exit) {
return new Promise((resolve) => {
return new Promise((resolve, reject) => {
if (exit === false) {
resolve = () => {
console.log('not exiting due to options.exit being false')
@@ -975,6 +990,10 @@ module.exports = {
}
const onEarlyExit = function (err) {
if (err.isFatalApiErr) {
return reject(err)
}
console.log('')
errors.log(err)
@@ -1154,6 +1173,9 @@ module.exports = {
reporterStats: null,
})
// dashboard told us to skip this spec
const skippedSpec = results.skippedSpec
if (startedVideoCapture) {
results.video = videoName
}
@@ -1196,11 +1218,11 @@ module.exports = {
const hasFailingTests = _.get(stats, 'failures') > 0
// we should upload the video if we upload on passes (by default)
// or if we have any failures and have started the video
const shouldUploadVideo = videoUploadOnPasses === true || Boolean((startedVideoCapture && hasFailingTests))
const shouldUploadVideo = !skippedSpec && videoUploadOnPasses === true || Boolean((startedVideoCapture && hasFailingTests))
results.shouldUploadVideo = shouldUploadVideo
if (!quiet) {
if (!quiet && !skippedSpec) {
this.displayResults(results, estimated)
if (screenshots && screenshots.length) {
this.displayScreenshots(screenshots)
@@ -1215,7 +1237,7 @@ module.exports = {
await openProject.closeBrowser()
}
if (videoExists && endVideoCapture && !videoCaptureFailed) {
if (videoExists && !skippedSpec && endVideoCapture && !videoCaptureFailed) {
const ffmpegChaptersConfig = videoCapture.generateFfmpegChaptersConfig(results.tests)
await this.postProcessRecording(
@@ -1599,19 +1621,22 @@ module.exports = {
const { projectName } = config
return recordMode.createRunAndRecordSpecs({
tag,
key,
sys,
specs,
group,
tag,
config,
browser,
parallel,
ciBuildId,
project,
projectId,
projectRoot,
projectName,
specPattern,
runAllSpecs,
onError,
})
}
+1 -1
View File
@@ -321,7 +321,7 @@ const moduleFactory = () => {
options.configFile = args.configFile
}
options = _.extend({}, args.config, options)
options = _.extend({}, args.config, options, { args })
// open the project and return
// the config for the project instance
+16 -1
View File
@@ -58,6 +58,7 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
protected _cfg?: Cfg
protected _server?: TServer
protected _automation?: Automation
private _recordTests = null
public browser: any
@@ -93,6 +94,10 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
throw new Error('Project#projectType must be defined')
}
setOnTestsReceived (fn) {
this._recordTests = fn
}
get server () {
return this.ensureProp(this._server, 'open')
}
@@ -348,12 +353,22 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
this.emit('socket:connected', id)
},
onSetRunnables (runnables) {
onTestsReceivedAndMaybeRecord: async (runnables, cb) => {
debug('received runnables %o', runnables)
if (reporter != null) {
reporter.setRunnables(runnables)
}
if (this._recordTests) {
await this._recordTests(runnables, cb)
this._recordTests = null
return
}
cb()
},
onMocha: (event, runnable) => {
+3 -5
View File
@@ -140,7 +140,7 @@ export class SocketBase {
_.defaults(options, {
socketId: null,
onResetServerState () {},
onSetRunnables () {},
onTestsReceivedAndMaybeRecord () {},
onMocha () {},
onConnect () {},
onRequest () {},
@@ -284,10 +284,8 @@ export class SocketBase {
return options.onConnect(socketId, socket)
})
socket.on('set:runnables', (runnables, cb) => {
options.onSetRunnables(runnables)
return cb()
socket.on('set:runnables:and:maybe:record:tests', async (runnables, cb) => {
return options.onTestsReceivedAndMaybeRecord(runnables, cb)
})
socket.on('mocha', (...args: unknown[]) => {
+2 -1
View File
@@ -50,7 +50,8 @@ const apiRoutes = makeRoutes(apiUrl, {
ping: 'ping',
runs: 'runs',
instances: 'runs/:id/instances',
instance: 'instances/:id',
instanceTests: 'instances/:id/tests',
instanceResults: 'instances/:id/results',
instanceStdout: 'instances/:id/stdout',
orgs: 'organizations',
projects: 'projects',
+20
View File
@@ -0,0 +1,20 @@
import _ from 'lodash'
export const flattenSuiteIntoRunnables = (suite, tests = [], hooks = []) => {
if (_.isArray(suite)) {
return _.map(suite, (s) => flattenSuiteIntoRunnables(s))
.reduce(
(arr1, arr2) => [arr1[0].concat(arr2[0]), arr1[1].concat(arr2[1])],
[tests, hooks],
)
}
tests = tests.concat(suite.tests)
hooks = hooks.concat(suite.hooks)
if (suite.suites.length) {
return flattenSuiteIntoRunnables(suite.suites, tests, hooks)
}
return [tests, hooks]
}
+1 -1
View File
@@ -127,7 +127,7 @@
"@babel/core": "7.9.0",
"@babel/preset-env": "7.9.0",
"@cypress/debugging-proxy": "2.0.1",
"@cypress/json-schemas": "5.35.1",
"@cypress/json-schemas": "5.37.3",
"@cypress/sinon-chai": "2.9.1",
"@ffprobe-installer/ffprobe": "1.1.0",
"@packages/desktop-gui": "0.0.0-development",
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,5 @@
describe('a spec', () => {
it('a test', () => {
})
})
@@ -0,0 +1,14 @@
Cypress.run = () => {
// force a plugin crash immediately
// e.g. to test when spec is skipped that Cypress.run is never called
Cypress.backend('task', {
task: 'plugins:crash',
arg: '',
timeout: 6000,
})
}
describe('a spec', () => {
it('a test', () => {
})
})
@@ -0,0 +1,5 @@
describe('b spec', () => {
it('b test', () => {
})
})
@@ -0,0 +1,20 @@
Cypress.config('defaultCommandTimeout', 1111)
Cypress.config('pageLoadTimeout', 2222)
describe('record pass', { defaultCommandTimeout: 1234 }, () => {
Cypress.config('pageLoadTimeout', 3333)
it('passes', { env: { foo: true }, retries: 2 }, () => {
Cypress.config('defaultCommandTimeout', 4444)
cy.visit('/scrollable.html')
cy.viewport(400, 400)
cy.get('#box')
cy.screenshot('yay it passes')
})
it('is pending')
// eslint-disable-next-line
it.skip('is pending due to .skip', () => {})
it('is skipped due to browser', { browser: 'edge' }, () => {})
})
// add retries and test in snapshot / assertion
@@ -1,8 +1,7 @@
/* eslint-disable no-undef */
describe('record pass', () => {
it('passes', () => {
it('passes', { env: { foo: true } }, () => {
cy.visit('/scrollable.html')
cy.viewport(400, 400)
cy.get('#box')
cy.screenshot('yay it passes')
@@ -82,6 +82,11 @@ module.exports = (on, config, mode) => {
throw new Error(message)
},
'plugins:crash' (message) {
console.log('\nPURPOSEFULLY CRASHING THE PLUGIN PROCESS FROM TEST')
process.exit(1)
},
'ensure:pixel:color' ({ name, colors, devicePixelRatio }) {
const imagePath = path.join(__dirname, '..', 'screenshots', `${name}.png`)
@@ -74,8 +74,7 @@ const normalizeTestTimings = function (obj, timings) {
export const expectRunsToHaveCorrectTimings = (runs = []) => {
runs.forEach((run) => {
expect(run.cypressConfig).to.be.a('object')
run.cypressConfig = {}
expect(run.config).to.not.exist
expectStartToBeBeforeEnd(run, 'stats.wallClockStartedAt', 'stats.wallClockEndedAt')
expectStartToBeBeforeEnd(run, 'reporterStats.start', 'reporterStats.end')
@@ -163,7 +162,7 @@ export const expectRunsToHaveCorrectTimings = (runs = []) => {
}
})
} catch (e) {
e.message = `Error during validation for test "${test.title.join(' / ')}"\n${e.message}`
e.message = `Error during validation for test \n${e.message}`
throw e
}
})
@@ -0,0 +1,298 @@
import _ from 'lodash'
import Bluebird from 'bluebird'
import bodyParser from 'body-parser'
import { api as jsonSchemas } from '@cypress/json-schemas'
import e2e from './e2e'
export const postRunResponseWithWarnings = jsonSchemas.getExample('postRunResponse')('2.2.0')
export const postRunInstanceResponse = jsonSchemas.getExample('postRunInstanceResponse')('2.1.0')
export const postInstanceTestsResponse = jsonSchemas.getExample('postInstanceTestsResponse')('1.0.0')
postInstanceTestsResponse.actions = []
export const postRunResponse = _.assign({}, postRunResponseWithWarnings, { warnings: [] })
type DeepPartial<T> = {
[P in keyof T]?: DeepPartial<T[P]>;
};
const sendUploadUrls = function (req, res) {
const { body } = req
let num = 0
const json = {} as any
if (body.video) {
json.videoUploadUrl = 'http://localhost:1234/videos/video.mp4'
}
const screenshotUploadUrls = _.map(body.screenshots, (s) => {
num += 1
return {
screenshotId: s.screenshotId,
uploadUrl: `http://localhost:1234/screenshots/${num}.png`,
}
})
json.screenshotUploadUrls = screenshotUploadUrls
return res.json(json)
}
const mockServerState = {
requests: [],
setSpecs (req) {
mockServerState.specs = [...req.body.specs]
mockServerState.allSpecs = [...req.body.specs]
},
allSpecs: [],
specs: [],
}
const routeHandlers = {
postRun: {
method: 'post',
url: '/runs',
req: 'postRunRequest@2.3.0',
resSchema: 'postRunResponse@2.2.0',
res: (req, res) => {
if (!req.body.specs) {
throw new Error('expected for Test Runner to post specs')
}
mockServerState.setSpecs(req)
return res.json(postRunResponse)
},
},
postRunInstance: {
method: 'post',
url: '/runs/:id/instances',
req: 'postRunInstanceRequest@2.1.0',
resSchema: 'postRunInstanceResponse@2.1.0',
res: (req, res) => {
console.log(mockServerState.allSpecs.length, mockServerState.specs.length)
const response = {
...postRunInstanceResponse,
spec: mockServerState.specs.shift() || null,
claimedInstances: mockServerState.allSpecs.length - mockServerState.specs.length,
totalInstances: mockServerState.allSpecs.length,
}
console.log('response', response)
return res.json(response)
},
},
postInstanceTests: {
method: 'post',
url: '/instances/:id/tests',
req: 'postInstanceTestsRequest@1.0.0',
resSchema: 'postInstanceTestsResponse@1.0.0',
res: postInstanceTestsResponse,
},
postInstanceResults: {
method: 'post',
url: '/instances/:id/results',
req: 'postInstanceResultsRequest@1.0.0',
resSchema: 'postInstanceResultsResponse@1.0.0',
res: sendUploadUrls,
},
putInstanceStdout: {
method: 'put',
url: '/instances/:id/stdout',
req: 'putInstanceStdoutRequest@1.0.0',
res (req, res) {
return res.sendStatus(200)
},
},
putVideo: {
method: 'put',
url: '/videos/:name',
res (req, res) {
return Bluebird.delay(200)
.then(() => {
return res.sendStatus(200)
})
},
},
putScreenshots: {
method: 'put',
url: '/screenshots/:name',
res (req, res) {
return res.sendStatus(200)
},
},
}
export const createRoutes = (props: DeepPartial<typeof routeHandlers>) => {
return _.values(_.merge(_.cloneDeep(routeHandlers), props))
}
beforeEach(() => {
mockServerState.requests.length = 0
mockServerState.specs.length = 0
mockServerState.allSpecs.length = 0
})
export const getRequestUrls = () => {
return _.map(mockServerState.requests, 'url')
}
export const getRequests = () => {
return mockServerState.requests
}
const getSchemaErr = (tag, err, schema) => {
return {
errors: err.errors,
object: err.object,
example: err.example,
message: `${tag} should follow ${schema} schema`,
}
}
const getResponse = function (responseSchema) {
if (!responseSchema) {
throw new Error('No response schema supplied')
}
if (_.isObject(responseSchema)) {
return responseSchema
}
const [name, version] = responseSchema.split('@')
return jsonSchemas.getExample(name)(version)
}
const sendResponse = function (req, res, responseBody) {
return new Promise((resolve) => {
const _writeRaw = res._writeRaw
res._writeRaw = function () {
resolve()
return _writeRaw.apply(this, arguments)
}
if (_.isFunction(responseBody)) {
return responseBody(req, res)
}
res.json(getResponse(responseBody))
resolve()
})
}
const ensureSchema = function (expectedRequestSchema, responseBody, expectedResponseSchema) {
let reqName; let reqVersion
if (expectedRequestSchema) {
[reqName, reqVersion] = expectedRequestSchema.split('@')
}
return async function (req, res) {
const { body } = req
try {
if (expectedRequestSchema) {
jsonSchemas.assertSchema(reqName, reqVersion)(body)
}
res.expectedResponseSchema = expectedResponseSchema
await sendResponse(req, res, responseBody)
const key = [req.method, req.url].join(' ')
mockServerState.requests.push({
url: key,
body,
})
} catch (err) {
console.log('Schema Error:', err.message)
return res.status(412).json(getSchemaErr('request', err, expectedRequestSchema))
}
}
}
const assertResponseBodySchema = function (req, res, next) {
const oldWrite = res.write
const oldEnd = res.end
const chunks = []
res.write = (chunk) => {
// buffer the response, we'll really write it on end
return chunks.push(chunk)
}
res.end = function (chunk) {
if (chunk) {
chunks.push(chunk)
}
res.write = oldWrite
res.end = oldEnd
if (res.expectedResponseSchema && _.inRange(res.statusCode, 200, 299)) {
const body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
const [resName, resVersion] = res.expectedResponseSchema.split('@')
try {
jsonSchemas.assertSchema(resName, resVersion)(body)
} catch (err) {
console.log('Schema Error:', err.message)
return res.status(412).json(getSchemaErr('response', err, res.expectedResponseSchema))
}
}
chunks.map((chunk) => {
return res.write(chunk)
})
return res.end()
}
return next()
}
const onServer = (routes) => {
return (function (app) {
app.use(bodyParser.json())
app.use(assertResponseBodySchema)
return _.each(routes, (route) => {
return app[route.method](route.url, ensureSchema(
route.req,
route.res,
route.resSchema,
))
})
})
}
export const setupStubbedServer = (routes, settings = {}) => {
e2e.setup({
settings: _.extend({
projectId: 'pid123',
videoUploadOnPasses: false,
}, settings),
servers: [{
port: 1234,
onServer: onServer(routes),
}, {
port: 3131,
static: true,
}],
})
return mockServerState
}
+21 -19
View File
@@ -344,6 +344,10 @@ describe('lib/api', () => {
remoteOrigin: 'https://github.com/foo/bar.git',
},
specs: ['foo.js', 'bar.js'],
runnerCapabilities: {
'dynamicSpecsInSerialMode': true,
'skipSpecAction': true,
},
}
})
@@ -559,38 +563,36 @@ describe('lib/api', () => {
error: 'err msg',
video: true,
screenshots: [],
cypressConfig: {},
reporterStats: {},
stdout: null,
}
this.putProps = _.omit(this.updateProps, 'instanceId')
this.postProps = _.pick(this.updateProps, 'stats', 'video', 'screenshots', 'reporterStats')
})
it('PUTs /instances/:id', function () {
it('POSTs /instances/:id/results', function () {
nock(API_BASEURL)
.matchHeader('x-route-version', '3')
.matchHeader('x-route-version', '1')
.matchHeader('x-os-name', 'linux')
.matchHeader('x-cypress-version', pkg.version)
.put('/instances/instance-id-123', this.putProps)
.post('/instances/instance-id-123/results', this.postProps)
.reply(200)
return api.updateInstance(this.updateProps)
return api.postInstanceResults(this.updateProps)
})
it('PUT /instances/:id failure formatting', () => {
nock(API_BASEURL)
.matchHeader('x-route-version', '3')
.matchHeader('x-route-version', '1')
.matchHeader('x-os-name', 'linux')
.matchHeader('x-cypress-version', pkg.version)
.put('/instances/instance-id-123')
.post('/instances/instance-id-123/results')
.reply(422, {
errors: {
tests: ['is required'],
},
})
return api.updateInstance({ instanceId: 'instance-id-123' })
return api.postInstanceResults({ instanceId: 'instance-id-123' })
.then(() => {
throw new Error('should have thrown here')
}).catch((err) => {
@@ -610,14 +612,14 @@ describe('lib/api', () => {
it('handles timeouts', () => {
nock(API_BASEURL)
.matchHeader('x-route-version', '3')
.matchHeader('x-route-version', '1')
.matchHeader('x-os-name', 'linux')
.matchHeader('x-cypress-version', pkg.version)
.put('/instances/instance-id-123')
.post('/instances/instance-id-123/results')
.socketDelay(5000)
.reply(200, {})
return api.updateInstance({
return api.postInstanceResults({
instanceId: 'instance-id-123',
timeout: 100,
})
@@ -629,23 +631,23 @@ describe('lib/api', () => {
})
it('sets timeout to 60 seconds', () => {
sinon.stub(api.rp, 'put').resolves()
sinon.stub(api.rp, 'post').resolves()
return api.updateInstance({})
return api.postInstanceResults({})
.then(() => {
expect(api.rp.put).to.be.calledWithMatch({ timeout: 60000 })
expect(api.rp.post).to.be.calledWithMatch({ timeout: 60000 })
})
})
it('tags errors', function () {
nock(API_BASEURL)
.matchHeader('x-route-version', '2')
.matchHeader('x-route-version', '1')
.matchHeader('authorization', 'Bearer auth-token-123')
.matchHeader('accept-encoding', /gzip/)
.put('/instances/instance-id-123', this.putProps)
.post('/instances/instance-id-123/results', this.postProps)
.reply(500, {})
return api.updateInstance(this.updateProps)
return api.postInstanceResults(this.updateProps)
.then(() => {
throw new Error('should have thrown here')
}).catch((err) => {
+40 -12
View File
@@ -439,27 +439,24 @@ describe('lib/modes/record', () => {
})
})
it('does not createException when statusCode is 503', () => {
it('errors when statusCode is 503', async () => {
const err = new Error('foo')
err.statusCode = 503
sinon.spy(logger, 'createException')
sinon.spy(errors, 'get')
sinon.stub(api, 'retryWithBackoff').rejects(err)
return recordMode.createInstance({
await expect(recordMode.createInstance({
runId: 'run-123',
groupId: 'group-123',
machineId: 'machine-123',
platform: {},
spec: { relative: 'cypress/integration/app_spec.coffee' },
})
.then((ret) => {
expect(ret).to.be.null
})).to.be.rejected
expect(logger.createException).not.to.be.called
})
expect(errors.get).to.have.been.calledWith('DASHBOARD_CANNOT_PROCEED_IN_SERIAL')
})
})
@@ -511,9 +508,9 @@ describe('lib/modes/record', () => {
})
})
context('.updateInstance', () => {
context('.postInstanceTests', () => {
beforeEach(function () {
sinon.stub(api, 'updateInstance')
sinon.stub(api, 'postInstanceTests')
sinon.stub(ciProvider, 'ciParams').returns({})
sinon.stub(ciProvider, 'provider').returns('')
sinon.stub(ciProvider, 'commitDefaults').returns({})
@@ -527,7 +524,7 @@ describe('lib/modes/record', () => {
it('retries with backoff strategy', function () {
sinon.stub(api, 'retryWithBackoff').yields().resolves()
recordMode.updateInstance(this.options)
recordMode._postInstanceTests(this.options)
expect(api.retryWithBackoff).to.be.called
})
@@ -535,7 +532,38 @@ describe('lib/modes/record', () => {
it('logs on retry', function () {
sinon.stub(api, 'retryWithBackoff').yields().resolves()
return recordMode.updateInstance(this.options)
return recordMode._postInstanceTests(this.options)
.then(() => {
expect(api.retryWithBackoff).to.be.calledOnce
})
})
})
context('.postInstanceResults', () => {
beforeEach(function () {
sinon.stub(api, 'postInstanceResults')
sinon.stub(ciProvider, 'ciParams').returns({})
sinon.stub(ciProvider, 'provider').returns('')
sinon.stub(ciProvider, 'commitDefaults').returns({})
this.options = {
results: {},
captured: '',
}
})
it('retries with backoff strategy', function () {
sinon.stub(api, 'retryWithBackoff').yields().resolves()
recordMode.postInstanceResults(this.options)
expect(api.retryWithBackoff).to.be.called
})
it('logs on retry', function () {
sinon.stub(api, 'retryWithBackoff').yields().resolves()
return recordMode.postInstanceResults(this.options)
.then(() => {
expect(api.retryWithBackoff).to.be.calledOnce
})
@@ -24,8 +24,12 @@ describe('lib/util/routes', () => {
expect(apiRoutes.instances(123)).to.eq('http://localhost:1234/runs/123/instances')
})
it('instance', () => {
expect(apiRoutes.instance(123)).to.eq('http://localhost:1234/instances/123')
it('instanceTests', () => {
expect(apiRoutes.instanceTests(123)).to.eq('http://localhost:1234/instances/123/tests')
})
it('instanceResults', () => {
expect(apiRoutes.instanceResults(123)).to.eq('http://localhost:1234/instances/123/results')
})
it('projects', () => {
+34
View File
@@ -2132,6 +2132,17 @@
lodash.merge "^4.6.2"
lodash.omit "^4.5.0"
"@cypress/json-schemas@5.37.3":
version "5.37.3"
resolved "https://registry.yarnpkg.com/@cypress/json-schemas/-/json-schemas-5.37.3.tgz#2aed2fadc9533cb8d812ab27df77ea40ccf17759"
integrity sha512-vMhydN3Ysx+d/PnbOixPzlWdvaOeHmano61D2Cwshexo3F8L68xsXqF6zy3T6teNWHXTMtNZ6fQUpcy3r082XQ==
dependencies:
"@cypress/schema-tools" "4.7.7"
lodash "^4.17.21"
lodash.clonedeep "^4.5.0"
lodash.merge "^4.6.2"
lodash.omit "^4.5.0"
"@cypress/listr-verbose-renderer@^0.4.1":
version "0.4.1"
resolved "https://registry.yarnpkg.com/@cypress/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz#a77492f4b11dcc7c446a34b3e28721afd33c642a"
@@ -2228,6 +2239,24 @@
quote "0.4.0"
ramda "0.25.0"
"@cypress/schema-tools@4.7.7":
version "4.7.7"
resolved "https://registry.yarnpkg.com/@cypress/schema-tools/-/schema-tools-4.7.7.tgz#251a9864caba0eded884ff5c71de16c76dbf556a"
integrity sha512-RRzksoJIXDTeUjt7YE9xAhOynqc7R+j8Tx8ebpkSPJB6Z3WujdLP0sigVh2AV24G/CySOvJGuQQY94aBEpCZaA==
dependencies:
"@bahmutov/all-paths" "1.0.2"
"@bahmutov/is-my-json-valid" "2.17.3"
"@types/ramda" "0.25.47"
debug "4.3.1"
json-stable-stringify "1.0.1"
json2md "1.6.3"
lodash.camelcase "4.3.0"
lodash.get "4.4.2"
lodash.reduce "^4.6.0"
lodash.set "4.3.2"
quote "0.4.0"
ramda "0.25.0"
"@cypress/set-commit-status@1.3.4":
version "1.3.4"
resolved "https://registry.yarnpkg.com/@cypress/set-commit-status/-/set-commit-status-1.3.4.tgz#9c96e6b8c192de5723a995910ccdcca60f6c17fb"
@@ -22755,6 +22784,11 @@ lodash@4.17.4:
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
integrity sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=
lodash@^4.17.21:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
log-ok@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334"