Files
cypress/packages/server/lib/api.js
Ben Kucera 860a20af30 feat: support test retries (#3968)
* add retries e2e test

* restore runner/test/eslintrc

* use mocha pass event, move runner.spec to runner package

* fix .eslintignore

* remove npmInstall logic in helper/e2e script, force custom reporters to use our mocha

* temp 04/09/20 [skip ci]

* add retries output to server/reporter, fix mocha pass event order, cleanup

* e2e tests - dont run electron tests in other browsers

* Update readme to reflect how to start server for cypress tests

* fix after merge

* fix .coffee -> .js after merge

* fix attempt.tsx

* fix runnable titles emitted to terminal reporters

* fix more tests: update snapshots, fix 7_record_spec, 8_reporters_spec

* remove styling for 'attempt-error-region' so it's not indented

- This was the older styling before error improvements and is no longer
necessary.

* try 2: fix rerun before/after hooks

* fix runner with only, runner snapshots, lint fixes

* temp 04/29/20 [skip ci]

* backport changes from test-retries

* change logic to rerun before hooks after top navigation

* fix windowSize for browser e2e test

* fix windowSize for xvfb chrome in e2e test

* ok fine, just disable screenshots

* fix after merge: decaffed navigation.js

* update server/unit test snapshots

* fix after merge: decaffed aliases.js

* fix usage of cypress --parallel flag in circle.yml

* fix circle.yml integration-tests jobs

* fix decaf related typo

* fix circle.yml separate command for runner-integration-tests

* update runner/integration tests + snapshot after error improvements

* fix runner/integration snapshots for chrome/ff stacktrace differences

* rerun ci

* fix passing --parallel to runner-integration tests

* perf: faster lookup for hooks without runnables

* fix afterAll hook switch logic

* simplify mocha prototype patches

* fix decaf utils.coffee after merge

* backport to before/after fix

* backport to before/after fix 2

* cleanup from decaf, fix ui/package.json

* update helpers, simplify runner.spec

* fix lint-types errors, flaky spec

* fix noExit passed to e2e test inline options

* cleanup snapshot utility - refactor to use util file

* remove before/after changes

* make cy obj a class instance

* cleanup/unmerge before/after fixes PR...

* more cleanup

* add comment

* fix runner.spec

* cleanup snapshot utility more, cleanup reporter.spec

* fix after merge

* minor rename variable

* fix after merge: decaffed files

* fix specName in reporterHeader, spec_helper require

* replace reporter specPath usages with spec object from config

* cleanup, fix specs, fix types tests

* fix config spec paths in isolated runner, fix snapshot plugin button

* combine runner.spec.js and runner_spec.js

* fix incorrect merge

* minor minor cleanup

* rename driver/test/cypress to driver/test

* use yarn workspace over lerna for individual package commands

* add error message to driver start

* remove usage of wait-on

* update <reference types/>, import string

* fix driver/readme

* fix readmes after regex replace

* revert wait-on changes

* Revert "revert wait-on changes"

This reverts commit 6de684cf34.

* update yarn.lock

* fix broken path in spec

* fix broken paths in specs with @packages/driver

* move runner/test/cypress into runner/cypress

* start server in pluginsFile in runner/cypress tests

* fix more broken spec paths

* fix broken paths after runner/cypress folder move

* move type definition loading for driver/cypress into dedicated file

* move internal-types to "types" folder, fix driver/index.d.ts

* fix type-check in packages/runner. not exactly sure why

* fix runner type-check by excluding test folder in tsconfig

* bump timeout on e2e/8_error_ui_spec

* update snapshot utility, rename tests in runner/runner.spec, fix README yarn commands

* delete old spec

* fix snapshot naming, remove redundant test in reporter_spec

* fix file renames after merge

* rename runner/ snapshot

* update server/unit/reporter_spec snapshot

* update runner/runner_spec snapshot

* rename runner snapshot file

* address feedback: move server reporter snapshot specs out

* address feedback: add comment about exposing globals

* fix test-retries after merging isolated-runner

* fix runner/test helper, update snapshot

* address feedback: split out runner/retries spec, move reporter/ui tests to runner/ui spec (mostly done), various cleanup

* fix scrolling, attempt opening, update snapshots

* fix e2e support file

* fix 5_spec_isolation

* fix mislabeling attempt screenshots

* only add test results prevAttempts if exists

* fix reporter/unit tests, server/unit tests

* remove dead code, fix test isOpen

* update snapshots for retries.mochaEvents, fix snapshot error in state hydration test, remove dead snapshots

* new moduleAPI schema using attempts array, fix wrapping errors from hook retries, update snapshots

* add displayError, null out fields in moduleAPI schema

* change default retries to {runMode:2, openMode:0}

* fix reporter type-check

* upgrade json-schemas, update snapshots

* reformat error.stack to be only stacktrace, update snapshots

* fix stacktrace replacing in 5_spec_isolation

* fix navigation error causing infinite reloading, bump timeout on e2e/8_error_ui

* fix server/unit tests for new schema

* fix reporter/unit tests

* fix reporting duplicate screenshots using cy.screenshot during test retry

* update snapshot for 6_uncaught_support_file_spec

* bump x-route-version: 3

* fix test.tsx collapsible content, css, fix e2e/8_error_ui, e2e projects excluding retries

* fix css, fix padding in runnable-instruments, fix runner/integration tests

* fixup after merge

* fix reporter/runner to work with split hooks

* update api tests, runner/cypress tests, reporter

* fix 5_spec_isolation snapshots, fix runner/cypress errors.spec, fix null reference in test.tsx

* fix e2e/non_root spec, fix type_check, fix reporter/unit tests

* setup percy snapshots in runner/cypress, fix driver/runner test:after:run event, add tests for only,skip in runner/cypress, fix retried css

* add customPercySnapshot

* fix circle.yml

* fix circle.yml 2

* fix circle.yml 3

* add warning for incompatible retries plugin

* add more percy snapshots

* fix firefox screenshot resolution in e2e test

* Fix testConfigOverrides not affecting viewport (#8006)

* finish adding percy snapshots to runner/cypress retries spec, update error msgs, add tests to be fixed

* remove .only

* fixing missing repo argument

* fix testConfigOverrides usage with retries, fix test

* fix issues from previous merge

* add script that can query CircleCI workflow status

* add circleci job to poll

* add retries

* try yarn lock

* retry, percy finalize

* check for current running job

* do not swallow request error

* better print

* use job name from circle environment

* use debug instead

* renamed circle polling script

* refactor circle to conditionally run percy-finalize when env var is available

- pass job-names to wait on as an argument

* use multi-line strings and quote --job-names

- rename —circle-jobs to —job-names

* add comment

* only poll until the jobs to wait for are blocked or running

* fix running hooks at correct depth after attempt fails from hook and will retry, update e2e snapshots

* fix reporter/unit tests, remove unused toggleOpen code

* move custom percy command into @packages/ui-components and apply them to desktop-gui

* halt percy finalize job if env variable is not set

* if only I could code

* update runner/cypress mochaEvent snapshots, fix e2e firefox resolution

* fix css for attempt border-left, fix attempt-tag open/close icon, add color to attempt collapsible dot

* try percy set viewport width

* set default retries back to {runMode:0, openMode:0}

* formatting: add backticks to warning message

* write explicit test for screenshot overwriting behavior, fix snapshots after changing retries defaults

* fix e2e.it.only`

* cleanup whitespace

* update snapshots

* fix cypress module API types for new result schema

* build and upload binary for test-retries branch too (linux)

* add pre-release PR comment

* fix pre-release commit comment

* rename runner/cypress test

* update retries.ui.spec test titles

* fix after merge: use most recent attempt for before/after hooks

* add suite title to hook error in runner/cypress tests

Co-authored-by: Jennifer Shehane <jennifer@cypress.io>
Co-authored-by: Brian Mann <brian.mann86@gmail.com>
Co-authored-by: Gleb Bahmutov <gleb.bahmutov@gmail.com>
2020-08-10 18:36:45 -04:00

443 lines
9.2 KiB
JavaScript

const _ = require('lodash')
const os = require('os')
const debug = require('debug')('cypress:server:api')
const request = require('@cypress/request-promise')
const errors = require('@cypress/request-promise/errors')
const Promise = require('bluebird')
const humanInterval = require('human-interval')
const { agent } = require('@packages/network')
const pkg = require('@packages/root')
const machineId = require('./util/machine_id')
const routes = require('./util/routes')
const THIRTY_SECONDS = humanInterval('30 seconds')
const SIXTY_SECONDS = humanInterval('60 seconds')
const TWO_MINUTES = humanInterval('2 minutes')
let intervals
let DELAYS = [
THIRTY_SECONDS,
SIXTY_SECONDS,
TWO_MINUTES,
]
let responseCache = {}
intervals = process.env.API_RETRY_INTERVALS
if (intervals) {
DELAYS = _
.chain(intervals)
.split(',')
.map(_.toNumber)
.value()
}
const rp = request.defaults((params = {}, callback) => {
let resp
if (params.cacheable && (resp = getCachedResponse(params))) {
debug('resolving with cached response for ', params.url)
return Promise.resolve(resp)
}
_.defaults(params, {
agent,
proxy: null,
gzip: true,
cacheable: false,
})
const headers = params.headers != null ? params.headers : (params.headers = {})
_.defaults(headers, {
'x-os-name': os.platform(),
'x-cypress-version': pkg.version,
})
const method = params.method.toLowerCase()
// use %j argument to ensure deep nested properties are serialized
debug(
'request to url: %s with params: %j and token: %s',
`${params.method} ${params.url}`,
_.pick(params, 'body', 'headers'),
params.auth && params.auth.bearer,
)
return request[method](params, callback)
.promise()
.tap((resp) => {
if (params.cacheable) {
debug('caching response for ', params.url)
cacheResponse(resp, params)
}
return debug('response %o', resp)
})
})
const cacheResponse = (resp, params) => {
return responseCache[params.url] = resp
}
const getCachedResponse = (params) => {
return responseCache[params.url]
}
const formatResponseBody = function (err) {
// if the body is JSON object
if (_.isObject(err.error)) {
// transform the error message to include the
// stringified body (represented as the 'error' property)
const body = JSON.stringify(err.error, null, 2)
err.message = [err.statusCode, body].join('\n\n')
}
throw err
}
const tagError = function (err) {
err.isApiError = true
throw err
}
// retry on timeouts, 5xx errors, or any error without a status code
const isRetriableError = (err) => {
return (err instanceof Promise.TimeoutError) ||
(500 <= err.statusCode && err.statusCode < 600) ||
(err.statusCode == null)
}
module.exports = {
rp,
ping () {
return rp.get(routes.ping())
.catch(tagError)
},
getMe (authToken) {
return rp.get({
url: routes.me(),
json: true,
auth: {
bearer: authToken,
},
})
},
getAuthUrls () {
return rp.get({
url: routes.auth(),
json: true,
cacheable: true,
headers: {
'x-route-version': '2',
},
})
.catch(tagError)
},
getOrgs (authToken) {
return rp.get({
url: routes.orgs(),
json: true,
auth: {
bearer: authToken,
},
})
.catch(tagError)
},
getProjects (authToken) {
return rp.get({
url: routes.projects(),
json: true,
auth: {
bearer: authToken,
},
})
.catch(tagError)
},
getProject (projectId, authToken) {
return rp.get({
url: routes.project(projectId),
json: true,
auth: {
bearer: authToken,
},
headers: {
'x-route-version': '2',
},
})
.catch(tagError)
},
getProjectRuns (projectId, authToken, options = {}) {
if (options.page == null) {
options.page = 1
}
return rp.get({
url: routes.projectRuns(projectId),
json: true,
timeout: options.timeout != null ? options.timeout : 10000,
auth: {
bearer: authToken,
},
headers: {
'x-route-version': '3',
},
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
createRun (options = {}) {
const body = _.pick(options, [
'ci',
'specs',
'commit',
'group',
'platform',
'parallel',
'ciBuildId',
'projectId',
'recordKey',
'specPattern',
'tags',
])
return rp.post({
body,
url: routes.runs(),
json: true,
timeout: options.timeout != null ? options.timeout : SIXTY_SECONDS,
headers: {
'x-route-version': '4',
},
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
createInstance (options = {}) {
const { runId, timeout } = options
const body = _.pick(options, [
'spec',
'groupId',
'machineId',
'platform',
])
return rp.post({
body,
url: routes.instances(runId),
json: true,
timeout: timeout != null ? timeout : SIXTY_SECONDS,
headers: {
'x-route-version': '5',
},
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
updateInstanceStdout (options = {}) {
return rp.put({
url: routes.instanceStdout(options.instanceId),
json: true,
timeout: options.timeout != null ? options.timeout : SIXTY_SECONDS,
body: {
stdout: options.stdout,
},
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
updateInstance (options = {}) {
return rp.put({
url: routes.instance(options.instanceId),
json: true,
timeout: options.timeout != null ? options.timeout : SIXTY_SECONDS,
headers: {
'x-route-version': '3',
},
body: _.pick(options, [
'stats',
'tests',
'error',
'video',
'hooks',
'stdout',
'screenshots',
'cypressConfig',
'reporterStats',
]),
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
createCrashReport (body, authToken, timeout = 3000) {
return rp.post({
url: routes.exceptions(),
json: true,
body,
auth: {
bearer: authToken,
},
})
.timeout(timeout)
.catch(tagError)
},
postLogout (authToken) {
return Promise.join(
this.getAuthUrls(),
machineId.machineId(),
(urls, machineId) => {
return rp.post({
url: urls.dashboardLogoutUrl,
json: true,
auth: {
bearer: authToken,
},
headers: {
'x-machine-id': machineId,
},
})
.catch({ statusCode: 401 }, () => {}) // do nothing on 401
.catch(tagError)
},
)
},
createProject (projectDetails, remoteOrigin, authToken) {
debug('create project with args %o', {
projectDetails,
remoteOrigin,
authToken,
})
return rp.post({
url: routes.projects(),
json: true,
auth: {
bearer: authToken,
},
headers: {
'x-route-version': '2',
},
body: {
name: projectDetails.projectName,
orgId: projectDetails.orgId,
public: projectDetails.public,
remoteOrigin,
},
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
getProjectRecordKeys (projectId, authToken) {
return rp.get({
url: routes.projectRecordKeys(projectId),
json: true,
auth: {
bearer: authToken,
},
})
.catch(tagError)
},
requestAccess (projectId, authToken) {
return rp.post({
url: routes.membershipRequests(projectId),
json: true,
auth: {
bearer: authToken,
},
})
.catch(errors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
_projectToken (method, projectId, authToken) {
return rp({
method,
url: routes.projectToken(projectId),
json: true,
auth: {
bearer: authToken,
},
headers: {
'x-route-version': '2',
},
})
.get('apiToken')
.catch(tagError)
},
getProjectToken (projectId, authToken) {
return this._projectToken('get', projectId, authToken)
},
updateProjectToken (projectId, authToken) {
return this._projectToken('put', projectId, authToken)
},
retryWithBackoff (fn, options = {}) {
// for e2e testing purposes
let attempt
if (process.env.DISABLE_API_RETRIES) {
debug('api retries disabled')
return Promise.try(fn)
}
return (attempt = (retryIndex) => {
return Promise
.try(fn)
.catch(isRetriableError, (err) => {
if (retryIndex > DELAYS.length) {
throw err
}
const delay = DELAYS[retryIndex]
if (options.onBeforeRetry) {
options.onBeforeRetry({
err,
delay,
retryIndex,
total: DELAYS.length,
})
}
retryIndex++
return Promise
.delay(delay)
.then(() => {
debug(`retry #${retryIndex} after ${delay}ms`)
return attempt(retryIndex)
})
})
})(0)
},
clearCache () {
responseCache = {}
},
}