mirror/cypress
1
0
Fork 0
mirror of https://github.com/cypress-io/cypress.git synced 2026-04-23 07:34:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: percy updates (#20380)

Browse Source 
Co-authored-by: Brian Mann <brian.mann86@gmail.com>
This commit is contained in:
Mark Noonan
2022-03-06 18:45:48 +00:00
committed by GitHub
parent 1c987c88c6
commit de4cb66050
5 changed files with 230 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
circle.yml
+33 -47
View File
@@ -49,7 +49,6 @@ windowsWorkflowFilters: &windows-workflow-filters
or:
- equal: [ develop, << pipeline.git.branch >> ]
- equal: [ '10.0-release', << pipeline.git.branch >> ]
- equal: [ merge-develop-2-28-22, << pipeline.git.branch >> ]
- matches:
pattern: "-release$"
value: << pipeline.git.branch >>
@@ -473,11 +472,6 @@ commands:
# https://www.gep13.co.uk/blog/chocolatey-error-hashes-do-not-match
[[ $PLATFORM == 'windows' && '<<parameters.browser>>' == 'chrome' ]] && choco install googlechrome --ignore-checksums || [[ $PLATFORM != 'windows' ]]
- run:
# the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
# are a temporary hack to workaround percy issues with
# PR's not diffing the right base branch due to other problems
# upstream with finalizing builds. These will be removed
# once we implement a more permanent solution.
command: |
cmd=$([[ <<parameters.percy>> == 'true' ]] && echo 'yarn percy exec --parallel -- --') || true
DEBUG=<<parameters.debug>> \
@@ -486,8 +480,6 @@ commands:
PERCY_PARALLEL_NONCE=$CIRCLE_SHA1 \
PERCY_ENABLE=${PERCY_TOKEN:-0} \
PERCY_PARALLEL_TOTAL=-1 \
PERCY_TARGET_BRANCH="10.0-release" \
PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
$cmd yarn workspace @packages/<<parameters.package>> cypress:run:<<parameters.type>> --browser <<parameters.browser>> --record --parallel --group <<parameters.package>>-<<parameters.type>>
- store_test_results:
path: /tmp/cypress
@@ -881,6 +873,17 @@ commands:
path: /tmp/<<parameters.repo>>/cypress/videos
- store-npm-logs
wait-on-circle-jobs:
description: Polls certain Circle CI jobs until they finish
parameters:
job-names:
description: comma separated list of circle ci job names to wait for
type: string
steps:
- run:
name: "Waiting on Circle CI jobs: <<parameters.job-names>>"
command: node ./scripts/wait-on-circle-jobs.js --job-names="<<parameters.job-names>>"
build-binary:
steps:
- run:
@@ -1083,16 +1086,19 @@ jobs:
echo "This is an external PR, cannot access other services"
circleci-agent step halt
fi
- wait-on-circle-jobs:
job-names: >
cli-visual-tests,
reporter-integration-tests,
npm-design-system,
run-app-component-tests-chrome,
run-app-integration-tests-chrome,
run-frontend-shared-component-tests-chrome,
run-launchpad-component-tests-chrome,
run-launchpad-integration-tests-chrome
- run:
# the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
# are a temporary hack to workaround percy issues with
# PR's not diffing the right base branch due to other problems
# upstream with finalizing builds. These will be removed
# once we implement a more permanent solution.
command: |
PERCY_PARALLEL_NONCE=$CIRCLE_SHA1 \
PERCY_TARGET_BRANCH="10.0-release" \
PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
yarn percy build:finalize
cli-visual-tests:
@@ -1113,17 +1119,10 @@ jobs:
path: cli/visual-snapshots
- run:
name: Upload CLI snapshots for diffing
# the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
# are a temporary hack to workaround percy issues with
# PR's not diffing the right base branch due to other problems
# upstream with finalizing builds. These will be removed
# once we implement a more permanent solution.
command: |
PERCY_PARALLEL_NONCE=$CIRCLE_SHA1 \
PERCY_ENABLE=${PERCY_TOKEN:-0} \
PERCY_PARALLEL_TOTAL=-1 \
PERCY_TARGET_BRANCH="10.0-release" \
PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
yarn percy snapshot ./cli/visual-snapshots
unit-tests:
@@ -1415,19 +1414,12 @@ jobs:
command: yarn build-for-tests
working_directory: packages/reporter
- run:
# the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
# are a temporary hack to workaround percy issues with
# PR's not diffing the right base branch due to other problems
# upstream with finalizing builds. These will be removed
# once we implement a more permanent solution.
command: |
CYPRESS_KONFIG_ENV=production \
CYPRESS_RECORD_KEY=$PACKAGES_RECORD_KEY \
PERCY_PARALLEL_NONCE=$CIRCLE_SHA1 \
PERCY_ENABLE=${PERCY_TOKEN:-0} \
PERCY_PARALLEL_TOTAL=-1 \
PERCY_TARGET_BRANCH="10.0-release" \
PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
yarn percy exec --parallel -- -- \
yarn cypress:run --record --parallel --group reporter
working_directory: packages/reporter
@@ -1562,18 +1554,11 @@ jobs:
name: Run tests
# will use PERCY_TOKEN environment variable if available
#
# the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
# are a temporary hack to workaround percy issues with
# PR's not diffing the right base branch due to other problems
# upstream with finalizing builds. These will be removed
# once we implement a more permanent solution.
command: |
CYPRESS_KONFIG_ENV=production \
PERCY_PARALLEL_NONCE=$CIRCLE_SHA1 \
PERCY_ENABLE=${PERCY_TOKEN:-0} \
PERCY_PARALLEL_TOTAL=-1 \
PERCY_TARGET_BRANCH="10.0-release" \
PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
yarn percy exec --parallel -- -- \
yarn test --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json
working_directory: npm/design-system
@@ -2161,14 +2146,7 @@ linux-workflow: &linux-workflow
context: test-runner:poll-circle-workflow
required_env_var: PERCY_TOKEN # skips job if not defined (external PR)
requires:
- cli-visual-tests
- reporter-integration-tests
- npm-design-system
- run-app-component-tests-chrome
- run-app-integration-tests-chrome
- run-frontend-shared-component-tests-chrome
- run-launchpad-component-tests-chrome
- run-launchpad-integration-tests-chrome
- build
- lint-types:
requires:
- build
@@ -2236,18 +2214,18 @@ linux-workflow: &linux-workflow
requires:
- build
- run-launchpad-component-tests-chrome:
percy: true
context: test-runner:launchpad-tests
percy: true
requires:
- build
- run-app-integration-tests-chrome:
percy: true
context: test-runner:launchpad-tests
percy: true
requires:
- build
- run-app-component-tests-chrome:
percy: true
context: test-runner:launchpad-tests
percy: true
requires:
- build
- reporter-integration-tests:
@@ -2329,6 +2307,14 @@ linux-workflow: &linux-workflow
- ui-components-integration-tests
- unit-tests
- unit-tests-release
- cli-visual-tests
- reporter-integration-tests
- npm-design-system
- run-app-component-tests-chrome
- run-app-integration-tests-chrome
- run-frontend-shared-component-tests-chrome
- run-launchpad-component-tests-chrome
- run-launchpad-integration-tests-chrome
# various testing scenarios, like building full binary
# and testing it on a real project
TESTING_STRATEGY_AND_STYLEGUIDE.md → guides/testing-strategy-and-styleguide.md
-12
View File
@@ -71,18 +71,6 @@ For developer-facing behavior - props received, events emitted, any other side e
### E2E Tests
Certain side effects, like GraphQL mutations, do not fire from component tests, but can be monitored from an E2E test with `cy.intercept`. And some entire packages, like `reporter` are independent apps that can be mounted in an E2E test and tested for interactions with other parts of the system.
## Percy Snapshots
Percy snapshot diffs are reviewed and approved by somebody doing a PR review.
### Limitations and Notes
Percy snapshots confirm that the appearance of a given state matches the last approved snapshot of that state. They don't in and of themselves contain any information about correctness.
If a test only contains a percy snapshot, prefer a general name for that test as opposed to a specific detail of the snapshot. For example prefer "has expected appearance" to "has a purple outline" since nothing in the test is actually asserting that.
NOTE: 👆 This section needs more thought/discussion. Maybe "has a purple outline" is a good thing to have in the test name because it might alert somebody reviewing the snapshot about a change they should look for, or the thing we expected the snapshot to confirm. Ideally here we would end up with instructions and examples of what Percy tests we expect with new features and any conventions we have.
## Testing Style Guide
### String constants
package.json
+1
View File
@@ -165,6 +165,7 @@
"getenv": "^1.0.0",
"gift": "0.10.2",
"glob": "7.1.6",
"got": "11.8.3",
"graphql": "^15.5.1",
"gulp": "4.0.2",
"gulp-awspublish": "4.0.0",
scripts/wait-on-circle-jobs.js
+159
View File
@@ -0,0 +1,159 @@
/* eslint-disable no-console */
const _ = require('lodash')
const minimist = require('minimist')
const Promise = require('bluebird')
const retry = require('bluebird-retry')
const got = require('got')
// always print the debug logs
const debug = require('debug')('*')
const { seconds, minutes } = require('./utils')
// we expect CircleCI to set the current polling job name
const jobName = process.env.CIRCLE_JOB || 'wait-on-circle-jobs'
const workflowId = process.env.CIRCLE_WORKFLOW_ID
const branchesToAlwaysFinalize = ['develop', '10.0-release']
const requireAllJobsToPass = !branchesToAlwaysFinalize.includes(process.env.CIRCLE_BRANCH)
const getAuth = () => `${process.env.CIRCLE_TOKEN}:`
const verifyCI = () => {
if (!process.env.CIRCLE_TOKEN) {
console.error('Cannot find CIRCLE_TOKEN')
process.exit(1)
}
if (!process.env.CIRCLE_WORKFLOW_ID) {
console.error('Cannot find CIRCLE_WORKFLOW_ID')
process.exit(1)
}
}
/**
* Job status
* - blocked (has not run yet)
* - running (currently running)
* - failed | success
*/
const getJobStatus = async (workflowId) => {
const auth = getAuth()
// typo at https://circleci.com/docs/2.0/api-intro/
// to retrieve all jobs, the url is "/workflow/:id/job"
const url = `https://${auth}@circleci.com/api/v2/workflow/${workflowId}/job`
const response = await got(url).json()
// returns something like
// {
// next_page_token: null,
// items: [
// {
// dependencies: [],
// job_number: 400959,
// id: '7021bcc7-90c1-47d9-bf99-c0372a4f8f49',
// started_at: '2020-07-20T19:45:46Z',
// name: 'build',
// project_slug: 'gh/cypress-io/cypress',
// status: 'success',
// type: 'build',
// stopped_at: '2020-07-20T19:50:07Z'
// }
// ]
// }
return response
}
const waitForAllJobs = async (jobNames, workflowId) => {
let response
try {
response = await getJobStatus(workflowId)
} catch (e) {
console.error(e)
process.exit(1)
}
// if a job is pending, its status will be "blocked"
const blockedJobs = _.filter(response.items, { status: 'blocked' })
const failedJobs = _.filter(response.items, { status: 'failed' })
const runningJobs = _.filter(response.items, { status: 'running' })
const blockedJobNames = _.map(blockedJobs, 'name')
const runningJobNames = _.map(runningJobs, 'name')
const failedJobNames = _.map(failedJobs, 'name')
if (requireAllJobsToPass && _.intersection(jobNames, failedJobNames).length) {
console.error('At least one failing job has prevented percy-finalize from running', failedJobs)
process.exit(1)
}
debug('failed jobs %o', _.map(failedJobs, 'name'))
debug('blocked jobs %o', blockedJobNames)
debug('running jobs %o', runningJobNames)
if (!runningJobs.length || (runningJobs.length === 1 && runningJobs[0].name === jobName)) {
// there are no more jobs to run, or this is the last running job
console.log('all jobs are done, finishing this job')
return Promise.resolve()
}
const futureOrRunning = _.union(blockedJobs, runningJobNames)
const jobsToWaitFor = _.intersection(jobNames, futureOrRunning)
// logging something every time this runs will avoid CI timing out if there is no activity for 10 mins.
console.log(`waiting for jobs, jobs outstanding: ${response.items.length}`)
debug('jobs to wait for %o', jobsToWaitFor)
if (!jobsToWaitFor.length) {
console.log('No more jobs to wait for!')
return Promise.resolve()
}
return Promise.reject(new Error('Jobs have not finished'))
}
const main = () => {
verifyCI()
const args = minimist(process.argv.slice(2), { boolean: false })
const jobNames = _
.chain(args['job-names'])
.split(',')
.without('true')
.map(_.trim)
.compact()
.value()
if (!jobNames.length) {
console.error('Missing argument: --job-names')
console.error('You must pass a comma separated list of Circle CI job names to wait for.')
process.exit(1)
}
debug('received circle jobs: %o', jobNames)
// https://github.com/demmer/bluebird-retry
retry(waitForAllJobs.bind(null, jobNames, workflowId), {
timeout: minutes(30), // max time for this job
interval: seconds(30), // poll intervals
max_interval: seconds(30),
}).then(() => {
console.log('all done')
}, (err) => {
console.error(err)
process.exit(1)
})
}
// execute main function if called from command line
if (require.main === module) {
main()
}
yarn.lock
+37 -10
View File
@@ -7434,6 +7434,11 @@
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-2.1.1.tgz#ceff6a28a5b4867c2dd4a1ba513de278ccbe8bb1"
integrity sha512-/aPsuoj/1Dw/kzhkgz+ES6TxG0zfTMGLwuK2ZG00k/iJzYHTLCE8mVU8EPqEOp/lmxPoq1C1C9RYToRKb2KEfg==
"@sindresorhus/is@^4.0.0":
version "4.6.0"
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f"
integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==
"@sinonjs/commons@^1", "@sinonjs/commons@^1.2.0", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.7.2", "@sinonjs/commons@^1.8.1":
version "1.8.2"
resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.2.tgz#858f5c4b48d80778fde4b9d541f27edc0d56488b"
@@ -8525,10 +8530,10 @@
dependencies:
defer-to-connect "^1.0.1"
"@szmarczak/http-timer@^4.0.0":
version "4.0.5"
resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.5.tgz#bfbd50211e9dfa51ba07da58a14cdfd333205152"
integrity sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==
"@szmarczak/http-timer@^4.0.0", "@szmarczak/http-timer@^4.0.5":
version "4.0.6"
resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807"
integrity sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==
dependencies:
defer-to-connect "^2.0.0"
@@ -14594,6 +14599,11 @@ cacheable-lookup@^4.1.1:
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-4.3.0.tgz#86ff1cb38f648cc6aba28feffe008f808b403550"
integrity sha512-PTUoCeIjj2awloqyVRUL33SjquU1Qv5xuDalYY8WAzd9NnUMUivZnGsOzVsMfg2YuMsWXaXkd/hjnsVoWc/3YA==
cacheable-lookup@^5.0.3:
version "5.0.4"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005"
integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==
cacheable-request@^2.1.1:
version "2.1.4"
resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-2.1.4.tgz#0d808801b6342ad33c91df9d0b44dc09b91e5c3d"
@@ -14620,17 +14630,17 @@ cacheable-request@^6.0.0:
normalize-url "^4.1.0"
responselike "^1.0.2"
cacheable-request@^7.0.1:
version "7.0.1"
resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.1.tgz#062031c2856232782ed694a257fa35da93942a58"
integrity sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==
cacheable-request@^7.0.1, cacheable-request@^7.0.2:
version "7.0.2"
resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.2.tgz#ea0d0b889364a25854757301ca12b2da77f91d27"
integrity sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==
dependencies:
clone-response "^1.0.2"
get-stream "^5.1.0"
http-cache-semantics "^4.0.0"
keyv "^4.0.0"
lowercase-keys "^2.0.0"
normalize-url "^4.1.0"
normalize-url "^6.0.1"
responselike "^2.0.0"
cached-path-relative@^1.0.0, cached-path-relative@^1.0.2:
@@ -22807,6 +22817,23 @@ got@11.1.0:
p-cancelable "^2.0.0"
responselike "^2.0.0"
got@11.8.3:
version "11.8.3"
resolved "https://registry.yarnpkg.com/got/-/got-11.8.3.tgz#f496c8fdda5d729a90b4905d2b07dbd148170770"
integrity sha512-7gtQ5KiPh1RtGS9/Jbv1ofDpBFuq42gyfEib+ejaRBJuj/3tQFeR5+gw57e4ipaU8c/rCjvX6fkQz2lyDlGAOg==
dependencies:
"@sindresorhus/is" "^4.0.0"
"@szmarczak/http-timer" "^4.0.5"
"@types/cacheable-request" "^6.0.1"
"@types/responselike" "^1.0.0"
cacheable-lookup "^5.0.3"
cacheable-request "^7.0.2"
decompress-response "^6.0.0"
http2-wrapper "^1.0.0-beta.5.2"
lowercase-keys "^2.0.0"
p-cancelable "^2.0.0"
responselike "^2.0.0"
got@^6.7.1:
version "6.7.1"
resolved "https://registry.yarnpkg.com/got/-/got-6.7.1.tgz#240cd05785a9a18e561dc1b44b41c763ef1e8db0"
@@ -23939,7 +23966,7 @@ http-signature@~1.3.6:
jsprim "^2.0.2"
sshpk "^1.14.1"
http2-wrapper@^1.0.0-beta.4.4:
http2-wrapper@^1.0.0-beta.4.4, http2-wrapper@^1.0.0-beta.5.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d"
integrity sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==