From bf6a52ab3d6b674985042c2871be4f36ade7652a Mon Sep 17 00:00:00 2001 From: Adam Stone Date: Fri, 18 Nov 2022 14:43:50 -0500 Subject: [PATCH] feat: add cloud recommendation message to CI output (#24680) --- cli/types/cypress-npm-api.d.ts | 3 + packages/config/test/project/utils.spec.ts | 4 + packages/config/test/utils.spec.ts | 4 + packages/server/__snapshots__/cypress_spec.js | 56 ++++++++++++++ packages/server/lib/modes/run.ts | 1 + packages/server/lib/util/ci_provider.js | 5 ++ packages/server/lib/util/print-run.ts | 26 ++++++- packages/server/package.json | 1 + .../server/test/integration/cypress_spec.js | 77 +++++++++++++++++++ packages/server/test/unit/config_spec.js | 2 + .../v8-snapshot/src/setup/force-no-rewrite.ts | 2 + 11 files changed, 179 insertions(+), 2 deletions(-) diff --git a/cli/types/cypress-npm-api.d.ts b/cli/types/cypress-npm-api.d.ts index 126f502028..394c12a7a0 100644 --- a/cli/types/cypress-npm-api.d.ts +++ b/cli/types/cypress-npm-api.d.ts @@ -229,6 +229,7 @@ declare namespace CypressCommandLine { startedAt: dateTimeISO endedAt: dateTimeISO duration: ms + wallClockDuration?: number } /** * Reporter name like "spec" @@ -259,8 +260,10 @@ declare namespace CypressCommandLine { * resolved filename of the spec */ absolute: string + relativeToCommonRoot: string } shouldUploadVideo: boolean + skippedSpec: boolean } /** diff --git a/packages/config/test/project/utils.spec.ts b/packages/config/test/project/utils.spec.ts index 75d01a0113..63b7904e2f 100644 --- a/packages/config/test/project/utils.spec.ts +++ b/packages/config/test/project/utils.spec.ts @@ -27,6 +27,10 @@ import path from 'node:path' const debug = Debug('test') describe('config/src/project/utils', () => { + beforeEach(function () { + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + }) + before(function () { this.env = process.env; diff --git a/packages/config/test/utils.spec.ts b/packages/config/test/utils.spec.ts index 3f19939654..ae74a5c1c0 100644 --- a/packages/config/test/utils.spec.ts +++ b/packages/config/test/utils.spec.ts @@ -10,6 +10,10 @@ import { } from '../src/project/utils' describe('config/src/utils', () => { + beforeEach(function () { + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + }) + describe('hideKeys', () => { it('removes middle part of the string', () => { const hidden = hideKeys('12345-xxxx-abcde') diff --git a/packages/server/__snapshots__/cypress_spec.js b/packages/server/__snapshots__/cypress_spec.js index aed09a3725..db7ef02924 100644 --- a/packages/server/__snapshots__/cypress_spec.js +++ b/packages/server/__snapshots__/cypress_spec.js @@ -351,3 +351,59 @@ exports['Long Cypress Cloud URL'] = ` Recorded Run: http://cloud.cypress.io/this-is-a-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-long-url ` + +exports['CLOUD_RECOMMENDATION_MESSAGE'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (test1.js) │ + │ Searched: tests/test1.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test1.js (1 of 1) + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: undefined │ + │ Passing: undefined │ + │ Failing: 1 │ + │ Pending: undefined │ + │ Skipped: undefined │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: undefined seconds │ + │ Spec Ran: test1.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ test1.js XX:XX - - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX - - 1 - - + +---------------------------------------------------------------------------------------------------- + + Having trouble debugging your CI failures? + + Record your runs to Cypress Cloud to watch video recordings for each test, + debug failing and flaky tests, and integrate with your favorite tools. + + >> https://on.cypress.io/cloud-get-started + +---------------------------------------------------------------------------------------------------- +` diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 384d6642ef..67969484ac 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -1012,6 +1012,7 @@ async function ready (options: { projectRoot: string, record: boolean, key: stri if (!options.quiet) { printResults.renderSummaryTable(runUrl, results) + printResults.maybeLogCloudRecommendationMessage(results.runs || [], record) } return results diff --git a/packages/server/lib/util/ci_provider.js b/packages/server/lib/util/ci_provider.js index 965b13f522..4942e22477 100644 --- a/packages/server/lib/util/ci_provider.js +++ b/packages/server/lib/util/ci_provider.js @@ -1,6 +1,9 @@ const _ = require('lodash') +const isCi = require('is-ci') const debug = require('debug')('cypress:server') +const getIsCi = () => isCi + const join = (char, ...pieces) => { return _.chain(pieces).compact().join(char).value() } @@ -668,6 +671,8 @@ const detectableCiBuildIdProviders = () => { } module.exports = { + getIsCi, + list, provider, diff --git a/packages/server/lib/util/print-run.ts b/packages/server/lib/util/print-run.ts index e9cb1a7689..1de7fc562b 100644 --- a/packages/server/lib/util/print-run.ts +++ b/packages/server/lib/util/print-run.ts @@ -9,6 +9,7 @@ import duration from './duration' import newlines from './newlines' import env from './env' import terminal from './terminal' +import { getIsCi } from './ci_provider' import * as experiments from '../experiments' import type { SpecFile } from '@packages/types' import type { Cfg } from '../project-base' @@ -22,11 +23,18 @@ type Screenshot = { specName: string } +export const cloudRecommendationMessage = ` + Having trouble debugging your CI failures? + + Record your runs to Cypress Cloud to watch video recordings for each test, + debug failing and flaky tests, and integrate with your favorite tools. +` + function color (val: any, c: string) { return chalk[c](val) } -function gray (val: any) { +export function gray (val: any) { return color(val, 'gray') } @@ -274,7 +282,21 @@ export function displaySpecHeader (name: string, curr: number, total: number, es } } -export function renderSummaryTable (runUrl: string | undefined, results: any) { +export function maybeLogCloudRecommendationMessage (runs: CypressCommandLine.RunResult[], record: boolean) { + if (!getIsCi() || env.get('CYPRESS_COMMERCIAL_RECOMMENDATIONS') === '0' || record) { + return + } + + if (runs.some((run) => run.stats.failures > 0)) { + terminal.divider('-') + console.log(cloudRecommendationMessage) + console.log(` >>`, color('https://on.cypress.io/cloud-get-started', 'cyan')) + console.log('') + terminal.divider('-') + } +} + +export function renderSummaryTable (runUrl: string | undefined, results: CypressCommandLine.CypressRunResult) { const { runs } = results console.log('') diff --git a/packages/server/package.json b/packages/server/package.json index 50b1baccc7..81f342bef2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -74,6 +74,7 @@ "http-proxy": "https://github.com/cypress-io/node-http-proxy.git#9322b4b69b34f13a6f3874e660a35df3305179c6", "human-interval": "1.0.0", "image-size": "0.8.3", + "is-ci": "^3.0.0", "is-fork-pr": "2.5.0", "is-html": "2.0.0", "jimp": "0.14.0", diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index b7bc6e7fa4..1e0fa23cfb 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -44,6 +44,7 @@ const electronApp = require('../../lib/util/electron-app') const savedState = require(`../../lib/saved_state`) const { getCtx, clearCtx, setCtx, makeDataContext } = require(`../../lib/makeDataContext`) const { BrowserCriClient } = require(`../../lib/browsers/browser-cri-client`) +const { cloudRecommendationMessage } = require('../../lib/util/print-run') const TYPICAL_BROWSERS = [ { @@ -349,6 +350,58 @@ describe('lib/cypress', () => { sinon.stub(commitInfo, 'getRemoteOrigin').resolves('remoteOrigin') }) + describe('cloud recommendation message', () => { + it('gets logged when in CI and there is a failure', function () { + const relativePath = path.relative(cwd(), this.todosPath) + + sinon.stub(ciProvider, 'getIsCi').returns(true) + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + globalThis.CY_TEST_MOCK.listenForProjectEnd = { stats: { failures: 1 } } + + return cypress.start([`--run-project=${this.todosPath}`, `--spec=${relativePath}/tests/test1.js`]).then(() => { + expect(console.log).to.be.calledWith(cloudRecommendationMessage) + + snapshotConsoleLogs('CLOUD_RECOMMENDATION_MESSAGE') + }) + }) + + it('does not get logged if CYPRESS_COMMERCIAL_RECOMMENDATIONS is set to 0', function () { + const relativePath = path.relative(cwd(), this.todosPath) + + sinon.stub(ciProvider, 'getIsCi').returns(true) + process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS = '0' + globalThis.CY_TEST_MOCK.listenForProjectEnd = { stats: { failures: 1 } } + + return cypress.start([`--run-project=${this.todosPath}`, `--spec=${relativePath}/tests/test1.js`]).then(() => { + expect(console.log).not.to.be.calledWith(cloudRecommendationMessage) + }) + }) + + it('does not get logged if all tests pass', function () { + const relativePath = path.relative(cwd(), this.todosPath) + + sinon.stub(ciProvider, 'getIsCi').returns(true) + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + globalThis.CY_TEST_MOCK.listenForProjectEnd = { stats: { failures: 0 } } + + return cypress.start([`--run-project=${this.todosPath}`, `--spec=${relativePath}/tests/test1.js`]).then(() => { + expect(console.log).not.to.be.calledWith(cloudRecommendationMessage) + }) + }) + + it('does not get logged if not running in CI', function () { + const relativePath = path.relative(cwd(), this.todosPath) + + sinon.stub(ciProvider, 'getIsCi').returns(undefined) + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + globalThis.CY_TEST_MOCK.listenForProjectEnd = { stats: { failures: 1 } } + + return cypress.start([`--run-project=${this.todosPath}`, `--spec=${relativePath}/tests/test1.js`]).then(() => { + expect(console.log).not.to.be.calledWith(cloudRecommendationMessage) + }) + }) + }) + it('runs project headlessly and exits with exit code 0', function () { return cypress.start([`--run-project=${this.todosPath}`]) .then(() => { @@ -881,6 +934,10 @@ describe('lib/cypress', () => { }) describe('config overrides', () => { + beforeEach(function () { + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + }) + it('can override default values', function () { return cypress.start([`--run-project=${this.todosPath}`, '--config=requestTimeout=1234,videoCompression=false']) .then(() => { @@ -1069,6 +1126,7 @@ describe('lib/cypress', () => { describe('--env', () => { beforeEach(() => { process.env = _.omit(process.env, 'CYPRESS_DEBUG') + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS globalThis.CY_TEST_MOCK.listenForProjectEnd = { stats: { failures: 0 } } }) @@ -1566,6 +1624,25 @@ describe('lib/cypress', () => { return snapshotConsoleLogs('CLOUD_STALE_RUN 1') }) }) + + describe('cloud recommendation message', () => { + it('does not display if --record is passed', function () { + sinon.stub(ciProvider, 'getIsCi').returns(true) + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + globalThis.CY_TEST_MOCK.listenForProjectEnd = { stats: { failures: 1 } } + + return cypress.start([ + `--run-project=${this.recordPath}`, + '--record', + '--key=token-123', + '--group=electron-smoke-tests', + '--ciBuildId=ciBuildId123', + ]) + .then(() => { + expect(console.log).not.to.be.calledWith(cloudRecommendationMessage) + }) + }) + }) }) context('--return-pkg', () => { diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index 0aee784d16..a3cafbe2f5 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -25,6 +25,8 @@ describe('lib/config', () => { context('.get', () => { beforeEach(async function () { + delete process.env.CYPRESS_COMMERCIAL_RECOMMENDATIONS + this.ctx = getCtx() this.projectRoot = '/_test-output/path/to/project' diff --git a/tooling/v8-snapshot/src/setup/force-no-rewrite.ts b/tooling/v8-snapshot/src/setup/force-no-rewrite.ts index 7e8f448fb6..9c69566620 100644 --- a/tooling/v8-snapshot/src/setup/force-no-rewrite.ts +++ b/tooling/v8-snapshot/src/setup/force-no-rewrite.ts @@ -69,4 +69,6 @@ export default [ 'node_modules/prettier/parser-meriyah.js', 'node_modules/prettier/parser-typescript.js', 'node_modules/prettier/third-party.js', + 'packages/server/node_modules/is-ci/index.js', + 'packages/server/node_modules/ci-info/index.js', ]