feat: Add 'slowTestThreshold' and fix this.slow() inside specs (#18496)

Co-authored-by: Chris Breiding <chrisbreiding@users.noreply.github.com>

cli/schema/cypress.schema.json

@@ -44,6 +44,11 @@
           "default": null,
           "description": "The reporter options used. Supported options depend on the reporter. See https://on.cypress.io/reporters#Reporter-Options"
         },
+        "slowTestThreshold": {
+          "type": "number",
+          "default": 10000,
+          "description": "Slow test threshold in milliseconds. Only affects the visual output of some reporters. For example, the spec reporter will display the test time in yellow if over the threshold. See https://on.cypress.io/configuration#Timeouts"
+        },
         "testFiles": {
           "type": [
             "string",

cli/types/cypress-npm-api.d.ts

@@ -91,6 +91,10 @@ declare namespace CypressCommandLine {
      * Specify mocha reporter options
      */
     reporterOptions: any
+    /**
+     * Slow test threshold in milliseconds. Only affects the visual output of some reporters. For example, the spec reporter will display the test time in yellow if over the threshold.
+     */
+    slowTestThreshold: number
     /**
      * Specify the specs to run
      */

cli/types/cypress.d.ts

@@ -2577,6 +2577,11 @@ declare namespace Cypress {
      * @default "spec"
      */
     reporterOptions: { [key: string]: any }
+    /**
+     * Slow test threshold in milliseconds. Only affects the visual output of some reporters. For example, the spec reporter will display the test time in yellow if over the threshold.
+     * @default 10000
+     */
+    slowTestThreshold: number
     /**
      * Whether Cypress will watch and restart tests on test file changes
      * @default true

packages/driver/src/cypress/mocha.ts

@@ -500,6 +500,8 @@ const create = (specWindow, Cypress, config) => {
 
   const _mocha = createMocha(specWindow)
 
+  _mocha.slow(config('slowTestThreshold'))
+
   const _runner = getRunner(_mocha)
 
   _mocha.suite.file = Cypress.spec.relative

packages/driver/src/cypress/runner.ts

@@ -21,7 +21,7 @@ const TEST_BEFORE_RUN_EVENT = 'runner:test:before:run'
 const TEST_AFTER_RUN_EVENT = 'runner:test:after:run'
 
 const RUNNABLE_LOGS = 'routes agents commands hooks'.split(' ')
-const RUNNABLE_PROPS = '_testConfig id order title _titlePath root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
+const RUNNABLE_PROPS = '_testConfig id order title _titlePath root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries _slow'.split(' ')
 
 const debug = debugFn('cypress:driver:runner')
 const debugErrors = debugFn('cypress:driver:errors')
@@ -581,6 +581,10 @@ const normalize = (runnable, tests, initialTests, onRunnable, onLogsById, getRun
         wrappedRunnable._testConfig = cfg
       }
 
+      if (cfg.slowTestThreshold) {
+        runnable.slow(cfg.slowTestThreshold)
+      }
+
       wrappedRunnable._titlePath = runnable.titlePath()
     }
 

packages/server/lib/config.ts

@@ -257,6 +257,11 @@ export function mergeDefaults (config: Record<string, any> = {}, options: Record
 
   _.defaults(config, defaultValues)
 
+  // Default values can be functions, in which case they are evaluated
+  // at runtime - for example, slowTestThreshold where the default value
+  // varies between e2e and component testing.
+  config = _.mapValues(config, (value) => (typeof value === 'function' ? value(options) : value))
+
   // split out our own app wide env from user env variables
   // and delete envFile
   config.env = parseEnv(config, options.env, resolved)
@@ -279,7 +284,7 @@ export function mergeDefaults (config: Record<string, any> = {}, options: Record
     config.numTestsKeptInMemory = 0
   }
 
-  config = setResolvedConfigValues(config, defaultValues, resolved)
+  config = setResolvedConfigValues(config, defaultValues, resolved, options)
 
   if (config.port) {
     config = setUrls(config)
@@ -304,10 +309,10 @@ export function mergeDefaults (config: Record<string, any> = {}, options: Record
   .then(_.partialRight(setNodeBinary, options.onWarning))
 }
 
-export function setResolvedConfigValues (config, defaults, resolved) {
+export function setResolvedConfigValues (config, defaults, resolved, options) {
   const obj = _.clone(config)
 
-  obj.resolved = resolveConfigValues(config, defaults, resolved)
+  obj.resolved = resolveConfigValues(config, defaults, resolved, options)
   debug('resolved config is %o', obj.resolved.browsers)
 
   return obj
@@ -412,7 +417,7 @@ export function updateWithPluginValues (cfg, overrides) {
 // combines the default configuration object with values specified in the
 // configuration file like "cypress.json". Values in configuration file
 // overwrite the defaults.
-export function resolveConfigValues (config, defaults, resolved = {}) {
+export function resolveConfigValues (config, defaults, resolved = {}, options = {}) {
   // pick out only known configuration keys
   return _
   .chain(config)
@@ -436,7 +441,9 @@ export function resolveConfigValues (config, defaults, resolved = {}) {
       return source(r)
     }
 
-    if (!(!_.isEqual(config[key], defaults[key]) && key !== 'browsers')) {
+    const defaultValue = typeof defaults[key] === 'function' ? defaults[key](options) : defaults[key]
+
+    if (!(!_.isEqual(config[key], defaultValue) && key !== 'browsers')) {
       // "browsers" list is special, since it is dynamic by default
       // and can only be ovewritten via plugins file
       return source('default')

packages/server/lib/config_options.ts

@@ -216,6 +216,10 @@ export const options = [
     defaultValue: 'cypress/screenshots',
     validation: v.isStringOrFalse,
     isFolder: true,
+  }, {
+    name: 'slowTestThreshold',
+    defaultValue: (options: Record<string, any>) => options.testingType === 'component' ? 250 : 10000,
+    validation: v.isNumber,
   }, {
     name: 'socketId',
     defaultValue: null,

packages/server/lib/reporter.js

@@ -8,6 +8,7 @@ const Mocha = require('mocha-7.0.1')
 const mochaReporters = require('mocha-7.0.1/lib/reporters')
 const mochaCreateStatsCollector = require('mocha-7.0.1/lib/stats-collector')
 const mochaColor = mochaReporters.Base.color
+const mochaSymbols = mochaReporters.Base.symbols
 
 const debug = require('debug')('cypress:server:reporter')
 const Promise = require('bluebird')
@@ -293,6 +294,34 @@ class Reporter {
       reporterOptions: this.reporterOptions,
     })
 
+    if (this.reporterName === 'spec') {
+      // Unfortunately the reporter doesn't expose its indentation logic, so we have to replicate it here
+      let indents = 0
+
+      this.runner.on('suite', function (suite) {
+        ++indents
+      })
+
+      this.runner.on('suite end', function () {
+        --indents
+      })
+
+      // Override the default reporter to always show test timing even for fast tests
+      // and display slow ones in yellow rather than red
+      this.runner._events.pass[2] = function (test) {
+        const durationColor = test.speed === 'slow' ? 'medium' : 'fast'
+        const fmt =
+          Array(indents).join('  ') +
+          mochaColor('checkmark', `  ${ mochaSymbols.ok}`) +
+          mochaColor('pass', ' %s') +
+          mochaColor(durationColor, ' (%dms)')
+
+        // Log: `✓ test title (300ms)` when a test passes
+        // eslint-disable-next-line no-console
+        console.log(fmt, test.title, test.duration)
+      }
+    }
+
     this.runner.ignoreLeaks = true
   }
 

packages/server/test/unit/browsers/browsers_spec.js

@@ -5,7 +5,7 @@ const utils = require(`${root}../lib/browsers/utils`)
 const snapshot = require('snap-shot-it')
 
 const normalizeBrowsers = (message) => {
-  return message.replace(/(found on your system are:)((\n.*)*)/, '$1\n- chrome\n- firefox\n- electron')
+  return message.replace(/(found on your system are:)(?:\n- .*)*/, '$1\n- chrome\n- firefox\n- electron')
 }
 
 // When we added component testing mode, we added the option for electron to be omitted

packages/server/test/unit/config_spec.js

@@ -1079,6 +1079,14 @@ describe('lib/config', () => {
       }
     })
 
+    it('slowTestThreshold=10000 for e2e', function () {
+      return this.defaults('slowTestThreshold', 10000, {}, { testingType: 'e2e' })
+    })
+
+    it('slowTestThreshold=250 for component', function () {
+      return this.defaults('slowTestThreshold', 250, {}, { testingType: 'component' })
+    })
+
     it('port=null', function () {
       return this.defaults('port', null)
     })
@@ -1462,6 +1470,7 @@ describe('lib/config', () => {
             retries: { value: { runMode: 0, openMode: 0 }, from: 'default' },
             screenshotOnRunFailure: { value: true, from: 'default' },
             screenshotsFolder: { value: 'cypress/screenshots', from: 'default' },
+            slowTestThreshold: { value: 10000, from: 'default' },
             supportFile: { value: 'cypress/support', from: 'default' },
             taskTimeout: { value: 60000, from: 'default' },
             testFiles: { value: '**/*.*', from: 'default' },
@@ -1568,6 +1577,7 @@ describe('lib/config', () => {
             retries: { value: { runMode: 0, openMode: 0 }, from: 'default' },
             screenshotOnRunFailure: { value: true, from: 'default' },
             screenshotsFolder: { value: 'cypress/screenshots', from: 'default' },
+            slowTestThreshold: { value: 10000, from: 'default' },
             supportFile: { value: 'cypress/support', from: 'default' },
             taskTimeout: { value: 60000, from: 'default' },
             testFiles: { value: '**/*.*', from: 'default' },

packages/server/test/unit/reporter_spec.js

@@ -111,7 +111,6 @@ describe('lib/reporter', () => {
     it('recursively creates suites for fullTitle', function () {
       const args = this.reporter.parseArgs('fail', [this.testObj])
 
-      console.log(args)
       expect(args[0]).to.eq('fail')
 
       const title = 'TodoMVC - React When page is initially opened should focus on the todo input field'

system-tests/projects/e2e/cypress/integration/slowTestThreshold_spec.js

@@ -0,0 +1,16 @@
+/* eslint-disable no-undef */
+describe('slowTestThreshold', () => {
+  it('passes inherited', () => {
+    cy.wait(5)
+    cy.wrap(true).should('be.true')
+  })
+
+  it('passes quickly', { slowTestThreshold: 10000 }, () => {
+    cy.wrap(true).should('be.true')
+  })
+
+  it('passes slowly', { slowTestThreshold: 1 }, () => {
+    cy.wait(5)
+    cy.wrap(true).should('be.true')
+  })
+})

system-tests/test/reporters_spec.js

@@ -145,4 +145,21 @@ describe('e2e reporters', () => {
       reporterOptions: 'topLevelSuite=top suite,flowId=12345,useStdError=\'true\',useStdError=\'true\',recordHookFailures=\'true\',actualVsExpected=\'true\'',
     })
   })
+
+  it('shows slow tests in yellow', function () {
+    return systemTests.exec(this, {
+      spec: 'slowTestThreshold_spec.js',
+      snapshot: false,
+      config: {
+        slowTestThreshold: 1,
+      },
+      processEnv: {
+        MOCHA_COLORS: 1,
+      },
+    }).then((result) => {
+      expect(result.stdout.match(/passes inherited(.*)/)[1]).to.contain('\u001b[33m')
+      expect(result.stdout.match(/passes quickly(.*)/)[1]).not.to.contain('\u001b[33m')
+      expect(result.stdout.match(/passes slowly(.*)/)[1]).to.contain('\u001b[33m')
+    })
+  })
 })