feat: add burnIn Configuration option (currently a no-op) (#27377)

* feat: add the burnIn Configuration to the config package. Option
currently is a no-op

* chore: make burn in experimental

* chore: set experimentalBurnIn to false by default

cli/types/cypress.d.ts

@@ -3055,6 +3055,18 @@ declare namespace Cypress {
      * @default 'top'
      */
     scrollBehavior: scrollBehaviorOptions
+    /**
+     * Enables flake detection on risky passing tests, such as for new or modified tests or tests which failed or flaked on previous runs.
+     * Note: This feature is only available for Team, Business, and Enterprise plans on Cypress Cloud.
+     * Furthermore, only Business and Enterprise plans can override the `experimentalBurnIn.default` or `experimentalBurnIn.flaky` settings.
+     * TODO: make sure experimental link is available in documentation before experimentalBurnIn release!
+     * @see https://on.cypress.io/experiments#Experimental-Burn-In
+     * @default false
+     */
+    experimentalBurnIn: boolean | {
+      default: number
+      flaky: number
+    },
     /**
      * Indicates whether Cypress should allow CSP header directives from the application under test.
      * - When this option is set to `false`, Cypress will strip the entire CSP header.

packages/config/__snapshots__/index.spec.ts.js

@@ -21,6 +21,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1
   'animationDistanceThreshold': 5,
   'baseUrl': null,
   'blockHosts': null,
+  'experimentalBurnIn': false,
   'chromeWebSecurity': true,
   'clientCertificates': [],
   'component': {
@@ -109,6 +110,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f
   'animationDistanceThreshold': 5,
   'baseUrl': null,
   'blockHosts': null,
+  'experimentalBurnIn': false,
   'chromeWebSecurity': true,
   'clientCertificates': [],
   'component': {
@@ -198,6 +200,7 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key
   'arch',
   'baseUrl',
   'blockHosts',
+  'experimentalBurnIn',
   'chromeWebSecurity',
   'clientCertificates',
   'component',

packages/config/__snapshots__/validation.spec.ts.js

@@ -261,3 +261,74 @@ exports['invalid upper bound'] = {
   'value': 52,
   'type': 'a valid CRF number between 1 & 51, 0 or false to disable compression, or true to use the default compression of 32',
 }
+
+exports['missing key "flaky"'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': 1,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['missing key "default"'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'flaky': 1,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['keys cannot be zero key'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': 0,
+    'flaky': 0,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['keys cannot be zero'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': 0,
+    'flaky': 0,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['keys cannot be negative'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': -3,
+    'flaky': -40,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['keys cannot be floating point numbers'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': 5.7,
+    'flaky': 8.22,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['keys cannot be Infinity'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': null,
+    'flaky': null,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}
+
+exports['extraneous keys'] = {
+  'key': 'experimentalBurnIn',
+  'value': {
+    'default': 3,
+    'flaky': 5,
+    'notflaky': null,
+  },
+  'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.',
+}

packages/config/src/options.ts

@@ -152,6 +152,11 @@ const driverConfigOptions: Array<DriverConfigOption> = [
     validation: validate.isStringOrArrayOfStrings,
     overrideLevel: 'any',
     requireRestartOnChange: 'server',
+  }, {
+    // experimentalBurnIn config is only considered in run-mode while recording to cypress-cloud
+    name: 'experimentalBurnIn',
+    defaultValue: false,
+    validation: validate.isValidBurnInConfig,
   }, {
     name: 'chromeWebSecurity',
     defaultValue: true,

packages/config/src/validation.ts

@@ -384,3 +384,25 @@ export function isNullOrArrayOfStrings (key: string, value: any): ErrResult | tr
 
   return errMsg(key, value, 'an array of strings or null')
 }
+
+export function isValidBurnInConfig (key: string, value: any): ErrResult | true {
+  // false can be provided to turn off test burn in
+  if (_.isBoolean(value)) {
+    return true
+  }
+
+  if (_.isPlainObject(value)) {
+    const { default: defaultKey, flaky: flakyKey, ...extraneousKeys } = value
+
+    if (defaultKey !== undefined && defaultKey !== undefined && _.isEmpty(extraneousKeys)) {
+      const isDefaultKeyValid = _.isInteger(defaultKey) && _.inRange(defaultKey, 1, Infinity)
+      const isFlakyKeyValid = _.isInteger(flakyKey) && _.inRange(flakyKey, 1, Infinity)
+
+      if (isDefaultKeyValid && isFlakyKeyValid) {
+        return true
+      }
+    }
+  }
+
+  return errMsg(key, value, 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.')
+}

packages/config/test/project/utils.spec.ts

@@ -1073,6 +1073,7 @@ describe('config/src/project/utils', () => {
             baseUrl: { value: null, from: 'default' },
             blockHosts: { value: null, from: 'default' },
             browsers: { value: [], from: 'default' },
+            experimentalBurnIn: { value: false, from: 'default' },
             chromeWebSecurity: { value: true, from: 'default' },
             clientCertificates: { value: [], from: 'default' },
             defaultCommandTimeout: { value: 4000, from: 'default' },
@@ -1172,6 +1173,7 @@ describe('config/src/project/utils', () => {
             baseUrl: { value: 'http://localhost:8080', from: 'config' },
             blockHosts: { value: null, from: 'default' },
             browsers: { value: [], from: 'default' },
+            experimentalBurnIn: { value: false, from: 'default' },
             chromeWebSecurity: { value: true, from: 'default' },
             clientCertificates: { value: [], from: 'default' },
             defaultCommandTimeout: { value: 4000, from: 'default' },

packages/config/test/validation.spec.ts

@@ -509,4 +509,119 @@ describe('config/src/validation', () => {
       return snapshot('invalid upper bound', upperBoundMsg)
     })
   })
+
+  describe('.isValidBurnInConfig', () => {
+    it('validates default config if enabled', () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: 3,
+        flaky: 5,
+      })
+
+      expect(validate).to.be.true
+    })
+
+    it('validates false', () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', false)
+
+      expect(validate).to.be.true
+    })
+
+    it('validates true', () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', true)
+
+      expect(validate).to.be.true
+    })
+
+    it('does not allow extraneous keys', () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: 3,
+        flaky: 5,
+        // extraneous key
+        notflaky: null,
+      })
+
+      expect(validate).to.not.be.true
+
+      return snapshot(`extraneous keys`, validate)
+    })
+
+    const possibleConfigKeys = ['default', 'flaky']
+
+    possibleConfigKeys.forEach((burnInConfigKey) => {
+      it(`Does not populate missing config value with default values (key "${burnInConfigKey}")`, () => {
+        const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+          [burnInConfigKey]: 1,
+        })
+
+        expect(validate).to.not.be.true
+
+        const missingKey = possibleConfigKeys.find((key) => key !== burnInConfigKey)
+
+        return snapshot(`missing key "${missingKey}"`, validate)
+      })
+    })
+
+    it(`does not allow keys to be zero`, () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: 0,
+        flaky: 0,
+      })
+
+      expect(validate).to.not.be.true
+
+      return snapshot(`keys cannot be zero`, validate)
+    })
+
+    it(`does not allow keys to be negative`, () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: -3,
+        flaky: -40,
+      })
+
+      expect(validate).to.not.be.true
+
+      return snapshot(`keys cannot be negative`, validate)
+    })
+
+    it(`does not allow keys to be floating point numbers`, () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: 5.7,
+        flaky: 8.22,
+      })
+
+      expect(validate).to.not.be.true
+
+      return snapshot(`keys cannot be floating point numbers`, validate)
+    })
+
+    it(`does not allow keys to be Infinity`, () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: Infinity,
+        flaky: Infinity,
+      })
+
+      expect(validate).to.not.be.true
+
+      return snapshot(`keys cannot be Infinity`, validate)
+    })
+
+    it(`tests lower bound`, () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: 1,
+        flaky: 1,
+      })
+
+      expect(validate).to.be.true
+    })
+
+    // TODO: revisit limit on 'default' and 'flaky' keys to set sane limits
+    it(`tests upper bound (currently no limit and is subject to change)`, () => {
+      const validate = validation.isValidBurnInConfig('experimentalBurnIn', {
+        default: 100000,
+        flaky: 142342342,
+      })
+
+      expect(validate).to.be.true
+    })
+  })
 })

packages/frontend-shared/src/locales/en-US.json

@@ -570,6 +570,10 @@
     "experiments": {
       "title": "Experiments",
       "description": "If you'd like to try out new features that we're working on, you can enable beta features for your project by turning on the experimental features you'd like to try. {0}",
+      "experimentalBurnIn": {
+        "name": "Burn-in",
+        "description": "Enables flake detection on risky passing tests, such as for new or modified tests or tests which failed or flaked on previous runs. Note: This feature is only available for Team, Business, and Enterprise plans on Cypress Cloud. Furthermore, only Business and Enterprise plans can override the `experimentalBurnIn.default` or `experimentalBurnIn.flaky` settings."
+      },
       "experimentalCspAllowList": {
         "name": "CSP Allow List",
         "description": "Enables Cypress to selectively permit Content-Security-Policy and Content-Security-Policy-Report-Only header directives, including those that might otherwise block Cypress from running."