feat: Add after:spec event (#14178)

cli/schema/cypress.schema.json

@@ -243,7 +243,7 @@
     "experimentalRunEvents": {
       "type": "boolean",
       "default": false,
-      "description": "Allows listening to the `before:spec` event in the plugins file."
+      "description": "Allows listening to the `before:spec` and `after:spec` events in the plugins file."
     },
     "experimentalSourceRewriting": {
       "type": "boolean",

cli/types/cypress.d.ts

@@ -1,3 +1,5 @@
+/// <reference path="./cypress-npm-api.d.ts" />
+
 declare namespace Cypress {
   type FileContents = string | any[] | object
   type HistoryDirection = 'back' | 'forward'
@@ -116,6 +118,17 @@ declare namespace Cypress {
    */
   type CypressSpecType = 'integration' | 'component'
 
+  /**
+   * A Cypress spec.
+   */
+  interface Spec {
+    name: string // "config_passing_spec.js"
+    relative: string // "cypress/integration/config_passing_spec.js" or "__all" if clicked all specs button
+    absolute: string // "/Users/janelane/app/cypress/integration/config_passing_spec.js"
+    specFilter?: string // optional spec filter used by the user
+    specType?: CypressSpecType
+  }
+
   /**
    * Window type for Application Under Test(AUT)
    */
@@ -227,23 +240,17 @@ declare namespace Cypress {
     /**
      * Currently executing spec file.
      * @example
-    ```
-    Cypress.spec
-    // {
-    //  name: "config_passing_spec.coffee",
-    //  relative: "cypress/integration/config_passing_spec.coffee",
-    //  absolute: "/users/smith/projects/web/cypress/integration/config_passing_spec.coffee"
-    //  specType: "integration"
-    // }
-    ```
+     * ```
+     * Cypress.spec
+     * // {
+     * //  name: "config_passing_spec.coffee",
+     * //  relative: "cypress/integration/config_passing_spec.coffee",
+     * //  absolute: "/users/smith/projects/web/cypress/integration/config_passing_spec.coffee"
+     * //  specType: "integration"
+     * // }
+     * ```
      */
-    spec: {
-      name: string // "config_passing_spec.coffee"
-      relative: string // "cypress/integration/config_passing_spec.coffee" or "__all" if clicked all specs button
-      absolute: string
-      specFilter?: string // optional spec filter used by the user
-      specType?: CypressSpecType
-    }
+    spec: Spec
 
     /**
      * Information about the browser currently running the tests
@@ -2570,7 +2577,7 @@ declare namespace Cypress {
      */
     firefoxGcInterval: Nullable<number | { runMode: Nullable<number>, openMode: Nullable<number> }>
     /**
-     * Allows listening to the `before:spec` event in the plugins file.
+     * Allows listening to the `before:spec` and `after:spec` events in the plugins file.
      * @default false
      */
     experimentalRunEvents: boolean
@@ -2597,7 +2604,7 @@ declare namespace Cypress {
     includeShadowDom: boolean
   }
 
-  interface TestConfigOverrides extends Partial<Pick<ConfigOptions, 'animationDistanceThreshold' | 'baseUrl' | 'defaultCommandTimeout' | 'env' | 'execTimeout' | 'includeShadowDom' | 'requestTimeout' | 'responseTimeout' | 'retries' | 'scrollBehavior' | 'taskTimeout' | 'viewportHeight' | 'viewportWidth' |  'waitForAnimations'>> {
+  interface TestConfigOverrides extends Partial<Pick<ConfigOptions, 'animationDistanceThreshold' | 'baseUrl' | 'defaultCommandTimeout' | 'env' | 'execTimeout' | 'includeShadowDom' | 'requestTimeout' | 'responseTimeout' | 'retries' | 'scrollBehavior' | 'taskTimeout' | 'viewportHeight' | 'viewportWidth' | 'waitForAnimations'>> {
     browser?: IsBrowserMatcher | IsBrowserMatcher[]
   }
 
@@ -4993,8 +5000,10 @@ declare namespace Cypress {
   }
 
   interface PluginEvents {
-    (action: 'before:browser:launch', fn: (browser: Browser, browserLaunchOptions: BrowserLaunchOptions) => void | BrowserLaunchOptions | Promise<BrowserLaunchOptions>): void
     (action: 'after:screenshot', fn: (details: ScreenshotDetails) => void | AfterScreenshotReturnObject | Promise<AfterScreenshotReturnObject>): void
+    (action: 'after:spec', fn: (spec: Spec, results: CypressCommandLine.RunResult) => void | Promise<void>): void
+    (action: 'before:spec', fn: (spec: Spec) => void | Promise<void>): void
+    (action: 'before:browser:launch', fn: (browser: Browser, browserLaunchOptions: BrowserLaunchOptions) => void | BrowserLaunchOptions | Promise<BrowserLaunchOptions>): void
     (action: 'file:preprocessor', fn: (file: FileObject) => string | Promise<string>): void
     (action: 'task', tasks: Tasks): void
   }

packages/server/__snapshots__/4_plugin_run_events_spec.ts.js

@@ -1,4 +1,13 @@
-exports['e2e plugin run events / sends server events'] = `
+exports['e2e plugin run events / fails if experimentalRunEvents is not enabled'] = `
+The following validation error was thrown by your plugins file (\`/foo/bar/.projects/plugin-run-events/cypress/plugins/index.js\`).
+
+ Error: The \`before:spec\` event requires the experimentalRunEvents flag to be enabled.
+
+To enable it, set \`"experimentalRunEvents": true\` in your cypress.json
+      [stack trace lines]
+`
+
+exports['e2e plugin run events / sends events'] = `
 
 ====================================================================================================
 
@@ -39,6 +48,8 @@ before:spec is awaited
   │ Spec Ran:     run_events_spec_1.js                                                             │
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
 
+spec:end: cypress/integration/run_events_spec_1.js { tests: 1, passes: 1, failures: 0 }
+after:spec is awaited
 
 ────────────────────────────────────────────────────────────────────────────────────────────────────
                                                                                                     
@@ -66,6 +77,8 @@ before:spec is awaited
   │ Spec Ran:     run_events_spec_2.js                                                             │
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
 
+spec:end: cypress/integration/run_events_spec_2.js { tests: 1, passes: 1, failures: 0 }
+after:spec is awaited
 
 ====================================================================================================
 
@@ -83,7 +96,7 @@ before:spec is awaited
 
 `
 
-exports['e2e plugin run events / fails run if server event handler throws'] = `
+exports['e2e plugin run events / fails run if event handler throws'] = `
 
 ====================================================================================================
 
@@ -110,12 +123,3 @@ Error: error thrown in before:spec
 
 
 `
-
-exports['e2e plugin run events / fails if experimentalRunEvents is not enabled'] = `
-The following validation error was thrown by your plugins file (\`/foo/bar/.projects/plugin-run-events/cypress/plugins/index.js\`).
-
- Error: The \`before:spec\` event requires the experimentalRunEvents flag to be enabled.
-
-To enable it, set \`"experimentalRunEvents": true\` in your cypress.json
-      [stack trace lines]
-`

packages/server/lib/experiments.ts

@@ -53,7 +53,7 @@ interface StringValues {
 const _summaries: StringValues = {
   experimentalComponentTesting: 'Framework-specific component testing, uses `componentFolder` to load component specs.',
   experimentalFetchPolyfill: 'Polyfills `window.fetch` to enable Network spying and stubbing.',
-  experimentalRunEvents: 'Allows listening to the `before:spec` event in the plugins file.',
+  experimentalRunEvents: 'Allows listening to the `before:spec` and `after:spec` events in the plugins file.',
   experimentalSourceRewriting: 'Enables AST-based JS/HTML rewriting. This may fix issues caused by the existing regex-based JS/HTML replacement algorithm.',
 }
 

packages/server/lib/modes/run.js

@@ -1080,7 +1080,7 @@ module.exports = {
   },
 
   waitForTestsToFinishRunning (options = {}) {
-    const { project, screenshots, startedVideoCapture, endVideoCapture, videoName, compressedVideoName, videoCompression, videoUploadOnPasses, exit, spec, estimated, quiet } = options
+    const { project, screenshots, startedVideoCapture, endVideoCapture, videoName, compressedVideoName, videoCompression, videoUploadOnPasses, exit, spec, estimated, quiet, config } = options
 
     // https://github.com/cypress-io/cypress/issues/2370
     // delay 1 second if we're recording a video to give
@@ -1090,8 +1090,8 @@ module.exports = {
 
     return this.listenForProjectEnd(project, exit)
     .delay(delay)
-    .then(async (obj) => {
-      _.defaults(obj, {
+    .then(async (results) => {
+      _.defaults(results, {
         error: null,
         hooks: null,
         tests: null,
@@ -1101,27 +1101,23 @@ module.exports = {
       })
 
       if (startedVideoCapture) {
-        obj.video = videoName
+        results.video = videoName
       }
 
       if (screenshots) {
-        obj.screenshots = screenshots
+        results.screenshots = screenshots
       }
 
-      obj.spec = spec
-
-      const finish = () => {
-        return obj
-      }
+      results.spec = spec
 
       if (!quiet) {
-        this.displayResults(obj, estimated)
+        this.displayResults(results, estimated)
         if (screenshots && screenshots.length) {
           this.displayScreenshots(screenshots)
         }
       }
 
-      const { tests, stats } = obj
+      const { tests, stats } = results
 
       const attempts = _.flatMap(tests, (test) => test.attempts)
 
@@ -1137,7 +1133,7 @@ module.exports = {
       // or if we have any failures and have started the video
       const suv = Boolean(videoUploadOnPasses === true || (startedVideoCapture && hasFailingTests))
 
-      obj.shouldUploadVideo = suv
+      results.shouldUploadVideo = suv
 
       let videoCaptureFailed = false
 
@@ -1147,6 +1143,8 @@ module.exports = {
         .catch(warnVideoRecordingFailed)
       }
 
+      await runEvents.execute('after:spec', config, _.cloneDeep(spec), _.cloneDeep(results))
+
       // always close the browser now as opposed to letting
       // it exit naturally with the parent process due to
       // electron bug in windows
@@ -1154,7 +1152,7 @@ module.exports = {
       await openProject.closeBrowser()
 
       if (endVideoCapture && !videoCaptureFailed) {
-        const ffmpegChaptersConfig = videoCapture.generateFfmpegChaptersConfig(obj.tests)
+        const ffmpegChaptersConfig = videoCapture.generateFfmpegChaptersConfig(results.tests)
 
         await this.postProcessRecording(
           videoName,
@@ -1167,7 +1165,7 @@ module.exports = {
         .catch(warnVideoRecordingFailed)
       }
 
-      return finish()
+      return results
     })
   },
 
@@ -1322,7 +1320,7 @@ module.exports = {
 
     const screenshots = []
 
-    return runEvents.execute('before:spec', config, spec)
+    return runEvents.execute('before:spec', config, _.cloneDeep(spec))
     .then(() => {
     // we know we're done running headlessly
     // when the renderer has connected and
@@ -1340,6 +1338,7 @@ module.exports = {
       return Promise.props({
         results: this.waitForTestsToFinishRunning({
           spec,
+          config,
           project,
           estimated,
           screenshots,

packages/server/lib/plugins/child/run_plugins.js

@@ -111,6 +111,9 @@ const execute = (ipc, event, ids, args = []) => {
     'after:screenshot' () {
       util.wrapChildPromise(ipc, invoke, ids, args)
     },
+    'after:spec' () {
+      util.wrapChildPromise(ipc, invoke, ids, args)
+    },
     'before:browser:launch' () {
       browserLaunch.wrap(ipc, invoke, ids, args)
     },

packages/server/lib/plugins/child/validate_event.js

@@ -34,8 +34,13 @@ const eventValidators = {
   '_get:task:body': isFunction,
 }
 
+const runEvents = {
+  'after:spec': true,
+  'before:spec': true,
+}
+
 const validateEvent = (event, handler, config) => {
-  if (event === 'before:spec') {
+  if (runEvents[event]) {
     return isValidRunEvent(event, handler, config)
   }
 

packages/server/lib/plugins/run_events.js

@@ -4,7 +4,7 @@ const errors = require('../errors')
 const plugins = require('../plugins')
 
 module.exports = {
-  execute: Promise.method((eventName, config, ...args) => {
+  execute: Promise.method((eventName, config = {}, ...args) => {
     if (!config.experimentalRunEvents) return
 
     if (!plugins.has(eventName)) return

packages/server/test/e2e/4_plugin_run_events_spec.ts

@@ -4,7 +4,7 @@ import Fixtures from '../support/helpers/fixtures'
 describe('e2e plugin run events', () => {
   e2e.setup()
 
-  e2e.it('sends server events', {
+  e2e.it('sends events', {
     browser: 'electron',
     project: Fixtures.projectPath('plugin-run-events'),
     spec: '*',
@@ -26,7 +26,7 @@ describe('e2e plugin run events', () => {
     },
   })
 
-  e2e.it('fails run if server event handler throws', {
+  e2e.it('fails run if event handler throws', {
     browser: 'electron',
     project: Fixtures.projectPath('plugin-run-event-throws'),
     spec: '*',

packages/server/test/support/fixtures/projects/plugin-run-events/cypress/plugins/index.js

@@ -9,4 +9,15 @@ module.exports = (on) => {
       return console.log('before:spec is awaited')
     })
   })
+
+  on('after:spec', (spec, results) => {
+    const { stats } = results
+    const { tests, passes, failures } = stats
+
+    console.log('spec:end:', spec.relative, { tests, passes, failures })
+
+    return Promise.delay(10).then(() => {
+      return console.log('after:spec is awaited')
+    })
+  })
 }

packages/server/test/unit/plugins/child/validate_event_spec.js

@@ -68,26 +68,33 @@ The following are valid events:
   })
 
   describe('run events', () => {
-    it('returns error when before:spec event is registered without experimentalRunEvents flag enabled', () => {
-      const { isValid, error } = validateEvent('before:spec', {}, { experimentalRunEvents: false })
+    const runEvents = [
+      'before:spec',
+      'after:spec',
+    ]
 
-      expect(isValid).to.be.false
-      expect(error.message).to.equal(`The \`before:spec\` event requires the experimentalRunEvents flag to be enabled.
+    _.each(runEvents, (event) => {
+      it(`returns error when ${event} event is registed without experimentalRunEvents flag enabled`, () => {
+        const { isValid, error } = validateEvent(event, {}, { experimentalRunEvents: false })
+
+        expect(isValid).to.be.false
+        expect(error.message).to.equal(`The \`${event}\` event requires the experimentalRunEvents flag to be enabled.
 
 To enable it, set \`"experimentalRunEvents": true\` in your cypress.json`)
-    })
+      })
 
-    it('returns error when event handler of before:spec is not a function', () => {
-      const { isValid, error } = validateEvent('before:spec', 'invalid type', { experimentalRunEvents: true })
+      it(`returns error when event handler of ${event} is not a function`, () => {
+        const { isValid, error } = validateEvent(event, 'invalid type', { experimentalRunEvents: true })
 
-      expect(isValid).to.be.false
-      expect(error.message).to.equal(`The handler for the event \`before:spec\` must be a function`)
-    })
+        expect(isValid).to.be.false
+        expect(error.message).to.equal(`The handler for the event \`${event}\` must be a function`)
+      })
 
-    it('returns success when event handler of before:spec is a function', () => {
-      const { isValid } = validateEvent('before:spec', () => {}, { experimentalRunEvents: true })
+      it(`returns success when event handler of ${event} is a function`, () => {
+        const { isValid } = validateEvent(event, () => {}, { experimentalRunEvents: true })
 
-      expect(isValid).to.be.true
+        expect(isValid).to.be.true
+      })
     })
   })
 })