feat: Fire run events in interactive mode (#15787)

cli/schema/cypress.schema.json

@@ -242,6 +242,11 @@
           "default": "bundled",
           "description": "If set to 'system', Cypress will try to find a Node.js executable on your path to use when executing your plugins. Otherwise, Cypress will use the Node version bundled with Cypress."
         },
+        "experimentalInteractiveRunEvents": {
+          "type": "boolean",
+          "default": false,
+          "description": "Allows listening to the `before:run`, `after:run`, `before:spec`, and `after:spec` events in the plugins file during interactive mode."
+        },
         "experimentalSourceRewriting": {
           "type": "boolean",
           "default": false,

cli/types/cypress.d.ts

@@ -2615,8 +2615,12 @@ declare namespace Cypress {
      */
     firefoxGcInterval: Nullable<number | { runMode: Nullable<number>, openMode: Nullable<number> }>
     /**
-     * Enables AST-based JS/HTML rewriting. This may fix issues caused by the existing regex-based JS/HTML replacement
-     * algorithm.
+     * Allows listening to the `before:run`, `after:run`, `before:spec`, and `after:spec` events in the plugins file during interactive mode.
+     * @default false
+     */
+    experimentalInteractiveRunEvents: boolean
+    /**
+     * Generate and save commands directly to your test suite by interacting with your app as an end user would.
      * @default false
      */
     experimentalSourceRewriting: boolean
@@ -5159,22 +5163,22 @@ declare namespace Cypress {
   }
 
   interface BeforeRunDetails {
-    browser: Browser
+    browser?: Browser
     config: ConfigOptions
     cypressVersion: string
     group?: string
-    parallel: boolean
+    parallel?: boolean
     runUrl?: string
-    specs: Spec[]
-    specPattern: string[]
+    specs?: Spec[]
+    specPattern?: string[]
     system: SystemDetails
     tag?: string
   }
 
   interface DevServerOptions {
     specs: Spec[]
-    config: ResolvedConfigOptions & RuntimeConfigOptions,
-    devServerEvents: NodeJS.EventEmitter,
+    config: ResolvedConfigOptions & RuntimeConfigOptions
+    devServerEvents: NodeJS.EventEmitter
   }
 
   interface ResolvedDevServerConfig {

packages/desktop-gui/src/projects/projects-api.js

@@ -102,7 +102,7 @@ const runSpec = (project, spec, browser, specFilter) => {
   .then(launchBrowser)
 }
 
-const closeBrowser = (project, spec) => {
+const onBrowserClose = (project, spec) => {
   if (!spec) {
     specsStore.setChosenSpec(null)
   }
@@ -112,6 +112,10 @@ const closeBrowser = (project, spec) => {
   }
 
   ipc.offLaunchBrowser()
+}
+
+const closeBrowser = (project, spec) => {
+  onBrowserClose(project, spec)
 
   return ipc.closeBrowser()
 }
@@ -127,10 +131,10 @@ const closeProject = (project) => {
   ipc.offOnProjectWarning()
   ipc.offOnConfigChanged()
 
-  return Promise.join(
-    closeBrowser(project),
+  return Promise.all([
+    onBrowserClose(project),
     ipc.closeProject(),
-  )
+  ])
 }
 
 const openProject = (project) => {

packages/server/lib/browsers/index.js

@@ -233,5 +233,4 @@ module.exports = {
       })
     })
   },
-
 }

packages/server/lib/config_options.ts

@@ -79,6 +79,11 @@ export const options = [
     defaultValue: false,
     validation: v.isBoolean,
     isExperimental: true,
+  }, {
+    name: 'experimentalInteractiveRunEvents',
+    defaultValue: false,
+    validation: v.isBoolean,
+    isExperimental: true,
   }, {
     name: 'experimentalSourceRewriting',
     defaultValue: false,

packages/server/lib/experiments.ts

@@ -52,6 +52,7 @@ interface StringValues {
 */
 const _summaries: StringValues = {
   experimentalFetchPolyfill: 'Polyfills `window.fetch` to enable Network spying and stubbing.',
+  experimentalInteractiveRunEvents: 'Allows listening to the `before:run`, `after:run`, `before:spec`, and `after:spec` events in the plugins file during interactive mode.',
   experimentalSourceRewriting: 'Enables AST-based JS/HTML rewriting. This may fix issues caused by the existing regex-based JS/HTML replacement algorithm.',
   experimentalStudio: 'Generate and save commands directly to your test suite by interacting with your app as an end user would.',
 }
@@ -68,6 +69,7 @@ const _summaries: StringValues = {
 */
 const _names: StringValues = {
   experimentalFetchPolyfill: 'Fetch polyfill',
+  experimentalInteractiveRunEvents: 'Interactive Mode Run Events',
   experimentalSourceRewriting: 'Improved source rewriting',
   experimentalStudio: 'Studio',
 }

packages/server/lib/open_project.js

@@ -9,6 +9,7 @@ const { ProjectE2E } = require('./project-e2e')
 const browsers = require('./browsers')
 const specsUtil = require('./util/specs')
 const preprocessor = require('./plugins/preprocessor')
+const runEvents = require('./plugins/run_events')
 
 const moduleFactory = () => {
   let openProject = null
@@ -123,6 +124,12 @@ const moduleFactory = () => {
             })
           }
 
+          const afterSpec = () => {
+            if (!openProject || cfg.isTextTerminal || !cfg.experimentalInteractiveRunEvents) return Promise.resolve()
+
+            return runEvents.execute('after:spec', cfg, spec)
+          }
+
           const { onBrowserClose } = options
 
           options.onBrowserClose = () => {
@@ -130,6 +137,11 @@ const moduleFactory = () => {
               preprocessor.removeFile(spec.absolute, cfg)
             }
 
+            afterSpec(cfg, spec)
+            .catch((err) => {
+              openProject.options.onError(err)
+            })
+
             if (onBrowserClose) {
               return onBrowserClose()
             }
@@ -144,7 +156,14 @@ const moduleFactory = () => {
               spec.relative,
             )
 
-            return browsers.open(browser, options, automation)
+            return Promise.try(() => {
+              if (!cfg.isTextTerminal && cfg.experimentalInteractiveRunEvents) {
+                return runEvents.execute('before:spec', cfg, spec)
+              }
+            })
+            .then(() => {
+              return browsers.open(browser, options, automation)
+            })
           }
 
           return relaunchBrowser()
@@ -216,8 +235,7 @@ const moduleFactory = () => {
         return get()
         .then(sendIfChanged)
         .catch(options.onError)
-      },
-      250, { leading: true })
+      }, 250, { leading: true })
 
       const createSpecsWatcher = (cfg) => {
         // TODO I keep repeating this to get the resolved value
@@ -287,10 +305,10 @@ const moduleFactory = () => {
     },
 
     closeOpenProjectAndBrowsers () {
-      return Promise.all([
-        this.closeBrowser(),
-        openProject ? openProject.close() : undefined,
-      ])
+      return this.closeBrowser()
+      .then(() => {
+        return openProject && openProject.close()
+      })
       .then(() => {
         reset()
 
@@ -335,6 +353,9 @@ const moduleFactory = () => {
       return openProject.open({ ...options, testingType: args.testingType })
       .return(this)
     },
+
+    // for testing purposes
+    __reset: reset,
   }
 }
 

packages/server/lib/plugins/index.js

@@ -132,7 +132,7 @@ const init = (config, options) => {
 
             // no argument is passed for cy.task()
             // This is necessary because undefined becomes null when it is sent through ipc.
-            if (args[1] === undefined) {
+            if (registration.event === 'task' && args[1] === undefined) {
               args[1] = {
                 __cypress_task_no_argument__: true,
               }

packages/server/lib/project-base.ts

@@ -8,7 +8,9 @@ import la from 'lazy-ass'
 import _ from 'lodash'
 import path from 'path'
 import R from 'ramda'
+
 import commitInfo from '@cypress/commit-info'
+import pkg from '@packages/root'
 import { RunnablesStore } from '@packages/reporter'
 import { ServerCt } from '@packages/server-ct'
 import api from './api'
@@ -19,9 +21,11 @@ import cwd from './cwd'
 import errors from './errors'
 import logger from './logger'
 import Reporter from './reporter'
+import runEvents from './plugins/run_events'
 import savedState from './saved_state'
 import scaffold from './scaffold'
 import { ServerE2E } from './server-e2e'
+import system from './util/system'
 import user from './user'
 import { ensureProp } from './util/class-helpers'
 import { escapeFilenameInUrl } from './util/escape_filename'
@@ -40,8 +44,6 @@ interface OpenOptions {
   onAfterOpen: (cfg: any) => Bluebird<any>
 }
 
-// type ProjectOptions = Record<string, any>
-
 export type Cfg = Record<string, any>
 
 const localCwd = cwd()
@@ -183,6 +185,20 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
           this.watchPluginsFile(cfg, options),
         )
       })
+      .then(() => {
+        if (cfg.isTextTerminal || !cfg.experimentalInteractiveRunEvents) return
+
+        return system.info()
+        .then((sys) => {
+          const beforeRunDetails = {
+            config: cfg,
+            cypressVersion: pkg.version,
+            system: _.pick(sys, 'osName', 'osVersion'),
+          }
+
+          return runEvents.execute('before:run', cfg, beforeRunDetails)
+        })
+      })
     })
     .return(this)
   }
@@ -227,6 +243,13 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
     )
     .then(() => {
       process.chdir(localCwd)
+
+      return this.getConfig()
+    })
+    .then((config) => {
+      if (config.isTextTerminal || !config.experimentalInteractiveRunEvents) return
+
+      return runEvents.execute('after:run', config)
     })
   }
 

packages/server/test/unit/config_spec.js

@@ -1329,6 +1329,7 @@ describe('lib/config', () => {
             env: {},
             execTimeout: { value: 60000, from: 'default' },
             experimentalFetchPolyfill: { value: false, from: 'default' },
+            experimentalInteractiveRunEvents: { value: false, from: 'default' },
             experimentalSourceRewriting: { value: false, from: 'default' },
             experimentalStudio: { value: false, from: 'default' },
             fileServerFolder: { value: '', from: 'default' },
@@ -1411,6 +1412,7 @@ describe('lib/config', () => {
             e2e: { from: 'default', value: {} },
             execTimeout: { value: 60000, from: 'default' },
             experimentalFetchPolyfill: { value: false, from: 'default' },
+            experimentalInteractiveRunEvents: { value: false, from: 'default' },
             experimentalSourceRewriting: { value: false, from: 'default' },
             experimentalStudio: { value: false, from: 'default' },
             env: {

packages/server/test/unit/open_project_spec.js

@@ -5,6 +5,7 @@ const browsers = require(`${root}lib/browsers`)
 const { ProjectE2E } = require(`${root}lib/project-e2e`)
 const openProject = require(`${root}lib/open_project`)
 const preprocessor = require(`${root}lib/plugins/preprocessor`)
+const runEvents = require(`${root}lib/plugins/run_events`)
 
 describe('lib/open_project', () => {
   beforeEach(function () {
@@ -91,6 +92,116 @@ describe('lib/open_project', () => {
         expect(this.browser.isHeadless).to.be.false
       })
     })
+
+    describe('spec events', function () {
+      beforeEach(function () {
+        sinon.stub(runEvents, 'execute').resolves()
+      })
+
+      it('executes before:spec if in interactive mode', function () {
+        this.config.experimentalInteractiveRunEvents = true
+        this.config.isTextTerminal = false
+
+        return openProject.launch(this.browser, this.spec).then(() => {
+          expect(runEvents.execute).to.be.calledWith('before:spec', this.config, this.spec)
+        })
+      })
+
+      it('does not execute before:spec if not in interactive mode', function () {
+        this.config.experimentalInteractiveRunEvents = true
+        this.config.isTextTerminal = true
+
+        return openProject.launch(this.browser, this.spec).then(() => {
+          expect(runEvents.execute).not.to.be.calledWith('before:spec')
+        })
+      })
+
+      it('does not execute before:spec if experimental flag is not enabled', function () {
+        this.config.experimentalInteractiveRunEvents = false
+        this.config.isTextTerminal = false
+
+        return openProject.launch(this.browser, this.spec).then(() => {
+          expect(runEvents.execute).not.to.be.calledWith('before:spec')
+        })
+      })
+
+      it('executes after:spec on browser close if in interactive mode', function () {
+        this.config.experimentalInteractiveRunEvents = true
+        this.config.isTextTerminal = false
+
+        return openProject.launch(this.browser, this.spec)
+        .then(() => {
+          browsers.open.lastCall.args[1].onBrowserClose()
+        })
+        .delay(100) // needs a tick or two for the event to fire
+        .then(() => {
+          expect(runEvents.execute).to.be.calledWith('after:spec', this.config, this.spec)
+        })
+      })
+
+      it('does not execute after:spec on browser close if not in interactive mode', function () {
+        this.config.experimentalInteractiveRunEvents = true
+        this.config.isTextTerminal = true
+
+        return openProject.launch(this.browser, this.spec)
+        .then(() => {
+          browsers.open.lastCall.args[1].onBrowserClose()
+        })
+        .delay(10) // wait a few ticks to make sure it hasn't fired
+        .then(() => {
+          expect(runEvents.execute).not.to.be.calledWith('after:spec')
+        })
+      })
+
+      it('does not execute after:spec on browser close if experimental flag is not enabled', function () {
+        this.config.experimentalInteractiveRunEvents = false
+        this.config.isTextTerminal = false
+
+        return openProject.launch(this.browser, this.spec)
+        .then(() => {
+          browsers.open.lastCall.args[1].onBrowserClose()
+        })
+        .delay(10) // wait a few ticks to make sure it hasn't fired
+        .then(() => {
+          expect(runEvents.execute).not.to.be.calledWith('after:spec')
+        })
+      })
+
+      it('does not execute after:spec on browser close if the project is no longer open', function () {
+        this.config.experimentalInteractiveRunEvents = true
+        this.config.isTextTerminal = false
+
+        return openProject.launch(this.browser, this.spec)
+        .then(() => {
+          openProject.__reset()
+          browsers.open.lastCall.args[1].onBrowserClose()
+        })
+        .delay(10) // wait a few ticks to make sure it hasn't fired
+        .then(() => {
+          expect(runEvents.execute).not.to.be.calledWith('after:spec')
+        })
+      })
+
+      it('sends after:spec errors through onError option', function () {
+        const err = new Error('thrown from after:spec handler')
+        const onError = sinon.stub()
+
+        this.config.experimentalInteractiveRunEvents = true
+        this.config.isTextTerminal = false
+        runEvents.execute.withArgs('after:spec').rejects(err)
+        openProject.getProject().options.onError = onError
+
+        return openProject.launch(this.browser, this.spec)
+        .then(() => {
+          browsers.open.lastCall.args[1].onBrowserClose()
+        })
+        .delay(100) // needs a tick or two for the event to fire
+        .then(() => {
+          expect(runEvents.execute).to.be.calledWith('after:spec')
+          expect(onError).to.be.calledWith(err)
+        })
+      })
+    })
   })
 
   context('#getSpecChanges', () => {

packages/server/test/unit/project_spec.js

@@ -3,6 +3,7 @@ require('../spec_helper')
 const mockedEnv = require('mocked-env')
 const path = require('path')
 const commitInfo = require('@cypress/commit-info')
+const pkg = require('@packages/root')
 const Fixtures = require('../support/helpers/fixtures')
 const api = require(`${root}lib/api`)
 const user = require(`${root}lib/user`)
@@ -11,16 +12,18 @@ const config = require(`${root}lib/config`)
 const scaffold = require(`${root}lib/scaffold`)
 const { ServerE2E } = require(`${root}lib/server-e2e`)
 const { ProjectE2E } = require(`${root}lib/project-e2e`)
-const Automation = require(`${root}lib/automation`)
+const { Automation } = require(`${root}lib/automation`)
 const savedState = require(`${root}lib/saved_state`)
 const preprocessor = require(`${root}lib/plugins/preprocessor`)
 const plugins = require(`${root}lib/plugins`)
+const runEvents = require(`${root}lib/plugins/run_events`)
+const system = require(`${root}lib/util/system`)
 const { fs } = require(`${root}lib/util/fs`)
 const settings = require(`${root}lib/util/settings`)
 const Watchers = require(`${root}lib/watchers`)
 const { SocketE2E } = require(`${root}lib/socket-e2e`)
 
-xdescribe('lib/project-e2e', () => {
+describe('lib/project-e2e', () => {
   beforeEach(function () {
     Fixtures.scaffold()
 
@@ -28,6 +31,9 @@ xdescribe('lib/project-e2e', () => {
     this.idsPath = Fixtures.projectPath('ids')
     this.pristinePath = Fixtures.projectPath('pristine')
 
+    sinon.stub(scaffold, 'isNewProject').resolves(false)
+    sinon.stub(runEvents, 'execute').resolves()
+
     return settings.read(this.todosPath).then((obj = {}) => {
       ({ projectId: this.projectId } = obj)
 
@@ -35,6 +41,8 @@ xdescribe('lib/project-e2e', () => {
       .then((config1) => {
         this.config = config1
         this.project = new ProjectE2E(this.todosPath)
+        this.project._server = { close () {} }
+        this.project._cfg = config1
       })
     })
   })
@@ -47,26 +55,24 @@ xdescribe('lib/project-e2e', () => {
     }
   })
 
-  it('requires a projectRoot', () => {
+  it('requires a projectRoot', function () {
     const fn = () => new ProjectE2E()
 
     expect(fn).to.throw('Instantiating lib/project requires a projectRoot!')
   })
 
-  it('always resolves the projectRoot to be absolute', () => {
+  it('always resolves the projectRoot to be absolute', function () {
     const p = new ProjectE2E('../foo/bar')
 
     expect(p.projectRoot).not.to.eq('../foo/bar')
-
     expect(p.projectRoot).to.eq(path.resolve('../foo/bar'))
   })
 
-  context('#saveState', () => {
+  context('#saveState', function () {
     beforeEach(function () {
       const integrationFolder = 'the/save/state/test'
 
       sinon.stub(config, 'get').withArgs(this.todosPath).resolves({ integrationFolder })
-      sinon.stub(this.project, 'determineIsNewProject').withArgs(integrationFolder).resolves(false)
       this.project.cfg = { integrationFolder }
 
       return savedState.create(this.project.projectRoot)
@@ -108,8 +114,9 @@ xdescribe('lib/project-e2e', () => {
     const integrationFolder = 'foo/bar/baz'
 
     beforeEach(function () {
+      this.project._cfg = undefined
+
       sinon.stub(config, 'get').withArgs(this.todosPath, { foo: 'bar' }).resolves({ baz: 'quux', integrationFolder })
-      sinon.stub(this.project, 'determineIsNewProject').withArgs(integrationFolder).resolves(false)
     })
 
     it('calls config.get with projectRoot + options + saved state', function () {
@@ -117,7 +124,7 @@ xdescribe('lib/project-e2e', () => {
       .then((state) => {
         sinon.stub(state, 'get').resolves({ reporterWidth: 225 })
 
-        this.project.getConfig({ foo: 'bar' })
+        return this.project.getConfig({ foo: 'bar' })
         .then((cfg) => {
           expect(cfg).to.deep.eq({
             integrationFolder,
@@ -127,12 +134,14 @@ xdescribe('lib/project-e2e', () => {
               reporterWidth: 225,
             },
           })
+
+          this.project._cfg = cfg
         })
       })
     })
 
     it('resolves if cfg is already set', function () {
-      this.project.cfg = {
+      this.project._cfg = {
         integrationFolder,
         foo: 'bar',
       }
@@ -151,7 +160,7 @@ xdescribe('lib/project-e2e', () => {
       .then((state) => {
         sinon.stub(state, 'get').resolves({ showedOnBoardingModal: true })
 
-        this.project.getConfig({ foo: 'bar' })
+        return this.project.getConfig({ foo: 'bar' })
         .then((cfg) => {
           expect(cfg).to.deep.eq({
             integrationFolder,
@@ -161,6 +170,8 @@ xdescribe('lib/project-e2e', () => {
               showedOnBoardingModal: true,
             },
           })
+
+          this.project._cfg = cfg
         })
       })
     })
@@ -203,13 +214,13 @@ xdescribe('lib/project-e2e', () => {
     })
 
     it('calls #scaffold with server config promise', function () {
-      return this.project.open().then(() => {
+      return this.project.open({}).then(() => {
         expect(this.project.scaffold).to.be.calledWith(this.config)
       })
     })
 
     it('calls #checkSupportFile with server config when scaffolding is finished', function () {
-      return this.project.open().then(() => {
+      return this.project.open({}).then(() => {
         expect(this.project.checkSupportFile).to.be.calledWith(this.config)
       })
     })
@@ -292,7 +303,7 @@ xdescribe('lib/project-e2e', () => {
         chromeWebSecurity: false,
       })
 
-      return this.project.open()
+      return this.project.open({})
       .then(() => this.project.getConfig())
       .then((config) => {
         expect(config.chromeWebSecurity).eq(false)
@@ -315,21 +326,67 @@ This option will not have an effect in Some-other-name. Tests that rely on web s
         expect(config).ok
       })
     })
+
+    it('executes before:run if in interactive mode', function () {
+      const sysInfo = {
+        osName: 'darwin',
+        osVersion: '1.2.3',
+      }
+
+      sinon.stub(system, 'info').resolves(sysInfo)
+      this.config.experimentalInteractiveRunEvents = true
+      this.config.isTextTerminal = false
+
+      return this.project.open({})
+      .then(() => {
+        expect(runEvents.execute).to.be.calledWith('before:run', this.config, {
+          config: this.config,
+          cypressVersion: pkg.version,
+          system: sysInfo,
+        })
+      })
+    })
+
+    it('does not get system info or execute before:run if not in interactive mode', function () {
+      sinon.stub(system, 'info')
+      this.config.experimentalInteractiveRunEvents = true
+      this.config.isTextTerminal = true
+
+      return this.project.open({})
+      .then(() => {
+        expect(system.info).not.to.be.called
+        expect(runEvents.execute).not.to.be.calledWith('before:run')
+      })
+    })
+
+    it('does not get system info or execute before:run if experimental flag is not enabled', function () {
+      sinon.stub(system, 'info')
+      this.config.experimentalInteractiveRunEvents = false
+      this.config.isTextTerminal = false
+
+      return this.project.open({})
+      .then(() => {
+        expect(system.info).not.to.be.called
+        expect(runEvents.execute).not.to.be.calledWith('before:run')
+      })
+    })
   })
 
   context('#close', () => {
     beforeEach(function () {
       this.project = new ProjectE2E('/_test-output/path/to/project-e2e')
 
+      this.project._server = { close () {} }
+
       sinon.stub(this.project, 'getConfig').resolves(this.config)
       sinon.stub(user, 'ensureAuthToken').resolves('auth-token-123')
     })
 
     it('closes server', function () {
-      this.project.server = sinon.stub({ close () {} })
+      this.project._server = sinon.stub({ close () {} })
 
       return this.project.close().then(() => {
-        expect(this.project.server.close).to.be.calledOnce
+        expect(this.project._server.close).to.be.calledOnce
       })
     })
 
@@ -344,19 +401,49 @@ This option will not have an effect in Some-other-name. Tests that rely on web s
     it('can close when server + watchers arent open', function () {
       return this.project.close()
     })
+
+    it('executes after:run if in interactive mode', function () {
+      this.config.experimentalInteractiveRunEvents = true
+      this.config.isTextTerminal = false
+
+      return this.project.close()
+      .then(() => {
+        expect(runEvents.execute).to.be.calledWith('after:run', this.config)
+      })
+    })
+
+    it('does not execute after:run if not in interactive mode', function () {
+      this.config.experimentalInteractiveRunEvents = true
+      this.config.isTextTerminal = true
+
+      return this.project.close()
+      .then(() => {
+        expect(runEvents.execute).not.to.be.calledWith('after:run')
+      })
+    })
+
+    it('does not execute after:run if experimental flag is not enabled', function () {
+      this.config.experimentalInteractiveRunEvents = false
+      this.config.isTextTerminal = false
+
+      return this.project.close()
+      .then(() => {
+        expect(runEvents.execute).not.to.be.calledWith('after:run')
+      })
+    })
   })
 
   context('#reset', () => {
     beforeEach(function () {
       this.project = new ProjectE2E(this.pristinePath)
-      this.project.automation = { reset: sinon.stub() }
-      this.project.server = { reset: sinon.stub() }
+      this.project._automation = { reset: sinon.stub() }
+      this.project._server = { close () {}, reset: sinon.stub() }
     })
 
     it('resets server + automation', function () {
       return this.project.reset()
       .then(() => {
-        expect(this.project.automation.reset).to.be.calledOnce
+        expect(this.project._automation.reset).to.be.calledOnce
 
         expect(this.project.server.reset).to.be.calledOnce
       })
@@ -469,7 +556,7 @@ This option will not have an effect in Some-other-name. Tests that rely on web s
   context('#watchSettings', () => {
     beforeEach(function () {
       this.project = new ProjectE2E('/_test-output/path/to/project-e2e')
-      this.project.server = { startWebsockets () {} }
+      this.project._server = { close () {}, startWebsockets () {} }
       sinon.stub(settings, 'pathToConfigFile').returns('/path/to/cypress.json')
       sinon.stub(settings, 'pathToCypressEnvJson').returns('/path/to/cypress.env.json')
       this.watch = sinon.stub(this.project.watchers, 'watch')
@@ -629,9 +716,8 @@ This option will not have an effect in Some-other-name. Tests that rely on web s
     beforeEach(function () {
       this.project = new ProjectE2E('/_test-output/path/to/project-e2e')
       this.project.watchers = {}
-      this.project.server = sinon.stub({ startWebsockets () {} })
+      this.project._server = { close () {}, startWebsockets: sinon.stub() }
       sinon.stub(this.project, 'watchSettings')
-      sinon.stub(Automation, 'create').returns('automation')
     })
 
     it('calls server.startWebsockets with automation + config', function () {
@@ -639,7 +725,10 @@ This option will not have an effect in Some-other-name. Tests that rely on web s
 
       this.project.watchSettingsAndStartWebsockets({}, c)
 
-      expect(this.project.server.startWebsockets).to.be.calledWith('automation', c)
+      const args = this.project.server.startWebsockets.lastCall.args
+
+      expect(args[0]).to.be.an.instanceof(Automation)
+      expect(args[1]).to.equal(c)
     })
 
     it('passes onReloadBrowser callback', function () {
@@ -728,6 +817,13 @@ This option will not have an effect in Some-other-name. Tests that rely on web s
     beforeEach(function () {
       this.project2 = new ProjectE2E(this.idsPath)
 
+      this.project._cfg = {
+        browserUrl: 'http://localhost:8888/__/',
+        integrationFolder: path.join(this.todosPath, 'tests'),
+        componentFolder: path.join(this.todosPath, 'tests'),
+        projectRoot: this.todosPath,
+      }
+
       return settings.write(this.idsPath, { port: 2020 })
     })