chore: add e2e test for new test isolation events (#27222)

packages/app/cypress/e2e/reporter_header.cy.ts

@@ -13,7 +13,7 @@ describe('Reporter Header', () => {
       cy.get('body').type('f')
 
       cy.get('[data-selected-spec="true"]').should('contain', 'dom-content').should('have.length', '1')
-      cy.get('[data-selected-spec="false"]').should('have.length', '28')
+      cy.get('[data-selected-spec="false"]').should('have.length', '29')
     })
 
     // TODO: Reenable as part of https://github.com/cypress-io/cypress/issues/23902

packages/app/cypress/e2e/specs_list_e2e.cy.ts

@@ -152,7 +152,7 @@ describe('App: Spec List (E2E)', () => {
 
       it('displays only matching spec', function () {
         cy.get('button')
-        .contains('24 matches')
+        .contains('25 matches')
         .should('not.contain.text', 'of')
 
         clearSearchAndType('content')
@@ -160,13 +160,13 @@ describe('App: Spec List (E2E)', () => {
         .should('have.length', 3)
         .and('contain', 'dom-content.spec.js')
 
-        cy.get('button').contains('3 of 24 matches')
+        cy.get('button').contains('3 of 25 matches')
 
         cy.findByLabelText('Search specs').clear().type('asdf')
         cy.findAllByTestId('spec-item')
         .should('have.length', 0)
 
-        cy.get('button').contains('0 of 24 matches')
+        cy.get('button').contains('0 of 25 matches')
       })
 
       it('only shows matching folders', () => {
@@ -217,7 +217,7 @@ describe('App: Spec List (E2E)', () => {
         cy.findByLabelText('Search specs')
         .should('have.value', '')
 
-        cy.get('button').contains('24 matches')
+        cy.get('button').contains('25 matches')
       })
 
       it('clears the filter if the user presses ESC key', function () {
@@ -226,7 +226,7 @@ describe('App: Spec List (E2E)', () => {
 
         cy.get('@searchField').should('have.value', '')
 
-        cy.get('button').contains('24 matches')
+        cy.get('button').contains('25 matches')
       })
 
       it('shows empty message if no results', function () {
@@ -242,7 +242,7 @@ describe('App: Spec List (E2E)', () => {
         cy.findByText('Clear search').click()
         cy.focused().should('have.id', 'spec-filter')
 
-        cy.get('button').contains('24 matches')
+        cy.get('button').contains('25 matches')
       })
 
       it('normalizes directory path separators for Windows', function () {

packages/app/cypress/e2e/subscriptions/specChange-subscription.cy.ts

@@ -75,6 +75,7 @@ describe('specChange subscription', () => {
           getPathForPlatform('cypress/e2e/accounts/accounts_new.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin_users_list.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin.user/foo_list.spec.js'),
+          getPathForPlatform('cypress/e2e/test-isolation.spec.js'),
           getPathForPlatform('cypress/e2e/z001.spec.js'),
           getPathForPlatform('cypress/e2e/z002.spec.js'),
           getPathForPlatform('cypress/e2e/z003.spec.js'),
@@ -118,6 +119,7 @@ describe('specChange subscription', () => {
           getPathForPlatform('cypress/e2e/accounts/accounts_new.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin_users_list.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin.user/foo_list.spec.js'),
+          getPathForPlatform('cypress/e2e/test-isolation.spec.js'),
           getPathForPlatform('cypress/e2e/z001.spec.js'),
           getPathForPlatform('cypress/e2e/z002.spec.js'),
           getPathForPlatform('cypress/e2e/z003.spec.js'),
@@ -188,7 +190,7 @@ e2e: {
 
       cy.get('body').type('f')
       cy.get('[data-cy="spec-file-item"]')
-      .should('have.length', 24)
+      .should('have.length', 25)
       .should('contain', 'blank-contents.spec.js')
       .should('contain', 'dom-container.spec.js')
       .should('contain', 'dom-content.spec.js')
@@ -199,7 +201,7 @@ e2e: {
       }, { path: getPathForPlatform('cypress/e2e/new-file.spec.js') })
 
       cy.get('[data-cy="spec-file-item"]')
-      .should('have.length', 25)
+      .should('have.length', 26)
       .should('contain', 'blank-contents.spec.js')
       .should('contain', 'dom-container.spec.js')
       .should('contain', 'dom-content.spec.js')
@@ -214,7 +216,7 @@ e2e: {
 
       cy.get('body').type('f')
       cy.get('[data-cy="spec-file-item"]')
-      .should('have.length', 24)
+      .should('have.length', 25)
       .should('contain', 'blank-contents.spec.js')
       .should('contain', 'dom-container.spec.js')
       .should('contain', 'dom-content.spec.js')
@@ -225,7 +227,7 @@ e2e: {
       }, { path: getPathForPlatform('cypress/e2e/dom-list.spec.js') })
 
       cy.get('[data-cy="spec-file-item"]')
-      .should('have.length', 23)
+      .should('have.length', 24)
       .should('contain', 'blank-contents.spec.js')
       .should('contain', 'dom-container.spec.js')
       .should('contain', 'dom-content.spec.js')
@@ -255,6 +257,7 @@ e2e: {
           getPathForPlatform('cypress/e2e/accounts/accounts_new.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin_users_list.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin.user/foo_list.spec.js'),
+          getPathForPlatform('cypress/e2e/test-isolation.spec.js'),
           getPathForPlatform('cypress/e2e/z001.spec.js'),
           getPathForPlatform('cypress/e2e/z002.spec.js'),
           getPathForPlatform('cypress/e2e/z003.spec.js'),
@@ -286,7 +289,7 @@ e2e: {
 
       cy.get('body').type('f')
       cy.get('[data-cy="spec-file-item"]')
-      .should('have.length', 24)
+      .should('have.length', 25)
       .should('contain', 'blank-contents.spec.js')
       .should('contain', 'dom-container.spec.js')
       .should('contain', 'dom-content.spec.js')
@@ -329,14 +332,14 @@ e2e: {
       cy.get('[data-cy="spec-pattern"]').contains('cypress/e2e/**/*.spec.{js,ts}')
 
       cy.get('[data-cy="file-match-indicator"]')
-      .should('contain', '24 matches')
+      .should('contain', '25 matches')
 
       cy.withCtx(async (ctx, o) => {
         await ctx.actions.file.writeFileInProject(o.path, '')
       }, { path: getPathForPlatform('cypress/e2e/new-file.spec.js') })
 
       cy.get('[data-cy="file-match-indicator"]')
-      .should('contain', '25 matches')
+      .should('contain', '26 matches')
     })
 
     it('responds to specChange event for a removed file', () => {
@@ -346,14 +349,14 @@ e2e: {
       cy.get('[data-cy="spec-pattern"]').contains('cypress/e2e/**/*.spec.{js,ts}')
 
       cy.get('[data-cy="file-match-indicator"]')
-      .should('contain', '24 matches')
+      .should('contain', '25 matches')
 
       cy.withCtx(async (ctx, o) => {
         await ctx.actions.file.removeFileInProject(o.path)
       }, { path: getPathForPlatform('cypress/e2e/dom-list.spec.js') })
 
       cy.get('[data-cy="file-match-indicator"]')
-      .should('contain', '23 matches')
+      .should('contain', '24 matches')
     })
 
     it('handles removing the last file', () => {
@@ -380,6 +383,7 @@ e2e: {
           getPathForPlatform('cypress/e2e/accounts/accounts_new.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin_users_list.spec.js'),
           getPathForPlatform('cypress/e2e/admin_users/admin.user/foo_list.spec.js'),
+          getPathForPlatform('cypress/e2e/test-isolation.spec.js'),
           getPathForPlatform('cypress/e2e/z001.spec.js'),
           getPathForPlatform('cypress/e2e/z002.spec.js'),
           getPathForPlatform('cypress/e2e/z003.spec.js'),
@@ -410,7 +414,7 @@ e2e: {
       cy.get('[data-cy="spec-pattern"]').contains('cypress/e2e/**/*.spec.{js,ts}')
 
       cy.get('[data-cy="file-match-indicator"]')
-      .should('contain', '24 matches')
+      .should('contain', '25 matches')
 
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject('cypress.config.js',

packages/app/cypress/e2e/test-isolation.cy.ts

@@ -0,0 +1,12 @@
+describe('Test Isolation', () => {
+  it('fires events in the right order with the right arguments - open mode', () => {
+    cy.scaffoldProject('cypress-in-cypress')
+    cy.findBrowsers()
+    cy.openProject('cypress-in-cypress')
+    cy.startAppServer()
+
+    cy.visitApp(`/specs/runner?file=cypress/e2e/test-isolation.spec.js`)
+
+    cy.get('.passed > .num').should('contain', 3)
+  })
+})

packages/driver/src/cypress/runner.ts

@@ -492,7 +492,9 @@ const overrideRunnerHook = (Cypress, _runner, getTestById, getTest, setTest, get
 
         // If we're not in open mode or we're in open mode and not the last test we reset state.
         // The last test will needs to stay so that the user can see what the end result of the AUT was.
-        if (!Cypress.config('isInteractive') || !lastTestThatWillRunInSuite(test, getAllSiblingTests(topSuite, getTestById))) {
+        const isRunMode = !Cypress.config('isInteractive')
+
+        if (isRunMode || !lastTestThatWillRunInSuite(test, getAllSiblingTests(topSuite, getTestById))) {
           cy.state('duringUserTestExecution', false)
           Cypress.primaryOriginCommunicator.toAllSpecBridges('sync:state', { 'duringUserTestExecution': false })
           // Remove window:load and window:before:load listeners so that navigating to about:blank doesn't fire in user code.

system-tests/projects/cypress-in-cypress/cypress/e2e/test-isolation.spec.js

@@ -0,0 +1,86 @@
+const TEST_METADATA = {
+  'passes 1': {
+    start: 'about:blank',
+    firesTestBeforeAfterRunAsync: true,
+    end: 'about:blank',
+  },
+  'passes 2': {
+    start: 'about:blank',
+    firesTestBeforeAfterRunAsync: true,
+    end: 'about:blank',
+  },
+  'passes 3': {
+    start: 'about:blank',
+    firesTestBeforeAfterRunAsync: !Cypress.config('isInteractive'),
+    end: !Cypress.config('isInteractive') ? 'about:blank' : '/cypress/e2e/dom-content.html',
+  },
+}
+
+let cypressEventsHandled = 0
+
+const testBeforeRun = (Cypress, ...args) => {
+  expect(Cypress.state('window').location.href).to.eq(TEST_METADATA[args[1].title].start)
+  cypressEventsHandled += 1
+}
+
+const testBeforeRunAsync = (Cypress, ...args) => {
+  expect(Cypress.state('window').location.href).to.eq(TEST_METADATA[args[1].title].start)
+  cypressEventsHandled += 1
+}
+
+const testBeforeAfterRunAsync = (Cypress, ...args) => {
+  expect(TEST_METADATA[args[1].title].firesTestBeforeAfterRunAsync).to.be.true
+  cypressEventsHandled += 1
+}
+
+const testAfterRun = (Cypress, ...args) => {
+  expect(Cypress.state('window').location.href.endsWith(TEST_METADATA[args[1].title].end)).to.equal(true)
+  cypressEventsHandled += 1
+}
+
+const testAfterRunAsync = (Cypress, ...args) => {
+  expect(Cypress.state('window').location.href.endsWith(TEST_METADATA[args[1].title].end)).to.equal(true)
+  cypressEventsHandled += 1
+}
+
+const cypressEvents = [
+  ['test:before:run', testBeforeRun],
+  ['test:before:run:async', testBeforeRunAsync],
+  ['test:before:after:run:async', testBeforeAfterRunAsync],
+  ['test:after:run', testAfterRun],
+  ['test:after:run:async', testAfterRunAsync],
+]
+
+cypressEvents.forEach(([event, handler]) => {
+  Cypress.prependListener(event, (...args) => {
+    handler(Cypress, ...args)
+  })
+})
+
+Cypress.on('test:after:run:async', async (test) => {
+  if (test.title === 'passes 1') {
+    expect(cypressEventsHandled).to.equal(5)
+  } else if (test.title === 'passes 2') {
+    expect(cypressEventsHandled).to.equal(10)
+  } else if (test.title === 'passes 3') {
+    expect(cypressEventsHandled).to.equal(Cypress.config('isInteractive') ? 14 : 15)
+  }
+})
+
+describe('test isolation', () => {
+  beforeEach(() => {
+    cy.visit('cypress/e2e/dom-content.html')
+  })
+
+  it('passes 1', () => {
+    expect(true).to.equal(true)
+  })
+
+  it('passes 2', () => {
+    expect(true).to.equal(true)
+  })
+
+  it('passes 3', () => {
+    expect(true).to.equal(true)
+  })
+})

system-tests/test/test_isolation_spec.ts

@@ -0,0 +1,13 @@
+import systemTests from '../lib/system-tests'
+
+describe('Test Isolation', () => {
+  systemTests.setup()
+
+  systemTests.it('fires events in the right order with the right arguments - run mode', {
+    project: 'cypress-in-cypress',
+    spec: 'test-isolation.spec.js',
+    expectedExitCode: 0,
+    timeout: 20000,
+    browser: 'chrome',
+  })
+})