mirror/cypress
1
0
Fork 0
mirror of https://github.com/cypress-io/cypress.git synced 2026-02-21 22:50:49 -06:00
Code Issues Packages Projects Releases Wiki Activity

test: fix flaky tests (#26505)

Browse Source 
* test(launchpad): fix flaky tests

Closes #23153

Closes #23154

---------

Co-authored-by: astone123 <adams@cypress.io>
This commit is contained in:
Jordan
2023-04-17 14:24:23 +00:00
committed by GitHub
parent 6209b91865
commit 067e4fa304
8 changed files with 52 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/workflows.yml
View File

@@ -69,7 +69,7 @@ windowsWorkflowFilters: &windows-workflow-filters
or:
- equal: [ develop, << pipeline.git.branch >> ]
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
- equal: [ 'lmiller/fixing-vite-windows', << pipeline.git.branch >> ]
- equal: [ 'windows-flake', << pipeline.git.branch >> ]
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
- matches:
pattern: /^release\/\d+\.\d+\.\d+$/

2
packages/app/cypress/e2e/runs.cy.ts
View File

@@ -323,7 +323,7 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
moveToRunsPage()
cy.findByText(defaultMessages.runs.connect.buttonProject).click()
cy.contains('button', defaultMessages.runs.connect.modal.selectProject.createProject).click()
cy.findByText(defaultMessages.runs.connectSuccessAlert.title).should('be.visible')
cy.findByText(defaultMessages.runs.connectSuccessAlert.title, { timeout: 10000 }).should('be.visible')
cy.withCtx(async (ctx) => {
const config = await ctx.project.getConfig()

46
packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts
View File

@@ -31,7 +31,7 @@ describe('Config files error handling', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
})
it('shows the upgrade screen if there is a legacy config file', () => {
@@ -70,7 +70,7 @@ describe('Config files error handling', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
})
it('handles deprecated config fields', () => {
@@ -92,7 +92,7 @@ describe('Config files error handling', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Choose a browser')
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
})
})
@@ -102,7 +102,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-root-sync-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})
@@ -111,7 +111,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-root-syntax-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})
@@ -120,7 +120,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-root-async-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', 'Unexpected Error')
cy.contains('h1', 'Unexpected Error', { timeout: 10000 })
expectStackToBe('open')
})
@@ -129,7 +129,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-function-sync-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})
@@ -138,7 +138,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-invalid-browser', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('closed')
})
@@ -147,7 +147,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('plugins-function-sync-error', ['--e2e'])
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
expectStackToBe('open')
})
@@ -156,7 +156,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-ts-syntax-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
cy.withCtx(async (ctx) => {
@@ -165,7 +165,7 @@ describe('Launchpad: Error System Tests', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
})
it(`clears the error correctly after first 'try again' attempt`, () => {
@@ -174,7 +174,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-ts-syntax-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
// Try again while the config is still invalid
cy.findByRole('button', { name: 'Try again' }).click()
@@ -182,7 +182,7 @@ describe('Launchpad: Error System Tests', () => {
cy.wait('@resetErrorsAndLoadConfig')
// Wait until config error is on screen again
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.withCtx(async (ctx) => {
await ctx.actions.file.writeFileInProject('cypress.config.ts', 'export default { e2e: { supportFile: false } }')
@@ -190,7 +190,7 @@ describe('Launchpad: Error System Tests', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Welcome to Cypress')
cy.contains('h1', 'Welcome to Cypress')
})
it('shows correct user file instead of node file', () => {
@@ -198,7 +198,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-import-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.js:3:23')
@@ -209,7 +209,7 @@ describe('Launchpad: Error System Tests', () => {
cy.openProject('config-with-ts-module-error')
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.ts:6:10')
@@ -223,7 +223,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.findByText('E2E Testing').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
})
@@ -233,7 +233,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.findByText('E2E Testing').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
cy.get('[data-cy="alert-body"]').should('contain', 'integrationFolder')
@@ -245,7 +245,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.findByText('E2E Testing').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.percySnapshot()
})
@@ -279,7 +279,7 @@ describe('setupNodeEvents', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Choose a browser')
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
})
it('handles multiple config errors and then recovers', () => {
@@ -293,7 +293,7 @@ describe('setupNodeEvents', () => {
cy.visitLaunchpad()
cy.skipWelcome()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.get('[data-cy="alert-body"]').should('contain', 'Expected baseUrl to be a fully qualified URL')
cy.withCtx(async (ctx) => {
@@ -302,7 +302,7 @@ describe('setupNodeEvents', () => {
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('[data-cy-testingType=e2e]').click()
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
cy.get('[data-cy="alert-body"]').should('contain', 'The baseUrl configuration option is now invalid when set from the root of the config object')
cy.withCtx(async (ctx) => {
@@ -310,7 +310,7 @@ describe('setupNodeEvents', () => {
})
cy.findByRole('button', { name: 'Try again' }).click()
cy.get('h1').should('contain', 'Choose a browser')
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
cy.get('[data-cy="alert"]').should('contain', 'Warning: Cannot Connect Base Url Warning')
})

11
packages/launchpad/cypress/e2e/config-warning.cy.ts
View File

@@ -232,7 +232,7 @@ describe('component testing dependency warnings', () => {
cy.skipWelcome()
cy.contains('component-testing-outdated-dependencies').click()
cy.get('[data-cy="warning-alert"]').should('not.exist')
cy.get('a').contains('Projects').click()
cy.contains('a', 'Projects').click()
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist')
.should('contain.text', 'Warning: Component Testing Mismatched Dependencies')
@@ -243,8 +243,7 @@ describe('component testing dependency warnings', () => {
cy.get('.warning-markdown').find('li').should('have.length', 3)
})
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23154
it('warns against outdated @vue/cli dependency', { retries: 15 }, () => {
it('warns against outdated @vue/cli dependency', () => {
cy.scaffoldProject('outdated-deps-vuecli3')
cy.addProject('outdated-deps-vuecli3')
cy.openGlobalMode()
@@ -252,8 +251,8 @@ describe('component testing dependency warnings', () => {
cy.skipWelcome()
cy.contains('outdated-deps-vuecli3').click()
cy.get('[data-cy="warning-alert"]').should('not.exist')
cy.get('a').contains('Projects').click()
cy.get('[data-cy-testingtype="component"]').click()
cy.contains('a', 'Projects').click()
cy.get('[data-cy-testingtype="component"]', { timeout: 12000 }).click()
cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist')
.should('contain.text', 'Warning: Component Testing Mismatched Dependencies')
.should('contain.text', '@vue/cli-service. Expected ^=4.0.0 || ^=5.0.0, found 3.12.1.')
@@ -271,7 +270,7 @@ describe('component testing dependency warnings', () => {
cy.contains('vueclivue3-configured').click()
cy.get('[data-cy="warning-alert"]').should('not.exist')
cy.get('a').contains('Projects').click()
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-cy-testingtype="component"]', { timeout: 10000 }).click()
// Wait until launch browser screen and assert warning does not exist
cy.contains('Choose a browser', { timeout: 12000 })

4
packages/launchpad/cypress/e2e/migration.cy.ts
View File

@@ -59,7 +59,7 @@ function finishMigrationAndContinue () {
}
function checkOutcome () {
cy.contains('Welcome to Cypress!').should('be.visible')
cy.contains('Welcome to Cypress!', { timeout: 10000 }).should('be.visible')
}
function runAutoRename () {
@@ -95,7 +95,7 @@ describe('global mode', () => {
cy.contains('migration-e2e-export-default').click()
// rename integration->e2e
cy.get(renameAutoStep).should('exist')
cy.get(renameAutoStep, { timeout: 10000 }).should('exist')
cy.get(renameManualStep).should('not.exist')
// cypress/support/index.ts -> cypress/support/e2e.ts

15
packages/launchpad/cypress/e2e/project-setup.cy.ts
View File

@@ -556,8 +556,7 @@ describe('Launchpad: Setup Project', () => {
})
})
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153
describe('Command for package managers', { retries: 15 }, () => {
describe('Command for package managers', () => {
it('makes the right command for yarn', () => {
scaffoldAndOpenProject('pristine-yarn')
@@ -566,7 +565,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByDisplayValue('yarn add -D react-scripts react-dom react').should('be.visible')
})
@@ -578,7 +577,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByDisplayValue('pnpm install -D react-scripts react-dom react')
})
@@ -588,9 +587,9 @@ describe('Launchpad: Setup Project', () => {
cy.visitLaunchpad()
cy.get('[data-cy-testingtype="component"]').click()
cy.get('button').should('be.visible').contains('Vue.js 3(detected)')
cy.get('button').should('be.visible').contains('Vite(detected)')
cy.findByText('Next step').click()
cy.contains('button', 'Vue.js 3(detected)').should('be.visible')
cy.contains('button', 'Vite(detected)').should('be.visible')
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByTestId('alert').contains(`You've successfully installed all required dependencies.`)
})
@@ -602,7 +601,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-cy-testingtype="component"]').click()
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.contains('button', 'Next step').should('not.be.disabled').click()
cy.findByDisplayValue('npm install -D react-scripts react-dom react')
})
})

10
packages/launchpad/cypress/e2e/scaffold-project.cy.ts
View File

@@ -168,6 +168,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => {
})
it('generates valid config file for pristine project without cypress installed', () => {
cy.intercept('mutation-ScaffoldedFiles_completeSetup').as('mutationScaffoldedFiles')
cy.intercept('query-MainLaunchpadQuery').as('mainLaunchpadQuery')
cy.intercept('query-HeaderBar_HeaderBarQuery').as('headerBarQuery')
cy.intercept('query-CloudViewerAndProject_RequiredData').as('cloudViewerAndProjectRequiredData')
cy.scaffoldProject('pristine')
cy.openProject('pristine')
cy.withCtx((ctx) => ctx.currentProject).then((currentProject) => {
@@ -178,6 +182,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => {
cy.skipWelcome()
cy.contains('button', cy.i18n.testingType.e2e.name).click()
cy.contains('button', cy.i18n.setupPage.step.continue).click()
cy.contains('h1', cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible')
cy.wait('@mutationScaffoldedFiles')
cy.wait('@mainLaunchpadQuery')
cy.wait('@headerBarQuery')
cy.wait('@cloudViewerAndProjectRequiredData')
cy.get('h1').contains(cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible')
})
})

8
packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts
View File

@@ -366,12 +366,12 @@ describe('Launchpad Top Nav Workflows', () => {
options.sinon.stub(ctx._apis.authApi, 'logIn').callsFake(async (onMessage) => {
setTimeout(() => {
onMessage({ browserOpened: true })
}, 500)
}, 2000)
return new Promise((resolve) => {
setTimeout(() => {
resolve(options.user)
}, 2000)
}, 3000)
})
})
}, { user })
@@ -687,7 +687,7 @@ describe('Launchpad Top Nav Workflows', () => {
cy.get('[data-cy="project-card"]').click()
cy.contains('E2E Testing').click()
cy.contains('E2E Testing', { timeout: 10000 }).click()
mockLogInActionsForUser(mockUser)
logIn({ expectedNextStepText: 'Continue', displayName: mockUser.name })
@@ -707,7 +707,7 @@ describe('Launchpad Top Nav Workflows', () => {
cy.get('[data-cy="project-card"]').click()
cy.contains('E2E Testing').click()
cy.contains('E2E Testing', { timeout: 10000 }).click()
mockLogInActionsForUser(mockUser)
logIn({ expectedNextStepText: 'Connect project', displayName: mockUser.name })