fix: failing Create Instance api call no longer skips spec (#30697)

* do not retry instance creation, report that cannot proceed with failed cloud in serial

* update system tests

* changelog

* changelog

* Update cli/CHANGELOG.md

Co-authored-by: Jennifer Shehane <jennifer@cypress.io>

* update error language

* snapshots

---------

Co-authored-by: Jennifer Shehane <jennifer@cypress.io>
This commit is contained in:
Cacie Prins
2024-12-03 09:52:00 -05:00
committed by GitHub
parent 9c2277e73d
commit 32c7b72c7c
7 changed files with 158 additions and 143 deletions

View File

@@ -1,5 +1,4 @@
<!-- See the ../guides/writing-the-cypress-changelog.md for details on writing the changelog. -->
## 13.16.1
_Released 11/26/2024 (PENDING)_
@@ -7,6 +6,7 @@ _Released 11/26/2024 (PENDING)_
**Bugfixes:**
- Support multiple imports of one module with multiple lines. Addressed in [#30314](https://github.com/cypress-io/cypress/pull/30314).
- During recorded or parallel runs, execution will fail if Cypress is unable to confirm the creation of an instance instead of skipping the spec. Addresses [#30628](https://github.com/cypress-io/cypress/issues/30628).
## 13.16.0

View File

@@ -38,7 +38,7 @@
<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561"><span style="color:#de73ff">StatusCodeError: 500 - &quot;Internal Server Error&quot;<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561">Because you passed the <span style="color:#de73ff">--parallel<span style="color:#e05561"> flag, this run cannot proceed because it requires a valid response from our servers.<span style="color:#e6e6e6">
<span style="color:#e05561">Because you passed the <span style="color:#de73ff">--parallel<span style="color:#e05561"> flag, this run cannot proceed since it requires a valid response from our servers.<span style="color:#e6e6e6">
<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561">The --group flag you passed was: <span style="color:#e5e510">foo<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561">The --ciBuildId flag you passed was: <span style="color:#e5e510">invalid<span style="color:#e05561"><span style="color:#e6e6e6"></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span>

View File

@@ -38,6 +38,8 @@
<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561"><span style="color:#de73ff">StatusCodeError: 500 - &quot;Internal Server Error&quot;<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561">Because you passed the <span style="color:#de73ff">--record<span style="color:#e05561"> flag, this run cannot proceed since it requires a valid response from our servers.<span style="color:#e6e6e6">
<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561">The --group flag you passed was: <span style="color:#e5e510">foo<span style="color:#e05561"><span style="color:#e6e6e6">
<span style="color:#e05561">The --ciBuildId flag you passed was: <span style="color:#e5e510">invalid<span style="color:#e05561"><span style="color:#e6e6e6"></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span>
<span style="color:#e05561">The --ciBuildId flag you passed was: <span style="color:#e5e510">invalid<span style="color:#e05561"><span style="color:#e6e6e6"></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span></span>
</pre></body></html>

View File

@@ -175,7 +175,7 @@ export const AllCypressErrors = {
${fmt.highlightSecondary(arg1.response)}
Because you passed the ${fmt.flag(`--parallel`)} flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the ${fmt.flag(`--parallel`)} flag, this run cannot proceed since it requires a valid response from our servers.
${fmt.listFlags(arg1.flags, {
group: '--group',
@@ -188,6 +188,8 @@ export const AllCypressErrors = {
${fmt.highlightSecondary(arg1.response)}
Because you passed the ${fmt.flag(`--record`)} flag, this run cannot proceed since it requires a valid response from our servers.
${fmt.listFlags(arg1.flags, {
group: '--group',
ciBuildId: '--ciBuildId',

View File

@@ -458,22 +458,20 @@ export default {
'platform',
])
return retryWithBackoff((attemptIndex) => {
return rp.post({
body,
url: recordRoutes.instances(runId),
json: true,
encrypt: preflightResult.encrypt,
timeout: timeout ?? SIXTY_SECONDS,
headers: {
'x-route-version': '5',
'x-cypress-run-id': runId,
'x-cypress-request-attempt': attemptIndex,
},
})
.catch(RequestErrors.StatusCodeError, formatResponseBody)
.catch(tagError)
return rp.post({
body,
url: recordRoutes.instances(runId),
json: true,
encrypt: preflightResult.encrypt,
timeout: timeout ?? SIXTY_SECONDS,
headers: {
'x-route-version': '5',
'x-cypress-run-id': runId,
'x-cypress-request-attempt': 0,
},
})
.catch(RequestErrors.StatusCodeError, formatResponseBody)
.catch(tagError)
},
postInstanceTests (options) {

View File

@@ -1124,6 +1124,8 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
`
exports['e2e record api interaction errors create run 500 when grouping without parallelization errors and exits 1'] = `
@@ -1131,6 +1133,8 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1141,7 +1145,7 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1167,7 +1171,7 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1193,6 +1197,8 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
`
exports['e2e record api interaction errors update instance 500 does not proceed and exits with error when parallelizing and updating instance 1'] = `
@@ -1250,7 +1256,7 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1409,6 +1415,8 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
`
exports['e2e record api interaction errors postInstanceTests without parallelization errors and exits 1'] = `
@@ -1435,6 +1443,8 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: 1
@@ -1464,7 +1474,7 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1526,6 +1536,8 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
`
exports['e2e record api interaction errors update instance stdout warns but proceeds 1'] = `
@@ -1705,113 +1717,6 @@ exports['e2e record api interaction errors uploading assets warns but proceeds 1
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
`
exports['e2e record api interaction errors api retries on error warns and does not create or update instances 1'] = `
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 3 more times in X second(s)...
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 2 more times in X second(s)...
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 1 more time in X second(s)...
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass.cy.js) │
│ Searched: cypress/e2e/record_pass* │
│ Params: Tag: nightly, Group: foo, Parallel: true │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 3 more times in X second(s)...
────────────────────────────────────────────────────────────────────────────────────────────────────
Running: record_pass.cy.js (1 of 1)
Estimated: X second(s)
record pass
✓ passes
- is pending
1 passing
1 pending
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 1 │
│ Skipped: 0 │
│ Screenshots: 1 │
│ Video: false │
│ Duration: X seconds │
│ Estimated: X second(s) │
│ Spec Ran: record_pass.cy.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
(Screenshots)
- /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022)
(Uploading Cloud Artifacts)
- Video - Nothing to upload
- Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png
- Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings
Uploading Cloud Artifacts: . . . . .
(Uploaded Cloud Artifacts)
- Screenshot - Done Uploading 1 kB in Xm, Ys ZZ.ZZms 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✔ All specs passed! XX:XX 2 1 - 1 -
───────────────────────────────────────────────────────────────────────────────────────────────────────
Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12
`
exports['e2e record api interaction errors sendPreflight [F1] socket errors fails after retrying 1'] = `
@@ -1825,7 +1730,7 @@ We encountered an unexpected error communicating with our servers.
RequestError: Error: socket hang up
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1843,7 +1748,7 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1861,7 +1766,7 @@ We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1903,7 +1808,7 @@ We encountered an unexpected error communicating with our servers.
DecryptionError: JWE Recipients missing or incorrect type
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1915,7 +1820,7 @@ We encountered an unexpected error communicating with our servers.
DecryptionError: JWE Recipients missing or incorrect type
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -1927,7 +1832,7 @@ We encountered an unexpected error communicating with our servers.
DecryptionError: General JWE must be an object
Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers.
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
@@ -4479,3 +4384,88 @@ http://localhost:1234/capture-protocol/upload/?x-amz-credential=XXXXXXXX&x-amz-s
`
exports['e2e record api interaction errors api retries on error errors and does not create or update instances when parallel 1'] = `
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 3 more times in X second(s)...
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 2 more times in X second(s)...
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 1 more time in X second(s)...
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass.cy.js) │
│ Searched: cypress/e2e/record_pass* │
│ Params: Tag: nightly, Group: foo, Parallel: true │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers.
The --group flag you passed was: foo
The --ciBuildId flag you passed was: ciBuildId123
`
exports['e2e record api interaction errors api retries on error errors and does not create or update instances when recording and not parallel 1'] = `
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 3 more times in X second(s)...
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 2 more times in X second(s)...
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
We will retry 1 more time in X second(s)...
====================================================================================================
(Run Starting)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (record_pass.cy.js) │
│ Searched: cypress/e2e/record_pass* │
│ Params: Tag: nightly, Group: false, Parallel: false │
│ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
We encountered an unexpected error communicating with our servers.
StatusCodeError: 500 - "Internal Server Error"
Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers.
`

View File

@@ -1615,7 +1615,11 @@ describe('e2e record', () => {
setupStubbedServer(routes)
it('warns and does not create or update instances', function () {
beforeEach(() => {
count = 0
})
it('errors and does not create or update instances when parallel', function () {
process.env.API_RETRY_INTERVALS = '1000,2000,3000'
return systemTests.exec(this, {
@@ -1628,6 +1632,7 @@ describe('e2e record', () => {
parallel: true,
snapshot: true,
ciBuildId: 'ciBuildId123',
expectedExitCode: 1,
})
.then(() => {
const urls = getRequestUrls()
@@ -1638,12 +1643,30 @@ describe('e2e record', () => {
'POST /runs',
'POST /runs',
'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances',
'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances',
'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests',
'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results',
'PUT /screenshots/1.png',
'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/artifacts',
'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout',
])
})
})
it('errors and does not create or update instances when recording and not parallel', function () {
process.env.API_RETRY_INTERVALS = '1000,2000,3000'
return systemTests.exec(this, {
key: 'f858a2bc-b469-4e48-be67-0876339ee7e1',
configFile: 'cypress-with-project-id.config.js',
spec: 'record_pass*',
tag: 'nightly',
record: true,
snapshot: true,
expectedExitCode: 1,
})
.then(() => {
const urls = getRequestUrls()
expect(urls).to.deep.eq([
'POST /runs',
'POST /runs',
'POST /runs',
'POST /runs',
'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances',
])
})