mirror of
https://github.com/cypress-io/cypress.git
synced 2026-05-03 13:30:26 -05:00
Move fast_visit_spec to performance-test stage, track perf (#4989)
* Move fast_visit_spec to performance-test stage
* upload results from fast_visit_spec and proxy_performance_spec
* test
* Track URL in proxy_perf_spec
* also send commit timestamp
* use more params, use ciProviders where possible
* test
* add platform + arch
* use commit-info for timestamp, commit message
* Revert "use commit-info for timestamp, commit message"
This reverts commit a55d16af3d.
* use commit-info for timestamp, commit message
* just always have nock
* slice(1) the testCases
* send current retry with fast_visit_spec
* currentRetry
This commit is contained in:
Zach Bloomquist
committed by
Brian Mann
Brian Mann
parent
da497e44d5
commit
ca0b74930c
@@ -818,188 +818,4 @@ Error: ESOCKETTIMEDOUT
|
||||
1 of 1 failed (100%) XX:XX 3 - 3 - -
|
||||
|
||||
|
||||
`
|
||||
|
||||
exports['e2e visit resolves visits quickly in chrome (headed) 1'] = `
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Starting)
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Cypress: 1.2.3 │
|
||||
│ Browser: FooBrowser 88 │
|
||||
│ Specs: 1 found (fast_visit_spec.coffee) │
|
||||
│ Searched: cypress/integration/fast_visit_spec.coffee │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Running: fast_visit_spec.coffee... (1 of 1)
|
||||
|
||||
Warning: Cypress can only record videos when using the built in 'electron' browser.
|
||||
|
||||
You have set the browser to: 'chrome'
|
||||
|
||||
A video will not be recorded when using this browser.
|
||||
|
||||
|
||||
on localhost 95% of visits are faster than XX:XX, 80% are faster than XX:XX
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: close
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: keep-alive
|
||||
|
||||
|
||||
2 passing
|
||||
|
||||
|
||||
(Results)
|
||||
|
||||
┌──────────────────────────────────────┐
|
||||
│ Tests: 2 │
|
||||
│ Passing: 2 │
|
||||
│ Failing: 0 │
|
||||
│ Pending: 0 │
|
||||
│ Skipped: 0 │
|
||||
│ Screenshots: 0 │
|
||||
│ Video: false │
|
||||
│ Duration: X seconds │
|
||||
│ Spec Ran: fast_visit_spec.coffee │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Finished)
|
||||
|
||||
|
||||
Spec Tests Passing Failing Pending Skipped
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ ✔ fast_visit_spec.coffee XX:XX 2 2 - - - │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
All specs passed! XX:XX 2 2 - - -
|
||||
|
||||
|
||||
`
|
||||
|
||||
exports['e2e visit resolves visits quickly in electron (headless) 1'] = `
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Starting)
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Cypress: 1.2.3 │
|
||||
│ Browser: FooBrowser 88 │
|
||||
│ Specs: 1 found (fast_visit_spec.coffee) │
|
||||
│ Searched: cypress/integration/fast_visit_spec.coffee │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Running: fast_visit_spec.coffee... (1 of 1)
|
||||
|
||||
|
||||
on localhost 95% of visits are faster than XX:XX, 80% are faster than XX:XX
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: close
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: keep-alive
|
||||
|
||||
|
||||
2 passing
|
||||
|
||||
|
||||
(Results)
|
||||
|
||||
┌──────────────────────────────────────┐
|
||||
│ Tests: 2 │
|
||||
│ Passing: 2 │
|
||||
│ Failing: 0 │
|
||||
│ Pending: 0 │
|
||||
│ Skipped: 0 │
|
||||
│ Screenshots: 0 │
|
||||
│ Video: true │
|
||||
│ Duration: X seconds │
|
||||
│ Spec Ran: fast_visit_spec.coffee │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
|
||||
(Video)
|
||||
|
||||
- Started processing: Compressing to 32 CRF
|
||||
- Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds)
|
||||
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Finished)
|
||||
|
||||
|
||||
Spec Tests Passing Failing Pending Skipped
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ ✔ fast_visit_spec.coffee XX:XX 2 2 - - - │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
All specs passed! XX:XX 2 2 - - -
|
||||
|
||||
|
||||
`
|
||||
`
|
||||
@@ -0,0 +1,183 @@
|
||||
exports['cy.visit performance tests pass in chrome 1'] = `
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Starting)
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Cypress: 1.2.3 │
|
||||
│ Browser: FooBrowser 88 │
|
||||
│ Specs: 1 found (fast_visit_spec.coffee) │
|
||||
│ Searched: cypress/integration/fast_visit_spec.coffee │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Running: fast_visit_spec.coffee... (1 of 1)
|
||||
|
||||
Warning: Cypress can only record videos when using the built in 'electron' browser.
|
||||
|
||||
You have set the browser to: 'chrome'
|
||||
|
||||
A video will not be recorded when using this browser.
|
||||
|
||||
|
||||
on localhost 95% of visits are faster than XX:XX, 80% are faster than XX:XX
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: close
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: keep-alive
|
||||
|
||||
|
||||
2 passing
|
||||
|
||||
|
||||
(Results)
|
||||
|
||||
┌──────────────────────────────────────┐
|
||||
│ Tests: 2 │
|
||||
│ Passing: 2 │
|
||||
│ Failing: 0 │
|
||||
│ Pending: 0 │
|
||||
│ Skipped: 0 │
|
||||
│ Screenshots: 0 │
|
||||
│ Video: false │
|
||||
│ Duration: X seconds │
|
||||
│ Spec Ran: fast_visit_spec.coffee │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Finished)
|
||||
|
||||
|
||||
Spec Tests Passing Failing Pending Skipped
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ ✔ fast_visit_spec.coffee XX:XX 2 2 - - - │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
All specs passed! XX:XX 2 2 - - -
|
||||
|
||||
|
||||
`
|
||||
|
||||
exports['cy.visit performance tests pass in electron 1'] = `
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Starting)
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Cypress: 1.2.3 │
|
||||
│ Browser: FooBrowser 88 │
|
||||
│ Specs: 1 found (fast_visit_spec.coffee) │
|
||||
│ Searched: cypress/integration/fast_visit_spec.coffee │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Running: fast_visit_spec.coffee... (1 of 1)
|
||||
|
||||
|
||||
on localhost 95% of visits are faster than XX:XX, 80% are faster than XX:XX
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: close
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
histogram line
|
||||
✓ with connection: keep-alive
|
||||
|
||||
|
||||
2 passing
|
||||
|
||||
|
||||
(Results)
|
||||
|
||||
┌──────────────────────────────────────┐
|
||||
│ Tests: 2 │
|
||||
│ Passing: 2 │
|
||||
│ Failing: 0 │
|
||||
│ Pending: 0 │
|
||||
│ Skipped: 0 │
|
||||
│ Screenshots: 0 │
|
||||
│ Video: true │
|
||||
│ Duration: X seconds │
|
||||
│ Spec Ran: fast_visit_spec.coffee │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
|
||||
(Video)
|
||||
|
||||
- Started processing: Compressing to 32 CRF
|
||||
- Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds)
|
||||
|
||||
|
||||
====================================================================================================
|
||||
|
||||
(Run Finished)
|
||||
|
||||
|
||||
Spec Tests Passing Failing Pending Skipped
|
||||
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ ✔ fast_visit_spec.coffee XX:XX 2 2 - - - │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
All specs passed! XX:XX 2 2 - - -
|
||||
|
||||
|
||||
`
|
||||
@@ -41,7 +41,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@cypress/browserify-preprocessor": "1.1.2",
|
||||
"@cypress/commit-info": "2.1.4",
|
||||
"@cypress/commit-info": "2.2.0",
|
||||
"@cypress/get-windows-proxy": "1.5.4",
|
||||
"@cypress/icons": "0.7.0",
|
||||
"@ffmpeg-installer/ffmpeg": "1.0.19",
|
||||
|
||||
@@ -149,45 +149,3 @@ describe "e2e visit", ->
|
||||
snapshot: true
|
||||
expectedExitCode: 2
|
||||
})
|
||||
|
||||
## https://github.com/cypress-io/cypress/issues/4313
|
||||
context "resolves visits quickly", ->
|
||||
e2e.setup({
|
||||
servers: {
|
||||
port: 3434
|
||||
onServer: (app) ->
|
||||
app.get '/keepalive', (req, res) ->
|
||||
res
|
||||
.type('html').send('hi')
|
||||
|
||||
app.get '/close', (req, res) ->
|
||||
res
|
||||
.set('connection', 'close')
|
||||
.type('html').send('hi')
|
||||
}
|
||||
settings: {
|
||||
baseUrl: 'http://localhost:3434'
|
||||
}
|
||||
})
|
||||
|
||||
onStdout = (stdout) ->
|
||||
stdout
|
||||
.replace(/^\d+%\s+of visits to [^\s]+ finished in less than.*$/gm, 'histogram line')
|
||||
|
||||
it "in chrome (headed)", ->
|
||||
e2e.exec(@, {
|
||||
spec: "fast_visit_spec.coffee"
|
||||
snapshot: true
|
||||
expectedExitCode: 0
|
||||
browser: 'chrome'
|
||||
onStdout
|
||||
})
|
||||
|
||||
it "in electron (headless)", ->
|
||||
e2e.exec(@, {
|
||||
spec: "fast_visit_spec.coffee"
|
||||
snapshot: true
|
||||
expectedExitCode: 0
|
||||
browser: 'electron'
|
||||
onStdout
|
||||
})
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
const e2e = require('../support/helpers/e2e')
|
||||
|
||||
// https://github.com/cypress-io/cypress/issues/4313
|
||||
context('cy.visit performance tests', function () {
|
||||
this.retries(3)
|
||||
|
||||
e2e.setup({
|
||||
servers: {
|
||||
port: 3434,
|
||||
onServer (app) {
|
||||
app.get('/keepalive', function (req, res) {
|
||||
res.type('html').send('hi')
|
||||
})
|
||||
|
||||
app.get('/close', function (req, res) {
|
||||
res.set('connection', 'close').type('html').send('hi')
|
||||
})
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
baseUrl: 'http://localhost:3434',
|
||||
},
|
||||
})
|
||||
|
||||
const onStdout = (stdout) => {
|
||||
return stdout.replace(/^\d+%\s+of visits to [^\s]+ finished in less than.*$/gm, 'histogram line')
|
||||
}
|
||||
|
||||
context('pass', function () {
|
||||
[
|
||||
'chrome',
|
||||
'electron',
|
||||
].forEach((browser) => {
|
||||
it(`in ${browser}`, function () {
|
||||
return e2e.exec(this, {
|
||||
spec: 'fast_visit_spec.coffee',
|
||||
snapshot: true,
|
||||
expectedExitCode: 0,
|
||||
config: {
|
||||
env: {
|
||||
currentRetry: this.test._currentRetry,
|
||||
},
|
||||
},
|
||||
browser,
|
||||
onStdout,
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
+11
@@ -1,3 +1,5 @@
|
||||
require('../spec_helper')
|
||||
|
||||
const cp = require('child_process')
|
||||
const fse = require('fs-extra')
|
||||
const os = require('os')
|
||||
@@ -8,6 +10,7 @@ const { expect } = require('chai')
|
||||
const debug = require('debug')('test:proxy-performance')
|
||||
const DebuggingProxy = require('@cypress/debugging-proxy')
|
||||
const HarCapturer = require('chrome-har-capturer')
|
||||
const performance = require('../support/helpers/performance')
|
||||
const Promise = require('bluebird')
|
||||
const Table = require('console-table-printer').Table
|
||||
const sanitizeFilename = require('sanitize-filename')
|
||||
@@ -322,6 +325,7 @@ describe('Proxy Performance', function () {
|
||||
|
||||
beforeEach(function () {
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
|
||||
nock.enableNetConnect()
|
||||
})
|
||||
|
||||
before(function () {
|
||||
@@ -369,6 +373,7 @@ describe('Proxy Performance', function () {
|
||||
})
|
||||
})
|
||||
|
||||
// slice(1) since first test is used as baseline above
|
||||
testCases.slice(1).map((testCase) => {
|
||||
it(`${testCase.name} loads 1000 images, with 75% loading no more than 2x as slow as the slowest baseline request`, function () {
|
||||
debug('Current test: ', testCase.name)
|
||||
@@ -390,6 +395,12 @@ describe('Proxy Performance', function () {
|
||||
// console.log is bad for eslint, but nobody never said nothing about process.stdout.write
|
||||
process.stdout.write('Note: All times are in milliseconds.\n')
|
||||
t.printTable()
|
||||
|
||||
return Promise.map(testCases, (testCase) => {
|
||||
testCase['URL'] = urlUnderTest
|
||||
|
||||
return performance.track('Proxy Performance', testCase)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
+8
-4
@@ -24,12 +24,16 @@ fastVisitSpec = (url) ->
|
||||
i = Math.floor(p / 100 * times.length) - 1
|
||||
times[i]
|
||||
|
||||
message = [1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 97, 99, 100].map (p) ->
|
||||
"#{p}%\t of visits to #{url} finished in less than #{percentile(p)}ms"
|
||||
.join("\n")
|
||||
percentiles = [1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 97, 99, 100].map (p) ->
|
||||
[p, percentile(p)]
|
||||
|
||||
cy
|
||||
.task('console:log', message)
|
||||
.task('record:fast_visit_spec', {
|
||||
percentiles,
|
||||
url,
|
||||
browser: Cypress.config('browser').name
|
||||
currentRetry: Cypress.env('currentRetry')
|
||||
})
|
||||
.then ->
|
||||
expect(percentile(80)).to.be.lte(100)
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ const Jimp = require('jimp')
|
||||
const path = require('path')
|
||||
const Promise = require('bluebird')
|
||||
|
||||
const performance = require('../../../../test/support/helpers/performance')
|
||||
|
||||
module.exports = (on) => {
|
||||
// save some time by only reading the originals once
|
||||
let cache = {}
|
||||
@@ -97,10 +99,23 @@ module.exports = (on) => {
|
||||
})
|
||||
},
|
||||
|
||||
'console:log' (obj) {
|
||||
console.log(obj) // eslint-disable-line no-console
|
||||
'record:fast_visit_spec' ({ percentiles, url, browser, currentRetry }) {
|
||||
percentiles.forEach(([percent, percentile]) => {
|
||||
console.log(`${percent}%\t of visits to ${url} finished in less than ${percentile}ms`)
|
||||
})
|
||||
|
||||
return null
|
||||
const data = {
|
||||
url,
|
||||
browser,
|
||||
currentRetry,
|
||||
...percentiles.reduce((acc, pair) => {
|
||||
acc[pair[0]] = pair[1]
|
||||
return acc
|
||||
}, {})
|
||||
}
|
||||
|
||||
return performance.track('fast_visit_spec percentiles', data)
|
||||
.return(null)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
const ciProvider = require('../../../lib/util/ci_provider')
|
||||
const { commitInfo } = require('@cypress/commit-info')
|
||||
const pkg = require('../../../../../package.json')
|
||||
const Promise = require('bluebird')
|
||||
const rp = require('request-promise')
|
||||
|
||||
const API_URL = process.env.PERF_API_URL || 'http://localhost:2999/track'
|
||||
const API_KEY = process.env.PERF_API_KEY
|
||||
|
||||
// Store this performance record permanently.
|
||||
function track (type, data) {
|
||||
if (!API_KEY) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
return commitInfo()
|
||||
.then(({ message, timestamp }) => {
|
||||
const { sha, branch, author } = ciProvider.commitParams()
|
||||
|
||||
timestamp = new Date(timestamp * 1000).toISOString()
|
||||
|
||||
const body = {
|
||||
type,
|
||||
data: {
|
||||
'package.json Version': pkg.version,
|
||||
'Next Version': process.env.NEXT_DEV_VERSION,
|
||||
'Commit SHA': sha,
|
||||
'Commit Branch': branch,
|
||||
'Commit Author': author,
|
||||
'Commit Message': message,
|
||||
'Commit Timestamp': timestamp,
|
||||
'Build URL': process.env.CIRCLE_BUILD_URL,
|
||||
'Build Platform': process.platform,
|
||||
'Build Arch': process.arch,
|
||||
...data,
|
||||
},
|
||||
}
|
||||
|
||||
return rp.post({
|
||||
url: API_URL,
|
||||
json: true,
|
||||
headers: {
|
||||
authorization: `token ${API_KEY}`,
|
||||
},
|
||||
body,
|
||||
timeout: 5000,
|
||||
})
|
||||
})
|
||||
.catchReturn()
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
track,
|
||||
}
|
||||
Reference in New Issue
Block a user