chore: correlate proxied HTTP requests with browser pre-request data (#16835)

This commit is contained in:
Zach Bloomquist
2021-06-09 14:39:51 -04:00
committed by GitHub
parent 2938df6411
commit 66303f6331
29 changed files with 491 additions and 160 deletions
+2 -2
View File
@@ -975,14 +975,14 @@ jobs:
- run: yarn test-mocha
# test binary build code
- run: yarn test-scripts
# check for compile errors with the releaserc scripts
- run: yarn test-npm-package-release-script
# make sure our snapshots are compared correctly
- run: yarn test-mocha-snapshot
# make sure packages with TypeScript can be transpiled to JS
- run: yarn lerna run build-prod --stream
# run unit tests from each individual package
- run: yarn test
# check for compile errors with the releaserc scripts
- run: yarn test-npm-package-release-script
- verify-mocha-results:
expectedResultCount: 9
- store_test_results:
+41 -20
View File
@@ -3,10 +3,12 @@ import CyServer from '@packages/server'
import {
CypressIncomingRequest,
CypressOutgoingResponse,
BrowserPreRequest,
} from '@packages/proxy'
import debugModule from 'debug'
import Debug from 'debug'
import ErrorMiddleware from './error-middleware'
import { HttpBuffers } from './util/buffers'
import { GetPreRequestCb, PreRequests } from './util/prerequests'
import { IncomingMessage } from 'http'
import { NetStubbingState } from '@packages/net-stubbing'
import Bluebird from 'bluebird'
@@ -16,7 +18,7 @@ import RequestMiddleware from './request-middleware'
import ResponseMiddleware from './response-middleware'
import { DeferredSourceMapCache } from '@packages/rewriter'
const debug = debugModule('cypress:proxy:http')
const debugRequests = Debug('cypress-verbose:proxy:http')
export enum HttpStages {
IncomingRequest,
@@ -35,9 +37,12 @@ export type HttpMiddlewareStacks = {
type HttpMiddlewareCtx<T> = {
req: CypressIncomingRequest
res: CypressOutgoingResponse
shouldCorrelatePreRequests: () => boolean
stage: HttpStages
debug: Debug.Debugger
middleware: HttpMiddlewareStacks
deferSourceMapRewrite: (opts: { js: string, url: string }) => string
getPreRequest: (cb: GetPreRequestCb) => void
} & T
export const defaultMiddleware = {
@@ -48,6 +53,7 @@ export const defaultMiddleware = {
export type ServerCtx = Readonly<{
config: CyServer.Config
shouldCorrelatePreRequests?: () => boolean
getFileServerToken: () => string
getRemoteState: CyServer.getRemoteState
netStubbingState: NetStubbingState
@@ -82,10 +88,8 @@ type HttpMiddlewareThis<T> = HttpMiddlewareCtx<T> & ServerCtx & Readonly<{
skipMiddleware: (name: string) => void
}>
export function _runStage (type: HttpStages, ctx: any) {
const stage = HttpStages[type]
debug('Entering stage %o', { stage })
export function _runStage (type: HttpStages, ctx: any, onError) {
ctx.stage = HttpStages[type]
const runMiddlewareStack = () => {
const middlewares = ctx.middleware[type]
@@ -131,8 +135,6 @@ export function _runStage (type: HttpStages, ctx: any) {
return resolve()
}
debug('Running middleware %o', { stage, middlewareName })
const fullCtx = {
next: () => {
copyChangedCtx()
@@ -147,21 +149,19 @@ export function _runStage (type: HttpStages, ctx: any) {
_end()
},
onError: (error: Error) => {
debug('Error in middleware %o', { stage, middlewareName, error })
ctx.debug('Error in middleware %o', { middlewareName, error })
if (type === HttpStages.Error) {
return
}
ctx.error = error
_end(_runStage(HttpStages.Error, ctx))
onError(error)
_end(_runStage(HttpStages.Error, ctx, onError))
},
skipMiddleware: (name) => {
ctx.middleware[type] = _.omit(ctx.middleware[type], name)
},
...ctx,
}
@@ -174,19 +174,18 @@ export function _runStage (type: HttpStages, ctx: any) {
}
return runMiddlewareStack()
.then(() => {
debug('Leaving stage %o', { stage })
})
}
export class Http {
buffers: HttpBuffers
config: CyServer.Config
shouldCorrelatePreRequests: () => boolean
deferredSourceMapCache: DeferredSourceMapCache
getFileServerToken: () => string
getRemoteState: () => any
middleware: HttpMiddlewareStacks
netStubbingState: NetStubbingState
preRequests: PreRequests = new PreRequests()
request: any
socket: CyServer.Socket
@@ -195,6 +194,7 @@ export class Http {
this.deferredSourceMapCache = new DeferredSourceMapCache(opts.request)
this.config = opts.config
this.shouldCorrelatePreRequests = opts.shouldCorrelatePreRequests || (() => false)
this.getFileServerToken = opts.getFileServerToken
this.getRemoteState = opts.getRemoteState
this.middleware = opts.middleware
@@ -213,27 +213,43 @@ export class Http {
res,
buffers: this.buffers,
config: this.config,
shouldCorrelatePreRequests: this.shouldCorrelatePreRequests,
getFileServerToken: this.getFileServerToken,
getRemoteState: this.getRemoteState,
request: this.request,
middleware: _.cloneDeep(this.middleware),
netStubbingState: this.netStubbingState,
socket: this.socket,
debug: (formatter, ...args) => {
debugRequests(`%s %s %s ${formatter}`, ctx.req.method, ctx.req.proxiedUrl, ctx.stage, ...args)
},
deferSourceMapRewrite: (opts) => {
this.deferredSourceMapCache.defer({
resHeaders: ctx.incomingRes.headers,
...opts,
})
},
getPreRequest: (cb) => {
this.preRequests.get(ctx.req, ctx.debug, cb)
},
}
return _runStage(HttpStages.IncomingRequest, ctx)
const onError = () => {
if (ctx.req.browserPreRequest) {
// browsers will retry requests in the event of network errors, but they will not send pre-requests,
// so try to re-use the current browserPreRequest for the next retry
ctx.debug('Re-using pre-request data %o', ctx.req.browserPreRequest)
this.addPendingBrowserPreRequest(ctx.req.browserPreRequest)
}
}
return _runStage(HttpStages.IncomingRequest, ctx, onError)
.then(() => {
if (ctx.incomingRes) {
return _runStage(HttpStages.IncomingResponse, ctx)
return _runStage(HttpStages.IncomingResponse, ctx, onError)
}
return debug('warning: Request was not fulfilled with a response.')
return ctx.debug('Warning: Request was not fulfilled with a response.')
})
}
@@ -253,9 +269,14 @@ export class Http {
reset () {
this.buffers.reset()
this.preRequests = new PreRequests()
}
setBuffer (buffer) {
return this.buffers.set(buffer)
}
addPendingBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
this.preRequests.addPending(browserPreRequest)
}
}
+34 -6
View File
@@ -19,11 +19,40 @@ const LogRequest: RequestMiddleware = function () {
this.next()
}
const CorrelateBrowserPreRequest: RequestMiddleware = async function () {
if (!this.shouldCorrelatePreRequests()) {
return this.next()
}
if (this.req.headers['x-cypress-resolving-url']) {
this.debug('skipping prerequest for resolve:url')
delete this.req.headers['x-cypress-resolving-url']
return this.next()
}
this.debug('waiting for prerequest')
this.getPreRequest(((browserPreRequest) => {
this.req.browserPreRequest = browserPreRequest
this.next()
}))
}
const SendToDriver: RequestMiddleware = function () {
const { browserPreRequest } = this.req
if (browserPreRequest) {
this.socket.toDriver('proxy:incoming:request', browserPreRequest)
}
this.next()
}
const MaybeEndRequestWithBufferedResponse: RequestMiddleware = function () {
const buffer = this.buffers.take(this.req.proxiedUrl)
if (buffer) {
debug('got a buffer %o', _.pick(buffer, 'url'))
this.debug('ending request with buffered response')
this.res.wantsInjection = 'full'
return this.onResponse(buffer.response, buffer.stream)
@@ -52,10 +81,7 @@ const EndRequestsToBlockedHosts: RequestMiddleware = function () {
if (matches) {
this.res.set('x-cypress-matched-blocked-host', matches)
debug('blocking request %o', {
url: this.req.proxiedUrl,
matches,
})
this.debug('blocking request %o', { matches })
this.res.status(503).end()
@@ -127,7 +153,7 @@ const SendRequestOutgoing: RequestMiddleware = function () {
req.on('error', this.onError)
req.on('response', (incomingRes) => this.onResponse(incomingRes, req))
this.req.on('aborted', () => {
debug('request aborted')
this.debug('request aborted')
req.abort()
})
@@ -141,6 +167,8 @@ const SendRequestOutgoing: RequestMiddleware = function () {
export default {
LogRequest,
CorrelateBrowserPreRequest,
SendToDriver,
MaybeEndRequestWithBufferedResponse,
InterceptRequest,
RedirectToClientRouteIfUnloaded,
+112
View File
@@ -0,0 +1,112 @@
import {
CypressIncomingRequest,
BrowserPreRequest,
} from '@packages/proxy'
import Debug from 'debug'
import _ from 'lodash'
const debug = Debug('cypress:proxy:http:util:prerequests')
const debugVerbose = Debug('cypress-verbose:proxy:http:util:prerequests')
const metrics: any = {
browserPreRequestsReceived: 0,
proxyRequestsReceived: 0,
immediatelyMatchedRequests: 0,
eventuallyReceivedPreRequest: [],
neverReceivedPreRequest: [],
}
process.once('exit', () => {
debug('metrics: %o', metrics)
})
function removeOne<T> (a: Array<T>, predicate: (v: T) => boolean): T | void {
for (const i in a) {
const v = a[i]
if (predicate(v)) {
a.splice(i as unknown as number, 1)
return v
}
}
}
function matches (preRequest: BrowserPreRequest, req: Pick<CypressIncomingRequest, 'proxiedUrl' | 'method'>) {
return preRequest.method === req.method && preRequest.url === req.proxiedUrl
}
export type GetPreRequestCb = (browserPreRequest?: BrowserPreRequest) => void
export class PreRequests {
pendingBrowserPreRequests: Array<BrowserPreRequest> = []
requestsPendingPreRequestCbs: Array<{
cb: (browserPreRequest: BrowserPreRequest) => void
method: string
proxiedUrl: string
}> = []
get (req: CypressIncomingRequest, ctxDebug, cb: GetPreRequestCb) {
metrics.proxyRequestsReceived++
const pendingBrowserPreRequest = removeOne(this.pendingBrowserPreRequests, (browserPreRequest) => {
return matches(browserPreRequest, req)
})
if (pendingBrowserPreRequest) {
metrics.immediatelyMatchedRequests++
ctxDebug('matches pending pre-request %o', pendingBrowserPreRequest)
return cb(pendingBrowserPreRequest)
}
const timeout = setTimeout(() => {
metrics.neverReceivedPreRequest.push({ url: req.proxiedUrl })
ctxDebug('500ms passed without a pre-request, continuing request with an empty pre-request field!')
remove()
cb()
}, 500)
const startedMs = Date.now()
const remove = _.once(() => removeOne(this.requestsPendingPreRequestCbs, (v) => v === requestPendingPreRequestCb))
const requestPendingPreRequestCb = {
cb: (browserPreRequest) => {
const afterMs = Date.now() - startedMs
metrics.eventuallyReceivedPreRequest.push({ url: browserPreRequest.url, afterMs })
ctxDebug('received pre-request after %dms %o', afterMs, browserPreRequest)
clearTimeout(timeout)
remove()
cb(browserPreRequest)
},
proxiedUrl: req.proxiedUrl,
method: req.method,
}
this.requestsPendingPreRequestCbs.push(requestPendingPreRequestCb)
}
addPending (browserPreRequest: BrowserPreRequest) {
if (this.pendingBrowserPreRequests.indexOf(browserPreRequest) !== -1) {
return
}
metrics.browserPreRequestsReceived++
const requestPendingPreRequestCb = removeOne(this.requestsPendingPreRequestCbs, (req) => {
return matches(browserPreRequest, req)
})
if (requestPendingPreRequestCb) {
debugVerbose('immediately matched pre-request %o', browserPreRequest)
return requestPendingPreRequestCb.cb(browserPreRequest)
}
debugVerbose('queuing pre-request to be matched later %o %o', browserPreRequest, this.pendingBrowserPreRequests)
this.pendingBrowserPreRequests.push(browserPreRequest)
}
}
+5
View File
@@ -1,4 +1,5 @@
import { Http, ServerCtx } from './http'
import { BrowserPreRequest } from './types'
export class NetworkProxy {
http: Http
@@ -7,6 +8,10 @@ export class NetworkProxy {
this.http = new Http(opts)
}
addPendingBrowserPreRequest (preRequest: BrowserPreRequest) {
this.http.addPendingBrowserPreRequest(preRequest)
}
handleHttpRequest (req, res) {
this.http.handle(req, res)
}
+14
View File
@@ -8,6 +8,7 @@ export type CypressIncomingRequest = Request & {
proxiedUrl: string
abort: () => void
requestId: string
browserPreRequest?: BrowserPreRequest
body?: string
responseTimeout?: number
followRedirect?: boolean
@@ -28,3 +29,16 @@ export { ErrorMiddleware } from './http/error-middleware'
export { RequestMiddleware } from './http/request-middleware'
export { ResponseMiddleware } from './http/response-middleware'
export type ResourceType = 'fetch' | 'xhr' | 'websocket' | 'stylesheet' | 'script' | 'image' | 'font' | 'cspviolationreport' | 'ping' | 'manifest' | 'other'
/**
* Metadata about an HTTP request, according to the browser's pre-request event.
*/
export type BrowserPreRequest = {
requestId: string
method: string
url: string
resourceType: ResourceType
originalResourceType: string | undefined
}
@@ -6,6 +6,8 @@ describe('http/request-middleware', function () {
it('exports the members in the correct order', function () {
expect(_.keys(RequestMiddleware)).to.have.ordered.members([
'LogRequest',
'CorrelateBrowserPreRequest',
'SendToDriver',
'MaybeEndRequestWithBufferedResponse',
'InterceptRequest',
'RedirectToClientRouteIfUnloaded',
@@ -0,0 +1,34 @@
import { PreRequests } from '@packages/proxy/lib/http/util/prerequests'
import { BrowserPreRequest, CypressIncomingRequest } from '@packages/proxy'
import { expect } from 'chai'
import sinon from 'sinon'
describe('http/util/prerequests', () => {
let preRequests: PreRequests
beforeEach(() => {
preRequests = new PreRequests()
})
it('synchronously matches a pre-request that existed at the time of the request', () => {
preRequests.addPending({ requestId: '1234', url: 'foo', method: 'GET' } as BrowserPreRequest)
const cb = sinon.stub()
preRequests.get({ proxiedUrl: 'foo', method: 'GET' } as CypressIncomingRequest, () => {}, cb)
const { args } = cb.getCall(0)
expect(args[0]).to.include({ requestId: '1234', url: 'foo', method: 'GET' })
})
it('synchronously matches a pre-request added after the request', (done) => {
const cb = (preRequest) => {
expect(preRequest).to.include({ requestId: '1234', url: 'foo', method: 'GET' })
done()
}
preRequests.get({ proxiedUrl: 'foo', method: 'GET' } as CypressIncomingRequest, () => {}, cb)
preRequests.addPending({ requestId: '1234', url: 'foo', method: 'GET' } as BrowserPreRequest)
})
})
+1 -1
View File
@@ -45,7 +45,7 @@ export class ProjectCt extends ProjectBase<ServerCt> {
return this._initPlugins(cfgForComponentTesting, options)
.then(({ cfg, specsStore }) => {
return this.server.open(cfg, specsStore, this, options.onError, options.onWarning)
return this.server.open(cfg, specsStore, this, options.onError, options.onWarning, this.shouldCorrelatePreRequests)
.then(([port, warning]) => {
return {
cfg,
+2 -2
View File
@@ -13,7 +13,7 @@ type WarningErr = Record<string, any>
const debug = Debug('cypress:server-ct:server')
export class ServerCt extends ServerBase<SocketCt> {
open (config, specsStore, project, onError, onWarning) {
open (config, specsStore, project, onError, onWarning, shouldCorrelatePreRequests) {
debug('server open')
return Bluebird.try(() => {
@@ -35,7 +35,7 @@ export class ServerCt extends ServerBase<SocketCt> {
return this._getRemoteState()
}
this.createNetworkProxy(config, getRemoteState)
this.createNetworkProxy(config, getRemoteState, shouldCorrelatePreRequests)
createRoutes({
app,
+1 -1
View File
@@ -7,5 +7,5 @@
"files": [
"index.ts",
"./../ts/index.d.ts"
]
],
}
+4 -1
View File
@@ -2,9 +2,12 @@ import Bluebird from 'bluebird'
import { v4 as uuidv4 } from 'uuid'
import { Cookies } from './cookies'
import { Screenshot } from './screenshot'
import { BrowserPreRequest } from '@packages/proxy'
type NullableMiddlewareHook = (() => void) | null
export type OnBrowserPreRequest = (browserPreRequest: BrowserPreRequest) => void
interface IMiddleware {
onPush: NullableMiddlewareHook
onBeforeRequest: NullableMiddlewareHook
@@ -19,7 +22,7 @@ export class Automation {
private cookies: Cookies
private screenshot: { capture: (data: any, automate: any) => any }
constructor (cyNamespace: string, cookieNamespace: string, screenshotsFolder: string) {
constructor (cyNamespace: string, cookieNamespace: string, screenshotsFolder: string, public onBrowserPreRequest: OnBrowserPreRequest) {
this.requests = {}
// set the middleware
+130 -76
View File
@@ -1,8 +1,12 @@
/// <reference types='chrome'/>
import _ from 'lodash'
import Bluebird from 'bluebird'
import cdp from 'devtools-protocol'
import { cors } from '@packages/network'
import debugModule from 'debug'
import { Automation } from '../automation'
import { ResourceType, BrowserPreRequest } from '@packages/proxy'
const debugVerbose = debugModule('cypress-verbose:server:browsers:cdp_automation')
@@ -11,6 +15,10 @@ export type CyCookie = Pick<chrome.cookies.Cookie, 'name' | 'value' | 'expiratio
sameSite?: 'no_restriction' | 'lax' | 'strict'
}
// Cypress uses the webextension-style filtering
// https://developer.chrome.com/extensions/cookies#method-getAll
type CyCookieFilter = chrome.cookies.GetAllDetails
function convertSameSiteExtensionToCdp (str: CyCookie['sameSite']): cdp.Network.CookieSameSite | undefined {
return str ? ({
'no_restriction': 'None',
@@ -31,12 +39,6 @@ function convertSameSiteCdpToExtension (str: cdp.Network.CookieSameSite): chrome
return str.toLowerCase() as chrome.cookies.SameSiteStatus
}
// Cypress uses the webextension-style filtering
// https://developer.chrome.com/extensions/cookies#method-getAll
type CyCookieFilter = chrome.cookies.GetAllDetails
type SendDebuggerCommand = (message: string, data?: any) => Bluebird<any>
export const _domainIsWithinSuperdomain = (domain: string, suffix: string) => {
const suffixParts = suffix.split('.').filter(_.identity)
const domainParts = domain.split('.').filter(_.identity)
@@ -60,74 +62,128 @@ export const _cookieMatches = (cookie: CyCookie, filter: CyCookieFilter) => {
return true
}
export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => {
const normalizeGetCookieProps = (cookie: cdp.Network.Cookie): CyCookie => {
if (cookie.expires === -1) {
// @ts-ignore
delete cookie.expires
}
// @ts-ignore
cookie.sameSite = convertSameSiteCdpToExtension(cookie.sameSite)
// @ts-ignore
cookie.expirationDate = cookie.expires
const normalizeGetCookieProps = (cookie: cdp.Network.Cookie): CyCookie => {
if (cookie.expires === -1) {
// @ts-ignore
delete cookie.expires
// @ts-ignore
return cookie
}
const normalizeGetCookies = (cookies: cdp.Network.Cookie[]) => {
return _.map(cookies, normalizeGetCookieProps)
// @ts-ignore
cookie.sameSite = convertSameSiteCdpToExtension(cookie.sameSite)
// @ts-ignore
cookie.expirationDate = cookie.expires
// @ts-ignore
delete cookie.expires
// @ts-ignore
return cookie
}
const normalizeGetCookies = (cookies: cdp.Network.Cookie[]) => {
return _.map(cookies, normalizeGetCookieProps)
}
const normalizeSetCookieProps = (cookie: CyCookie): cdp.Network.SetCookieRequest => {
// this logic forms a SetCookie request that will be received by Chrome
// see MakeCookieFromProtocolValues for information on how this cookie data will be parsed
// @see https://cs.chromium.org/chromium/src/content/browser/devtools/protocol/network_handler.cc?l=246&rcl=786a9194459684dc7a6fded9cabfc0c9b9b37174
const setCookieRequest: cdp.Network.SetCookieRequest = _({
domain: cookie.domain,
path: cookie.path,
secure: cookie.secure,
httpOnly: cookie.httpOnly,
sameSite: convertSameSiteExtensionToCdp(cookie.sameSite),
expires: cookie.expirationDate,
})
// Network.setCookie will error on any undefined/null parameters
.omitBy(_.isNull)
.omitBy(_.isUndefined)
// set name and value at the end to get the correct typing
.extend({
name: cookie.name || '',
value: cookie.value || '',
})
.value()
// without this logic, a cookie being set on 'foo.com' will only be set for 'foo.com', not other subdomains
if (!cookie.hostOnly && cookie.domain[0] !== '.') {
const parsedDomain = cors.parseDomain(cookie.domain)
// normally, a non-hostOnly cookie should be prefixed with a .
// so if it's not a top-level domain (localhost, ...) or IP address
// prefix it with a . so it becomes a non-hostOnly cookie
if (parsedDomain && parsedDomain.tld !== cookie.domain) {
setCookieRequest.domain = `.${cookie.domain}`
}
}
const normalizeSetCookieProps = (cookie: CyCookie): cdp.Network.SetCookieRequest => {
// this logic forms a SetCookie request that will be received by Chrome
// see MakeCookieFromProtocolValues for information on how this cookie data will be parsed
// @see https://cs.chromium.org/chromium/src/content/browser/devtools/protocol/network_handler.cc?l=246&rcl=786a9194459684dc7a6fded9cabfc0c9b9b37174
if (setCookieRequest.name.startsWith('__Host-')) {
setCookieRequest.url = `https://${cookie.domain}`
delete setCookieRequest.domain
}
const setCookieRequest: cdp.Network.SetCookieRequest = _({
domain: cookie.domain,
path: cookie.path,
secure: cookie.secure,
httpOnly: cookie.httpOnly,
sameSite: convertSameSiteExtensionToCdp(cookie.sameSite),
expires: cookie.expirationDate,
return setCookieRequest
}
const normalizeResourceType = (resourceType: string | undefined): ResourceType => {
resourceType = resourceType ? resourceType.toLowerCase() : 'unknown'
if (validResourceTypes.includes(resourceType as ResourceType)) {
return resourceType as ResourceType
}
if (resourceType === 'img') {
return 'image'
}
return ffToStandardResourceTypeMap[resourceType] || 'other'
}
type SendDebuggerCommand = (message: string, data?: any) => Bluebird<any>
type OnFn = (eventName: string, cb: Function) => void
// the intersection of what's valid in CDP and what's valid in FFCDP
// Firefox: https://searchfox.org/mozilla-central/rev/98a9257ca2847fad9a19631ac76199474516b31e/remote/cdp/domains/parent/Network.jsm#22
// CDP: https://chromedevtools.github.io/devtools-protocol/tot/Network/#type-ResourceType
const validResourceTypes: ResourceType[] = ['fetch', 'xhr', 'websocket', 'stylesheet', 'script', 'image', 'font', 'cspviolationreport', 'ping', 'manifest', 'other']
const ffToStandardResourceTypeMap: { [ff: string]: ResourceType } = {
'img': 'image',
'csp': 'cspviolationreport',
'webmanifest': 'manifest',
}
export class CdpAutomation {
constructor (private sendDebuggerCommandFn: SendDebuggerCommand, onFn: OnFn, private automation: Automation) {
onFn('Network.requestWillBeSent', this.onNetworkRequestWillBeSent)
sendDebuggerCommandFn('Network.enable', {
maxTotalBufferSize: 0,
maxResourceBufferSize: 0,
maxPostDataSize: 0,
})
// Network.setCookie will error on any undefined/null parameters
.omitBy(_.isNull)
.omitBy(_.isUndefined)
// set name and value at the end to get the correct typing
.extend({
name: cookie.name || '',
value: cookie.value || '',
})
.value()
}
// without this logic, a cookie being set on 'foo.com' will only be set for 'foo.com', not other subdomains
if (!cookie.hostOnly && cookie.domain[0] !== '.') {
const parsedDomain = cors.parseDomain(cookie.domain)
private onNetworkRequestWillBeSent = (params: cdp.Network.RequestWillBeSentEvent) => {
let url = params.request.url
// normally, a non-hostOnly cookie should be prefixed with a .
// so if it's not a top-level domain (localhost, ...) or IP address
// prefix it with a . so it becomes a non-hostOnly cookie
if (parsedDomain && parsedDomain.tld !== cookie.domain) {
setCookieRequest.domain = `.${cookie.domain}`
}
// in Firefox, the hash is incorrectly included in the URL: https://bugzilla.mozilla.org/show_bug.cgi?id=1715366
if (url.includes('#')) url = url.slice(0, url.indexOf('#'))
// Firefox: https://searchfox.org/mozilla-central/rev/98a9257ca2847fad9a19631ac76199474516b31e/remote/cdp/domains/parent/Network.jsm#397
// Firefox lacks support for urlFragment and initiator, two nice-to-haves
const browserPreRequest: BrowserPreRequest = {
requestId: params.requestId,
method: params.request.method,
url,
resourceType: normalizeResourceType(params.type),
originalResourceType: params.type,
}
if (setCookieRequest.name.startsWith('__Host-')) {
setCookieRequest.url = `https://${cookie.domain}`
delete setCookieRequest.domain
}
return setCookieRequest
this.automation.onBrowserPreRequest(browserPreRequest)
}
const getAllCookies = (filter: CyCookieFilter) => {
return sendDebuggerCommandFn('Network.getAllCookies')
private getAllCookies = (filter: CyCookieFilter) => {
return this.sendDebuggerCommandFn('Network.getAllCookies')
.then((result: cdp.Network.GetAllCookiesResponse) => {
return normalizeGetCookies(result.cookies)
.filter((cookie: CyCookie) => {
@@ -140,8 +196,8 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => {
})
}
const getCookiesByUrl = (url): Bluebird<CyCookie[]> => {
return sendDebuggerCommandFn('Network.getCookies', {
private getCookiesByUrl = (url): Bluebird<CyCookie[]> => {
return this.sendDebuggerCommandFn('Network.getCookies', {
urls: [url],
})
.then((result: cdp.Network.GetCookiesResponse) => {
@@ -152,29 +208,29 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => {
})
}
const getCookie = (filter: CyCookieFilter): Bluebird<CyCookie | null> => {
return getAllCookies(filter)
private getCookie = (filter: CyCookieFilter): Bluebird<CyCookie | null> => {
return this.getAllCookies(filter)
.then((cookies) => {
return _.get(cookies, 0, null)
})
}
const onRequest = (message, data) => {
onRequest = (message, data) => {
let setCookie
switch (message) {
case 'get:cookies':
if (data.url) {
return getCookiesByUrl(data.url)
return this.getCookiesByUrl(data.url)
}
return getAllCookies(data)
return this.getAllCookies(data)
case 'get:cookie':
return getCookie(data)
return this.getCookie(data)
case 'set:cookie':
setCookie = normalizeSetCookieProps(data)
return sendDebuggerCommandFn('Network.setCookie', setCookie)
return this.sendDebuggerCommandFn('Network.setCookie', setCookie)
.then((result: cdp.Network.SetCookieResponse) => {
if (!result.success) {
// i wish CDP provided some more detail here, but this is really it in v1.3
@@ -182,10 +238,10 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => {
throw new Error(`Network.setCookie failed to set cookie: ${JSON.stringify(setCookie)}`)
}
return getCookie(data)
return this.getCookie(data)
})
case 'clear:cookie':
return getCookie(data)
return this.getCookie(data)
// tap, so we can resolve with the value of the removed cookie
// also, getting the cookie via CDP first will ensure that we send a cookie `domain` to CDP
// that matches the cookie domain that is really stored
@@ -194,14 +250,14 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => {
return
}
return sendDebuggerCommandFn('Network.deleteCookies', _.pick(cookieToBeCleared, 'name', 'domain'))
return this.sendDebuggerCommandFn('Network.deleteCookies', _.pick(cookieToBeCleared, 'name', 'domain'))
})
case 'is:automation:client:connected':
return true
case 'remote:debugger:protocol':
return sendDebuggerCommandFn(data.command, data.params)
return this.sendDebuggerCommandFn(data.command, data.params)
case 'take:screenshot':
return sendDebuggerCommandFn('Page.captureScreenshot', { format: 'png' })
return this.sendDebuggerCommandFn('Page.captureScreenshot', { format: 'png' })
.catch((err) => {
throw new Error(`The browser responded with an error when Cypress attempted to take a screenshot.\n\nDetails:\n${err.message}`)
})
@@ -212,6 +268,4 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => {
throw new Error(`No automation handler registered for: '${message}'`)
}
}
return { onRequest }
}
+5 -11
View File
@@ -177,12 +177,6 @@ const _writeChromePreferences = (userDir: string, originalPrefs: ChromePreferenc
.return()
}
const getRemoteDebuggingPort = async () => {
const port = Number(process.env.CYPRESS_REMOTE_DEBUGGING_PORT)
return port || utils.getPort()
}
/**
* Merge the different `--load-extension` arguments into one.
*
@@ -252,13 +246,13 @@ const _disableRestorePagesPrompt = function (userDir) {
// After the browser has been opened, we can connect to
// its remote interface via a websocket.
const _connectToChromeRemoteInterface = function (port, onError) {
const _connectToChromeRemoteInterface = function (port, onError, browserDisplayName) {
// @ts-ignore
la(check.userPort(port), 'expected port number to connect CRI to', port)
debug('connecting to Chrome remote interface at random port %d', port)
return protocol.getWsTargetFor(port)
return protocol.getWsTargetFor(port, browserDisplayName)
.then((wsUrl) => {
debug('received wsUrl %s for port %d', wsUrl, port)
@@ -338,7 +332,7 @@ const _handleDownloads = async function (client, dir, automation) {
const _setAutomation = (client, automation) => {
return automation.use(
CdpAutomation(client.send),
new CdpAutomation(client.send, client.on, automation),
)
}
@@ -451,7 +445,7 @@ export = {
const userDir = utils.getProfileDir(browser, isTextTerminal)
const [port, preferences] = await Bluebird.all([
getRemoteDebuggingPort(),
protocol.getRemoteDebuggingPort(),
_getChromePreferences(userDir),
])
@@ -504,7 +498,7 @@ export = {
// SECOND connect to the Chrome remote interface
// and when the connection is ready
// navigate to the actual url
const criClient = await this._connectToChromeRemoteInterface(port, options.onError)
const criClient = await this._connectToChromeRemoteInterface(port, options.onError, browser.displayName)
la(criClient, 'expected Chrome remote interface reference', criClient)
+4 -2
View File
@@ -30,12 +30,14 @@ namespace CRI {
'Page.navigate' |
'Page.startScreencast' |
'Page.screencastFrameAck' |
'Page.setDownloadBehavior'
'Page.setDownloadBehavior' |
string
export type EventName =
'Page.screencastFrame' |
'Page.downloadWillBegin' |
'Page.downloadProgress'
'Page.downloadProgress' |
string
}
/**
+11 -4
View File
@@ -35,7 +35,7 @@ const tryToCall = function (win, method) {
}
}
const _getAutomation = function (win, options) {
const _getAutomation = function (win, options, parent) {
const sendCommand = Bluebird.method((...args) => {
return tryToCall(win, () => {
return win.webContents.debugger.sendCommand
@@ -43,7 +43,15 @@ const _getAutomation = function (win, options) {
})
})
const automation = CdpAutomation(sendCommand)
const on = (eventName, cb) => {
win.webContents.debugger.on('message', (event, method, params) => {
if (method === eventName) {
cb(params)
}
})
}
const automation = new CdpAutomation(sendCommand, on, parent)
if (!options.onScreencastFrame) {
// after upgrading to Electron 8, CDP screenshots can hang if a screencast is not also running
@@ -177,10 +185,9 @@ module.exports = {
win.maximize()
}
automation.use(_getAutomation(win, preferences))
return this._launch(win, url, automation, preferences)
.tap(_maybeRecordVideo(win.webContents, preferences))
.tap(() => automation.use(_getAutomation(win, preferences, automation)))
},
_launchChild (e, url, parent, projectRoot, state, options, automation) {
@@ -6,6 +6,8 @@ import { Command } from 'marionette-client/lib/marionette/message.js'
import util from 'util'
import Foxdriver from '@benmalka/foxdriver'
import * as protocol from './protocol'
import { CdpAutomation } from './cdp_automation'
import * as CriClient from './cri-client'
const errors = require('../errors')
@@ -93,6 +95,13 @@ const attachToTabMemory = Bluebird.method((tab) => {
})
})
async function setupRemote (remotePort, automation, onError) {
const wsUrl = await protocol.getWsTargetFor(remotePort, 'Firefox')
const criClient = await CriClient.create(wsUrl, onError)
new CdpAutomation(criClient.send, criClient.on, automation)
}
const logGcDetails = () => {
const reducedTimings = {
...timings,
@@ -158,14 +167,18 @@ export default {
},
setup ({
automation,
extensions,
onError,
url,
marionettePort,
foxdriverPort,
remotePort,
}) {
return Bluebird.all([
this.setupFoxdriver(foxdriverPort),
this.setupMarionette(extensions, url, marionettePort),
remotePort && setupRemote(remotePort, automation, onError),
])
},
+16 -4
View File
@@ -15,6 +15,7 @@ import { EventEmitter } from 'events'
import os from 'os'
import treeKill from 'tree-kill'
import mimeDb from 'mime-db'
import { getRemoteDebuggingPort } from './protocol'
const errors = require('../errors')
@@ -356,7 +357,9 @@ export function _createDetachedInstance (browserInstance: BrowserInstance): Brow
return detachedInstance
}
export async function open (browser: Browser, url, options: any = {}): Promise<BrowserInstance> {
export async function open (browser: Browser, url, options: any = {}, automation): Promise<BrowserInstance> {
// see revision comment here https://wiki.mozilla.org/index.php?title=WebDriver/RemoteProtocol&oldid=1234946
const hasCdp = browser.majorVersion >= 86
const defaultLaunchOptions = utils.getDefaultLaunchOptions({
extensions: [] as string[],
preferences: _.extend({}, defaultPreferences),
@@ -369,6 +372,14 @@ export async function open (browser: Browser, url, options: any = {}): Promise<B
],
})
let remotePort
if (hasCdp) {
remotePort = await getRemoteDebuggingPort()
defaultLaunchOptions.args.push(`--remote-debugging-port=${remotePort}`)
}
if (browser.isHeadless) {
defaultLaunchOptions.args.push('-headless')
// we don't need to specify width/height since MOZ_HEADLESS_ env vars will be set
@@ -498,10 +509,11 @@ export async function open (browser: Browser, url, options: any = {}): Promise<B
MOZ_HEADLESS_HEIGHT: '1081',
})
await firefoxUtil.setup({ extensions: launchOptions.extensions, url, foxdriverPort, marionettePort })
.catch((err) => {
try {
await firefoxUtil.setup({ automation, extensions: launchOptions.extensions, url, foxdriverPort, marionettePort, remotePort, onError: options.onError })
} catch (err) {
errors.throw('FIREFOX_COULD_NOT_CONNECT', err)
})
}
if (os.platform() === 'win32') {
// override the .kill method for Windows so that the detached Firefox process closes between specs
+14 -6
View File
@@ -5,12 +5,13 @@ import Bluebird from 'bluebird'
import la from 'lazy-ass'
import Debug from 'debug'
import { Socket } from 'net'
import utils from './utils'
const errors = require('../errors')
const is = require('check-more-types')
const debug = Debug('cypress:server:browsers:protocol')
export function _getDelayMsForRetry (i) {
export function _getDelayMsForRetry (i, browserName) {
if (i < 10) {
return 100
}
@@ -20,7 +21,7 @@ export function _getDelayMsForRetry (i) {
}
if (i < 63) { // after 5 seconds, begin logging and retrying
errors.warning('CDP_RETRYING_CONNECTION', i)
errors.warning('CDP_RETRYING_CONNECTION', i, browserName)
return 1000
}
@@ -73,11 +74,18 @@ const findStartPageTarget = (connectOpts) => {
return CRI.List(_.clone(connectOpts)).then(findStartPage)
}
export async function getRemoteDebuggingPort () {
const port = Number(process.env.CYPRESS_REMOTE_DEBUGGING_PORT)
return port || utils.getPort()
}
/**
* Waits for the port to respond with connection to Chrome Remote Interface
* @param {number} port Port number to connect to
* @param {string} browserName Browser name, for warning/error messages
*/
export const getWsTargetFor = (port) => {
export const getWsTargetFor = (port: number, browserName: string) => {
debug('Getting WS connection to CRI on port %d', port)
la(is.port(port), 'expected port number', port)
@@ -91,7 +99,7 @@ export const getWsTargetFor = (port) => {
getDelayMsForRetry: (i) => {
retryIndex = i
return _getDelayMsForRetry(i)
return _getDelayMsForRetry(i, browserName)
},
}
@@ -103,7 +111,7 @@ export const getWsTargetFor = (port) => {
return findStartPageTarget(connectOpts)
.catch((err) => {
retryIndex++
const delay = _getDelayMsForRetry(retryIndex)
const delay = _getDelayMsForRetry(retryIndex, browserName)
debug('error finding CRI target, maybe retrying %o', { delay, err })
@@ -120,6 +128,6 @@ export const getWsTargetFor = (port) => {
})
.catch((err) => {
debug('failed to connect to CDP %o', { connectOpts, err })
errors.throw('CDP_COULD_NOT_CONNECT', port, err)
errors.throw('CDP_COULD_NOT_CONNECT', port, err, browserName)
})
}
+2 -2
View File
@@ -843,7 +843,7 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) {
return stripIndent`\
Cypress failed to make a connection to the Chrome DevTools Protocol after retrying for 50 seconds.
This usually indicates there was a problem opening the Chrome browser.
This usually indicates there was a problem opening the ${arg3} browser.
The CDP port requested was ${chalk.yellow(arg1)}.
@@ -865,7 +865,7 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) {
${arg1.stack}`
case 'CDP_RETRYING_CONNECTION':
return `Failed to connect to Chrome, retrying in 1 second (attempt ${chalk.yellow(arg1)}/62)`
return `Failed to connect to ${arg2}, retrying in 1 second (attempt ${chalk.yellow(arg1)}/62)`
case 'DEPRECATED_BEFORE_BROWSER_LAUNCH_ARGS':
return stripIndent`\
Deprecation Warning: The \`before:browser:launch\` plugin event changed its signature in version \`4.0.0\`
+15 -1
View File
@@ -373,7 +373,11 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
reporter = Reporter.create(reporter, cfg.reporterOptions, projectRoot)
}
this._automation = new Automation(cfg.namespace, cfg.socketIoCookie, cfg.screenshotsFolder)
const onBrowserPreRequest = (browserPreRequest) => {
this.server.addBrowserPreRequest(browserPreRequest)
}
this._automation = new Automation(cfg.namespace, cfg.socketIoCookie, cfg.screenshotsFolder, onBrowserPreRequest)
this.server.startWebsockets(this.automation, cfg, {
onReloadBrowser: options.onReloadBrowser,
@@ -439,6 +443,16 @@ export class ProjectBase<TServer extends ServerE2E | ServerCt> extends EE {
this.server.changeToUrl(url)
}
shouldCorrelatePreRequests = () => {
if (!this.browser) {
return false
}
const { family, majorVersion } = this.browser
return family === 'chromium' || (family === 'firefox' && majorVersion >= 86)
}
setCurrentSpecAndBrowser (spec, browser: Cypress.Browser) {
this.spec = spec
this.browser = browser
+1 -1
View File
@@ -30,7 +30,7 @@ export class ProjectE2E extends ProjectBase<ServerE2E> {
return updatedConfig
})
.then((cfg) => {
return this.server.open(cfg, this, options.onError, options.onWarning)
return this.server.open(cfg, this, options.onError, options.onWarning, this.shouldCorrelatePreRequests)
.then(([port, warning]) => {
return {
cfg,
+7 -2
View File
@@ -12,7 +12,7 @@ import url from 'url'
import httpsProxy from '@packages/https-proxy'
import { netStubbingState, NetStubbingState } from '@packages/net-stubbing'
import { agent, cors, httpUtils, uri } from '@packages/network'
import { NetworkProxy } from '@packages/proxy'
import { NetworkProxy, BrowserPreRequest } from '@packages/proxy'
import { SocketCt } from '@packages/server-ct'
import errors from './errors'
import logger from './logger'
@@ -197,7 +197,7 @@ export class ServerBase<TSocket extends SocketE2E | SocketCt> {
return e
}
createNetworkProxy (config, getRemoteState) {
createNetworkProxy (config, getRemoteState, shouldCorrelatePreRequests) {
const getFileServerToken = () => {
return this._fileServer.token
}
@@ -206,6 +206,7 @@ export class ServerBase<TSocket extends SocketE2E | SocketCt> {
// @ts-ignore
this._networkProxy = new NetworkProxy({
config,
shouldCorrelatePreRequests,
getRemoteState,
getFileServerToken,
socket: this.socket,
@@ -236,6 +237,10 @@ export class ServerBase<TSocket extends SocketE2E | SocketCt> {
})
}
addBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
this.networkProxy.addPendingBrowserPreRequest(browserPreRequest)
}
_createHttpServer (app): DestroyableHttpServer {
const svr = http.createServer(httpUtils.lenientOptions, app)
+6 -6
View File
@@ -53,7 +53,7 @@ export class ServerE2E extends ServerBase<SocketE2E> {
this._urlResolver = null
}
open (config: Record<string, any> = {}, project, onError, onWarning) {
open (config: Record<string, any> = {}, project, onError, onWarning, shouldCorrelatePreRequests) {
debug('server open')
la(_.isPlainObject(config), 'expected plain config object', config)
@@ -70,11 +70,8 @@ export class ServerE2E extends ServerBase<SocketE2E> {
return this._getRemoteState()
}
this.createNetworkProxy(config, getRemoteState)
this.createNetworkProxy(config, getRemoteState, shouldCorrelatePreRequests)
// TODO: this does not look like a good idea
// since we would be spawning new workers on every
// open + close of a project...
if (config.experimentalSourceRewriting) {
createInitialWorkers()
}
@@ -423,9 +420,12 @@ export class ServerE2E extends ServerBase<SocketE2E> {
if (matchesNetStubbingRoute(options)) {
// TODO: this is being used to force cy.visits to be interceptable by network stubbing
// however, network errors will be obsfucated by the proxying so this is not an ideal solution
_.assign(options, {
_.merge(options, {
proxy: `http://127.0.0.1:${this._port()}`,
agent: null,
headers: {
'x-cypress-resolving-url': '1',
},
})
}
@@ -69,8 +69,9 @@ context('lib/browsers/cdp_automation', () => {
context('.CdpAutomation', () => {
beforeEach(function () {
this.sendDebuggerCommand = sinon.stub()
this.onFn = sinon.stub()
this.automation = CdpAutomation(this.sendDebuggerCommand)
this.automation = new CdpAutomation(this.sendDebuggerCommand, this.onFn)
this.sendDebuggerCommand
.throws(new Error('not stubbed'))
@@ -66,12 +66,13 @@ describe('lib/browsers/chrome', () => {
return chrome.open('chrome', 'http://', {}, this.automation)
.then(() => {
expect(utils.getPort).to.have.been.calledOnce // to get remote interface port
expect(this.criClient.send.callCount).to.equal(4)
expect(this.criClient.send.callCount).to.equal(5)
expect(this.criClient.send).to.have.been.calledWith('Page.bringToFront')
expect(this.criClient.send).to.have.been.calledWith('Page.navigate')
expect(this.criClient.send).to.have.been.calledWith('Page.enable')
expect(this.criClient.send).to.have.been.calledWith('Page.setDownloadBehavior')
expect(this.criClient.send).to.have.been.calledWith('Network.enable')
})
})
@@ -289,6 +289,7 @@ describe('lib/browsers/electron', () => {
this.newWin = {
maximize: sinon.stub(),
setSize: sinon.stub(),
webContents: this.win.webContents,
}
this.preferences = { ...this.options }
@@ -25,7 +25,7 @@ describe('lib/browsers/protocol', () => {
let delay: number
let i = 0
while ((delay = protocol._getDelayMsForRetry(i))) {
while ((delay = protocol._getDelayMsForRetry(i, 'FooBrowser'))) {
delays.push(delay)
i++
}
@@ -35,7 +35,7 @@ describe('lib/browsers/protocol', () => {
log.getCalls().forEach((log, i) => {
const line = stripAnsi(log.args[0])
expect(line).to.include(`Failed to connect to Chrome, retrying in 1 second (attempt ${i + 18}/62)`)
expect(line).to.include(`Failed to connect to FooBrowser, retrying in 1 second (attempt ${i + 18}/62)`)
})
snapshot(delays)
@@ -46,7 +46,7 @@ describe('lib/browsers/protocol', () => {
const expectedCdpFailedError = stripIndents`
Cypress failed to make a connection to the Chrome DevTools Protocol after retrying for 50 seconds.
This usually indicates there was a problem opening the Chrome browser.
This usually indicates there was a problem opening the FooBrowser browser.
The CDP port requested was ${chalk.yellow('12345')}.
@@ -57,7 +57,7 @@ describe('lib/browsers/protocol', () => {
const innerErr = new Error('cdp connection failure')
sinon.stub(connect, 'createRetryingSocket').callsArgWith(1, innerErr)
const p = protocol.getWsTargetFor(12345)
const p = protocol.getWsTargetFor(12345, 'FooBrowser')
return expect(p).to.eventually.be.rejected
.and.property('message').include(expectedCdpFailedError)
@@ -77,7 +77,7 @@ describe('lib/browsers/protocol', () => {
sinon.stub(connect, 'createRetryingSocket').callsArgWith(1, null, { end })
const p = protocol.getWsTargetFor(12345)
const p = protocol.getWsTargetFor(12345, 'FooBrowser')
return expect(p).to.eventually.be.rejected
.and.property('message').include(expectedCdpFailedError)
@@ -106,7 +106,7 @@ describe('lib/browsers/protocol', () => {
sinon.stub(connect, 'createRetryingSocket').callsArgWith(1, null, { end })
const p = protocol.getWsTargetFor(12345)
const p = protocol.getWsTargetFor(12345, 'FooBrowser')
await expect(p).to.eventually.equal('bar')
expect(end).to.be.calledOnce
@@ -139,7 +139,7 @@ describe('lib/browsers/protocol', () => {
.onSecondCall().resolves([])
.onThirdCall().resolves(targets)
const targetUrl = await protocol.getWsTargetFor(port)
const targetUrl = await protocol.getWsTargetFor(port, 'FooBrowser')
expect(criList).to.have.been.calledThrice
expect(targetUrl).to.equal('ws://debug-url')
@@ -169,7 +169,7 @@ describe('lib/browsers/protocol', () => {
.onSecondCall().resolves([])
.onThirdCall().resolves(targets)
const targetUrl = await protocol.getWsTargetFor(port)
const targetUrl = await protocol.getWsTargetFor(port, 'FooBrowser')
expect(criList).to.have.been.calledThrice
expect(targetUrl).to.equal('ws://debug-url')
@@ -180,7 +180,7 @@ describe('lib/browsers/protocol', () => {
log.getCalls().forEach((log, i) => {
const line = stripAnsi(log.args[0])
expect(line).to.include(`Failed to connect to Chrome, retrying in 1 second (attempt ${i + 18}/62)`)
expect(line).to.include(`Failed to connect to FooBrowser, retrying in 1 second (attempt ${i + 18}/62)`)
})
})
})
+1 -1
View File
@@ -8,6 +8,6 @@
"./../ts/index.d.ts"
],
"compilerOptions": {
"types": ["mocha", "node", "chrome"]
"types": ["mocha", "node"]
}
}