mirror/cypress
1
0
Fork 0
mirror of https://github.com/cypress-io/cypress.git synced 2026-05-02 04:50:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: route all https traffic through proxy (#8827)

Browse Source
This commit is contained in:
Ben Kucera
2020-10-20 12:20:38 -04:00
committed by GitHub
parent 07ed546a66
commit 19fdf4306a
14 changed files with 44 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/https-proxy/lib/server.js
+3 -63
View File
@@ -1,9 +1,6 @@
const _ = require('lodash')
const { agent, allowDestroy, connect } = require('@packages/network')
const { allowDestroy, connect } = require('@packages/network')
const debug = require('debug')('cypress:https-proxy')
const {
getProxyForUrl,
} = require('proxy-from-env')
const https = require('https')
const net = require('net')
const parse = require('./util/parse')
@@ -76,29 +73,16 @@ class Server {
})
}
_onFirstHeadBytes (req, browserSocket, head, options) {
let odc
_onFirstHeadBytes (req, browserSocket, head) {
debug('Got first head bytes %o', { url: req.url, head: _.chain(head).invoke('toString').slice(0, 64).join('').value() })
browserSocket.pause()
odc = options.onDirectConnection
if (odc) {
// if onDirectConnection return true
// then dont proxy, just pass this through
if (odc.call(this, req, browserSocket, head) === true) {
return this._makeDirectConnection(req, browserSocket, head)
}
debug('Not making direct connection %o', { url: req.url })
}
return this._onServerConnectData(req, browserSocket, head)
}
_onUpgrade (fn, req, browserSocket, head) {
debug('upgrade', req.url)
if (fn) {
return fn.call(this, req, browserSocket, head)
}
@@ -118,31 +102,7 @@ class Server {
}
}
_getProxyForUrl (urlStr) {
const port = Number(_.get(url.parse(urlStr), 'port'))
debug('getting proxy URL %o', { port, serverPort: this._port, sniPort: this._sniPort, url: urlStr })
if ([this._sniPort, this._port].includes(port)) {
// https://github.com/cypress-io/cypress/issues/4257
// this is a tunnel to the SNI server or to the main server,
// it should never go through a proxy
return undefined
}
return getProxyForUrl(urlStr)
}
_makeDirectConnection (req, browserSocket, head) {
const { port, hostname } = url.parse(`https://${req.url}`)
debug(`Making connection to ${hostname}:${port}`)
return this._makeConnection(browserSocket, head, port, hostname)
}
_makeConnection (browserSocket, head, port, hostname) {
let upstreamProxy
const onSocket = (err, upstreamSocket) => {
debug('received upstreamSocket callback for request %o', { port, hostname, err })
@@ -174,26 +134,6 @@ class Server {
port = '443'
}
upstreamProxy = this._getProxyForUrl(`https://${hostname}:${port}`)
if (upstreamProxy) {
// todo: as soon as all requests are intercepted, this can go away since this is just for pass-through
debug('making proxied connection %o', {
host: `${hostname}:${port}`,
proxy: upstreamProxy,
})
return agent.httpsAgent.createUpstreamProxyConnection({
proxy: upstreamProxy,
href: `https://${hostname}:${port}`,
uri: {
port,
hostname,
},
shouldRetry: true,
}, onSocket)
}
return connect.createRetryingSocket({ port, host: hostname }, onSocket)
}
packages/https-proxy/package.json
-1
View File
@@ -19,7 +19,6 @@
"fs-extra": "8.1.0",
"lodash": "4.17.19",
"node-forge": "0.10.0",
"proxy-from-env": "1.0.0",
"semaphore": "1.1.0"
},
"devDependencies": {
packages/https-proxy/test/integration/proxy_spec.js
+5 -38
View File
@@ -89,34 +89,6 @@ describe('Proxy', () => {
})
})
// this will fail due to dynamic cert
// generation when strict ssl is true
it('can pass directly through', () => {
return request({
strictSSL: false,
url: 'https://localhost:8444/replace',
proxy: 'http://localhost:3333',
})
.then((html) => {
expect(html).to.include('https server')
})
})
it('retries 5 times', function () {
this.sandbox.spy(net, 'connect')
return request({
strictSSL: false,
url: 'https://localhost:12344',
proxy: 'http://localhost:3333',
})
.then(() => {
throw new Error('should not reach')
}).catch(() => {
expect(net.connect).to.have.callCount(5)
})
})
it('closes outgoing connections when client disconnects', function () {
this.sandbox.spy(net, 'connect')
@@ -130,12 +102,8 @@ describe('Proxy', () => {
// ensure client has disconnected
expect(res.socket.destroyed).to.be.true
// ensure the outgoing socket created for this connection was destroyed
const socket = net.connect.getCalls()
.find((call) => {
return (call.args[0].port === '8444') && (call.args[0].host === 'localhost')
}).returnValue
expect(socket.destroyed).to.be.true
expect(net.connect).calledOnce
expect(net.connect.getCalls()[0].returnValue.destroyed).to.be.true
})
})
@@ -227,6 +195,7 @@ describe('Proxy', () => {
})
})
// TODO
context('with an upstream proxy', () => {
beforeEach(function () {
// PROXY vars should override npm_config vars, so set them to cause failures if they are used
@@ -300,10 +269,8 @@ describe('Proxy', () => {
expect(res.socket.destroyed).to.be.true
// ensure the outgoing socket created for this connection was destroyed
const socket = net.connect.getCalls()
.find((call) => {
return (call.args[0].port === 9001) && (call.args[0].host === 'localhost')
}).returnValue
expect(net.connect).calledOnce
const socket = net.connect.getCalls()[0].returnValue
return new Promise((resolve) => {
return socket.on('close', () => {
packages/https-proxy/test/unit/server_spec.js
+3 -30
View File
@@ -64,7 +64,7 @@ describe('lib/server', () => {
this.setup({ onError })
.then((srv) => {
srv._makeDirectConnection({ url: 'localhost:8444' }, socket, head)
srv._makeConnection(socket, head, '8444', 'localhost')
})
})
@@ -76,7 +76,7 @@ describe('lib/server', () => {
const head = {}
const onError = function (err, socket2, head2, port) {
expect(err.message).to.eq('connect ECONNREFUSED 127.0.0.1:443')
expect(err.message).to.eq('getaddrinfo ENOTFOUND %7Balgolia_application_id%7D-dsn.algolia.net')
expect(socket).to.eq(socket2)
expect(head).to.eq(head2)
@@ -89,34 +89,7 @@ describe('lib/server', () => {
this.setup({ onError })
.then((srv) => {
srv._makeDirectConnection({ url: '%7Balgolia_application_id%7D-dsn.algolia.net:443' }, socket, head)
})
})
it('with proxied connection calls options.onError with err and port and destroys the client socket', function (done) {
const socket = new EE()
socket.destroy = this.sandbox.stub()
const head = {}
const onError = function (err, socket2, head2, port) {
expect(err.message).to.eq('A connection to the upstream proxy could not be established: connect ECONNREFUSED 127.0.0.1:8444')
expect(socket).to.eq(socket2)
expect(head).to.eq(head2)
expect(port).to.eq('11111')
expect(socket.destroy).to.be.calledOnce
done()
}
process.env.HTTPS_PROXY = 'http://localhost:8444'
process.env.NO_PROXY = ''
this.setup({ onError })
.then((srv) => {
srv._makeDirectConnection({ url: 'should-not-reach.invalid:11111' }, socket, head)
srv._makeConnection(socket, head, '443', '%7Balgolia_application_id%7D-dsn.algolia.net')
})
})
})
packages/net-stubbing/lib/server/index.ts
-2
View File
@@ -8,8 +8,6 @@ export { InterceptResponse } from './intercept-response'
export { NetStubbingState } from './types'
export { isHostInterceptable } from './is-host-interceptable'
import { state } from './state'
export const netStubbingState = state
packages/net-stubbing/lib/server/is-host-interceptable.ts
-60
View File
@@ -1,60 +0,0 @@
import { NetStubbingState } from './types'
import { URL } from 'url'
import minimatch from 'minimatch'
/**
* Returns `true` if there is any chance that a request for `host` could match a route defined in `state`.
* @param host `hostname:port`
*/
export function isHostInterceptable (host: string, { routes }: Pick<NetStubbingState, 'routes'>) {
const [hostname, portStr] = host.split(':')
const port = Number(portStr)
for (const { routeMatcher } of routes) {
if (routeMatcher.port) {
if (Array.isArray(routeMatcher.port) && !routeMatcher.port.includes(port)) {
continue // excluded by port list mismatch
}
if (!Array.isArray(routeMatcher.port) && routeMatcher.port !== port) {
continue // excluded by port mismatch
}
}
if (!routeMatcher.hostname && !routeMatcher.url) {
return true // route has no constraints on host, could match
}
if (routeMatcher.hostname) {
if (routeMatcher.hostname instanceof RegExp && routeMatcher.hostname.test(hostname)) {
return true // hostname RegExp is a match
}
if (routeMatcher.hostname === hostname) {
return true // hostname is an exact match
}
}
if (routeMatcher.url) {
if (routeMatcher.url instanceof RegExp) {
return true // possible that the RegExp could match a URL
}
if (host.includes(routeMatcher.url)) {
return true // possible for substring to match
}
try {
const url = new URL(routeMatcher.url)
if (minimatch(host, url.host) || minimatch(hostname, url.hostname)) {
return true // host could match
}
} catch (e) {
return true // invalid URL, so partial URL, could possibly match
}
}
}
return false // ruled out all possibilities for a match
}
packages/net-stubbing/test/unit/is-host-interceptable-spec.ts
-123
View File
@@ -1,123 +0,0 @@
import {
isHostInterceptable,
} from '../../lib/server/is-host-interceptable'
import { expect } from 'chai'
import { RouteMatcherOptions } from '../../lib/types'
const testMatchers = (host: string, matchers: RouteMatcherOptions[], expected: boolean) => {
const routes = matchers.map((routeMatcher) => {
return { routeMatcher }
})
// @ts-ignore
expect(isHostInterceptable(host, { routes })).to.eq(expected)
}
describe('is host interceptable?', () => {
it('returns false if no routes set', () => {
testMatchers('foo.com:123', [], false)
})
it('returns false if host mismatch', () => {
testMatchers('foo.com:123', [
{
hostname: 'bar.com',
},
], false)
})
it('returns true if matcher doesn\'t constrain host or port', () => {
testMatchers('foo.com:123', [{}], true)
testMatchers('foo.com:123', [{
pathname: 'foo',
}], true)
})
it('returns true if host equals', () => {
testMatchers('foo.com:123', [
{
hostname: 'foo.com',
},
], true)
})
it('returns true if host matches regex', () => {
testMatchers('foo.com:123', [
{
hostname: /foo/,
},
], true)
})
it('returns false if port mismatch', () => {
testMatchers('foo.com:123', [
{
port: 456,
},
], false)
})
it('returns true if port equals', () => {
testMatchers('foo.com:123', [
{
port: 123,
},
], true)
})
it('returns true if port in list', () => {
testMatchers('foo.com:123', [
{
port: [123],
},
], true)
})
it('returns true if url is a RegExp', () => {
testMatchers('foo.com:123', [
{
url: /anything-is-possible/,
},
], true)
})
it('returns true if url has same hostname', () => {
testMatchers('foo.com:123', [
{
url: 'http://foo.com:123/anything/really',
},
], true)
})
it('returns true if url is a fragment glob', () => {
testMatchers('foo.com:123', [
{
url: '/foo',
},
], true)
})
it('returns true if url domain matches', () => {
testMatchers('foo.com:80', [
{
url: 'http://foo.com',
},
], true)
})
it('returns true if url domain glob matches', () => {
testMatchers('foobar.com:80', [
{
url: 'http://foo*ar.com',
},
], true)
})
it('returns false if url domain glob does not match', () => {
testMatchers('foobar.com:80', [
{
url: 'http://bar*ar.com',
},
], false)
})
})
packages/network/lib/agent.ts
+2 -2
View File
@@ -293,14 +293,14 @@ class HttpsAgent extends https.Agent {
if (originalErr) {
const err: any = new Error(`A connection to the upstream proxy could not be established: ${originalErr.message}`)
err[0] = originalErr
err.originalErr = originalErr
err.upstreamProxyConnect = true
return cb(err, undefined)
}
const onClose = () => {
triggerRetry(new Error('The upstream proxy closed the socket after connecting but before sending a response.'))
triggerRetry(new Error('ERR_EMPTY_RESPONSE: The upstream proxy closed the socket after connecting but before sending a response.'))
}
const onError = (err: Error) => {
packages/network/test/unit/agent_spec.ts
+1 -1
View File
@@ -321,7 +321,7 @@ describe('lib/agent', function () {
throw new Error('should not succeed')
})
.catch((e) => {
expect(e.message).to.eq('Error: A connection to the upstream proxy could not be established: The upstream proxy closed the socket after connecting but before sending a response.')
expect(e.message).to.eq('Error: A connection to the upstream proxy could not be established: ERR_EMPTY_RESPONSE: The upstream proxy closed the socket after connecting but before sending a response.')
return proxy.destroyAsync()
})
packages/server/lib/request.js
+5 -1
View File
@@ -88,6 +88,10 @@ const hasRetriableStatusCodeFailure = (res, retryOnStatusCodeFailure) => {
])
}
const isErrEmptyResponseError = (err) => {
return _.startsWith(err.message, 'ERR_EMPTY_RESPONSE')
}
const isRetriableError = (err = {}, retryOnNetworkFailure) => {
return _.every([
retryOnNetworkFailure,
@@ -116,7 +120,7 @@ const maybeRetryOnNetworkFailure = function (err, options = {}) {
opts.minVersion = 'TLSv1'
}
if (!isTlsVersionError && !isRetriableError(err, retryOnNetworkFailure)) {
if (!isTlsVersionError && !isErrEmptyResponseError(err.originalErr || err) && !isRetriableError(err, retryOnNetworkFailure)) {
return onElse()
}
packages/server/lib/server.js
+11 -49
View File
@@ -15,13 +15,12 @@ const compression = require('compression')
const debug = require('debug')('cypress:server:server')
const {
agent,
blocked,
concatStream,
cors,
uri,
} = require('@packages/network')
const { NetworkProxy } = require('@packages/proxy')
const { netStubbingState, isHostInterceptable } = require('@packages/net-stubbing')
const { netStubbingState } = require('@packages/net-stubbing')
const { createInitialWorkers } = require('@packages/rewriter')
const origin = require('./util/origin')
const ensureUrl = require('./util/ensure-url')
@@ -252,7 +251,7 @@ class Server {
createServer (app, config, project, request, onWarning) {
return new Promise((resolve, reject) => {
const { port, fileServerFolder, socketIoRoute, baseUrl, blockHosts } = config
const { port, fileServerFolder, socketIoRoute, baseUrl } = config
this._server = http.createServer(app)
@@ -292,44 +291,7 @@ class Server {
socket.once('upstream-connected', this._socketAllowed.add)
return this._httpsProxy.connect(req, socket, head, {
onDirectConnection: (req) => {
if (this._netStubbingState && isHostInterceptable(req.url, this._netStubbingState)) {
debug('CONNECT request may match a net-stubbing route, proxying %o', _.pick(req, 'url'))
return false
}
const urlToCheck = `https://${req.url}`
let isMatching = cors.urlMatchesOriginPolicyProps(urlToCheck, this._remoteProps)
const word = isMatching ? 'does' : 'does not'
debug(`HTTPS request ${word} match URL: ${urlToCheck} with props: %o`, this._remoteProps)
// if we are currently matching then we're
// not making a direct connection anyway
// so we only need to check this if we
// have blocked hosts and are not matching.
//
// if we have blocked hosts lets
// see if this matches - if so then
// we cannot allow it to make a direct
// connection
if (blockHosts && !isMatching) {
isMatching = blocked.matches(urlToCheck, blockHosts)
debug(`HTTPS request ${urlToCheck} matches blockHosts?`, isMatching)
}
// make a direct connection only if
// our req url does not match the origin policy
// which is the superDomain + port
return !isMatching
},
})
return this._httpsProxy.connect(req, socket, head)
})
this._server.on('upgrade', onUpgrade)
@@ -786,7 +748,6 @@ class Server {
proxyWebsockets (proxy, socketIoRoute, req, socket, head) {
// bail if this is our own namespaced socket.io request
let host; let remoteOrigin
if (req.url.startsWith(socketIoRoute)) {
if (!this._socketAllowed.isRequestAllowed(req)) {
@@ -798,13 +759,14 @@ class Server {
return
}
if ((host = req.headers.host) && this._remoteProps && (remoteOrigin = this._remoteOrigin)) {
// get the port from @_remoteProps
// get the protocol from remoteOrigin
// get the hostname from host header
const { port } = this._remoteProps
const { protocol } = url.parse(remoteOrigin)
const { hostname } = url.parse(`http://${host}`)
const host = req.headers.host
if (host) {
// get the protocol using req.connection.encrypted
// get the port & hostname from host header
const fullUrl = `${req.connection.encrypted ? 'https' : 'http'}://${host}`
const { hostname, protocol } = url.parse(fullUrl)
const { port } = cors.parseUrlIntoDomainTldPort(fullUrl)
const onProxyErr = (err, req, res) => {
return debug('Got ERROR proxying websocket connection', { err, port, protocol, hostname, req })
packages/server/test/integration/websockets_spec.js
+7 -16
View File
@@ -1,6 +1,5 @@
require('../spec_helper')
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
require('../spec_helper')
const ws = require('ws')
const httpsProxyAgent = require('https-proxy-agent')
@@ -57,18 +56,6 @@ describe('Web Sockets', () => {
})
context('proxying external websocket requests', () => {
it('ends the socket connection without remoteHost', function (done) {
this.server._onDomainSet()
const client = new ws(`ws://localhost:${cyPort}`)
return client.on('error', (err) => {
expect(err.code).to.eq('ECONNRESET')
return done()
})
})
it('sends back ECONNRESET when error upgrading', function (done) {
const agent = new httpsProxyAgent(`http://localhost:${cyPort}`)
@@ -87,7 +74,8 @@ describe('Web Sockets', () => {
})
it('proxies https messages', function (done) {
this.server._onDomainSet(`https://localhost:${wssPort}`)
const agent = new httpsProxyAgent(`http://localhost:${cyPort}`, {
})
this.wss.on('connection', (c) => {
return c.on('message', (msg) => {
@@ -95,7 +83,10 @@ describe('Web Sockets', () => {
})
})
const client = new ws(`ws://localhost:${cyPort}`)
const client = new ws(`wss://localhost:${wssPort}`, {
rejectUnauthorized: false,
agent,
})
client.on('message', (data) => {
expect(data).to.eq('response:foo')
packages/server/test/support/fixtures/projects/e2e/cypress/integration/websockets_spec.coffee
+4 -5
View File
@@ -9,21 +9,20 @@ shouldCloseUrlWithCode = (win, url, code) ->
if evt.code is code
resolve()
else
reject("websocket connection should have been closed with code #{code} for url: #{url} but was instead closed with code: #{evt.code}")
reject(new Error "websocket connection should have been closed with code #{code} for url: #{url} but was instead closed with code: #{evt.code}")
ws.onopen = (evt) ->
reject("websocket connection should not have opened for url: #{url}")
reject(new Error "websocket connection should not have opened for url: #{url}")
describe "websockets", ->
it "does not crash", ->
cy.visit("http://localhost:3038/foo")
cy.log("should not crash on ECONNRESET websocket upgrade")
cy.window().then (win) ->
## Firefox should close with code 1015 when using SSL, chrome should close with 1006
## see https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent
Cypress.Promise.all([
shouldCloseUrlWithCode(win, "ws://localhost:3038/websocket", 1006)
shouldCloseUrlWithCode(win, "wss://localhost:3040/websocket", if Cypress.browser.family is 'firefox' then 1015 else 1006)
shouldCloseUrlWithCode(win, "wss://localhost:3040/websocket", 1006)
])
cy.log("should be able to send websocket messages")
@@ -35,7 +34,7 @@ describe "websockets", ->
ws = new win.WebSocket("ws://localhost:3039/")
ws.onmessage = (evt) ->
resolve(evt.data)
ws.onerror = reject
ws.onerror = -> reject(new Error 'connection failed, check console for error')
ws.onopen = ->
ws.send("foo")
.should("eq", "foobar")
packages/server/test/unit/server_spec.js
+3
View File
@@ -355,6 +355,9 @@ describe('lib/server', () => {
this.server._onDomainSet('https://www.google.com')
const req = {
connection: {
encrypted: true,
},
url: '/',
headers: {
host: 'www.google.com',